From: Hou Tao <houtao1@huawei.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 -------------------------------- Add tools/lib/nds with the NDS userspace API. The library registers device memory, submits asynchronous P2P I/O, and harvests completion events through the nds_p2p UAPI. Signed-off-by: Hou Tao <houtao1@huawei.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- tools/include/uapi/linux/nds_p2p.h | 132 ++++ tools/lib/nds/Makefile | 13 + tools/lib/nds/file_p2p/.gitignore | 8 + tools/lib/nds/file_p2p/Makefile | 35 + tools/lib/nds/file_p2p/file_p2p_api.c | 136 ++++ tools/lib/nds/file_p2p/file_p2p_api.h | 36 + tools/lib/nds/file_p2p/nds_api.c | 860 ++++++++++++++++++++++ tools/lib/nds/file_p2p/nds_api.h | 249 +++++++ tools/lib/nds/file_p2p/nds_api_internal.h | 12 + tools/lib/nds/file_p2p/p2p_common.c | 69 ++ tools/lib/nds/file_p2p/p2p_common.h | 16 + tools/lib/nds/file_p2p/py_file_p2p_api.c | 320 ++++++++ tools/lib/nds/file_p2p/py_nds_api.c | 576 +++++++++++++++ tools/lib/nds/file_p2p/setup.py | 32 + 14 files changed, 2494 insertions(+) create mode 100644 tools/include/uapi/linux/nds_p2p.h create mode 100644 tools/lib/nds/Makefile create mode 100644 tools/lib/nds/file_p2p/.gitignore create mode 100644 tools/lib/nds/file_p2p/Makefile create mode 100644 tools/lib/nds/file_p2p/file_p2p_api.c create mode 100644 tools/lib/nds/file_p2p/file_p2p_api.h create mode 100644 tools/lib/nds/file_p2p/nds_api.c create mode 100644 tools/lib/nds/file_p2p/nds_api.h create mode 100644 tools/lib/nds/file_p2p/nds_api_internal.h create mode 100644 tools/lib/nds/file_p2p/p2p_common.c create mode 100644 tools/lib/nds/file_p2p/p2p_common.h create mode 100644 tools/lib/nds/file_p2p/py_file_p2p_api.c create mode 100644 tools/lib/nds/file_p2p/py_nds_api.c create mode 100644 tools/lib/nds/file_p2p/setup.py diff --git a/tools/include/uapi/linux/nds_p2p.h b/tools/include/uapi/linux/nds_p2p.h new file mode 100644 index 000000000000..f69bd610a41f --- /dev/null +++ b/tools/include/uapi/linux/nds_p2p.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef NDS_P2P_H_ +#define NDS_P2P_H_ + +#include <linux/types.h> + +/* + * UAPI v2: PCI P2PDMA + asynchronous VFS I/O. + * + * All clients open /dev/p2p_device. An fd learns its provider from the first + * registered or one-shot buffer. Later buffers on that fd must use the same + * provider. No request or device-node name carries a PCI device id. + */ +/* GET_INFO uses bits 15:8 for major and bits 7:0 for minor. */ +#define P2P_UAPI_VERSION_MAJOR 2U +#define P2P_UAPI_VERSION_MINOR 0U +#define P2P_UAPI_VERSION ((P2P_UAPI_VERSION_MAJOR << 8) | P2P_UAPI_VERSION_MINOR) + +struct p2p_iov { + __u64 addr; /* HBM addr */ + __u32 size; + __u32 reserved; /* must be zero */ +}; + +#define P2P_IO_F_REGISTERED_MEM (1U << 0) +#define P2P_IO_F_MASK P2P_IO_F_REGISTERED_MEM + +#define P2P_IO_READ 0U +#define P2P_IO_WRITE 1U + +/* One logical I/O moves at most this many bytes (the kernel MAX_RW_COUNT). */ +#define P2P_MAX_IO_BYTES 0x7ffff000U + +/* + * Fixed-size request; iov is a userspace pointer. One batch item is one + * logical I/O at byte offset @offset in @file_fd, covering all iov segments + * and completing as one event carrying user_data. A transfer shorter than the + * iov total completes with -EIO. + */ +struct p2p_io_param { + unsigned int op; /* P2P_IO_READ / P2P_IO_WRITE */ + unsigned int flags; /* P2P_IO_F_* */ + int file_fd; /* regular file or block device; no O_APPEND */ + int host_pid; /* one-shot buffer owner; 0 if registered */ + __u64 mem_handle; /* from IOCTL_REGISTER_MEM, or 0 */ + __u64 user_data; /* echo'd in the completion event */ + __u64 offset; /* byte offset in file_fd */ + __u64 iov; /* userspace pointer to struct p2p_iov[] */ + unsigned int iov_nr; + unsigned int reserved; /* must be zero */ + __u64 reserved2[3]; /* must be zero */ +}; + +#define P2P_MAX_IO_NR 1024U + +/* + * Submit independent p2p_io_param records in array order. The ioctl stops at + * the first item that cannot be accepted and returns the accepted prefix, or + * a negative errno when the first item fails. Each accepted item produces one + * completion event. The ioctl may sleep in the filesystem. + */ +struct p2p_io_batch_param { + __u64 items; /* userspace pointer to p2p_io_param[] */ + __u32 nr; + __u32 reserved; +}; + +struct p2p_io_event { + __u64 user_data; + __s64 res; /* 0 on success or -errno */ + __u64 reserved[4]; +}; + +struct p2p_getevents_param { + __s32 min_nr; + __s32 max_nr; + __s64 timeout_ns; /* <0 infinite, 0 nonblock, >0 relative */ + __u64 events; /* userspace pointer to struct p2p_io_event[] */ +}; + +struct p2p_mem_unregister_param { + __u64 mem_handle; + __u64 reserved[3]; +}; + +struct p2p_mem_range { + __u64 addr; + __u64 size; +}; + +#define P2P_MEM_MAX_RANGES 8192U + +/* + * Register several non-overlapping VA ranges under one handle. Registered I/O + * with that handle may address any registered range. Each IOV must fit in one + * translation segment; crossing a segment boundary fails with -ERANGE. Adjacent + * ranges merge only at a page-aligned boundary with the same page size. Every + * range must be in one provider's window and match the fd's cached provider. + * A different provider fails with -EXDEV; an address outside every provider + * fails with -EFAULT. + * IOCTL_UNREGISTER_MEM releases the registration. + */ +struct p2p_mem_register_param { + __u64 ranges; /* userspace pointer to struct p2p_mem_range[] */ + __u32 nr; /* 1..P2P_MEM_MAX_RANGES */ + __u32 flags; /* must be zero */ + __u64 reserved[3]; /* must be zero */ + __u64 mem_handle; /* output */ +}; + +/* Validate a file, directory or block-device fd's filesystem and P2PDMA queue. */ +struct p2p_check_fd_param { + __s32 fd; + __u32 flags; /* must be zero */ + __u64 reserved; /* must be zero */ +}; + +/* Output only. Feature flags and reserved fields are zero for version 2.0. */ +struct p2p_device_info { + __u32 version; /* major in bits 15:8, minor in bits 7:0; upper bits zero */ + __u32 flags; + __u64 reserved[3]; +}; + +#define IOCTL_GET_INFO _IOR('k', 1, struct p2p_device_info) +#define IOCTL_CHECK_FD _IOW('k', 2, struct p2p_check_fd_param) +#define IOCTL_REGISTER_MEM _IOWR('k', 3, struct p2p_mem_register_param) +#define IOCTL_UNREGISTER_MEM _IOW('k', 4, struct p2p_mem_unregister_param) +#define IOCTL_SUBMIT_IO _IOWR('k', 5, struct p2p_io_batch_param) +#define IOCTL_GET_IO_EVENTS _IOWR('k', 6, struct p2p_getevents_param) + +#endif diff --git a/tools/lib/nds/Makefile b/tools/lib/nds/Makefile new file mode 100644 index 000000000000..dd691a8312f4 --- /dev/null +++ b/tools/lib/nds/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# NDS userspace library (tools/lib/nds) + +.PHONY: all clean lib + +all: lib + +lib: + $(MAKE) -C file_p2p + +clean: + $(MAKE) -C file_p2p clean diff --git a/tools/lib/nds/file_p2p/.gitignore b/tools/lib/nds/file_p2p/.gitignore new file mode 100644 index 000000000000..997fbf047228 --- /dev/null +++ b/tools/lib/nds/file_p2p/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +build/ +file_p2p*.so +nds*.so +libfile_p2p.so +libnds.so* +.*.d +/nds_example diff --git a/tools/lib/nds/file_p2p/Makefile b/tools/lib/nds/file_p2p/Makefile new file mode 100644 index 000000000000..ddb716d63221 --- /dev/null +++ b/tools/lib/nds/file_p2p/Makefile @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: GPL-2.0 +CC ?= cc +CFLAGS ?= -O2 -Wall -Wextra -Werror -g -ggdb -fno-omit-frame-pointer +UAPI_DIR ?= $(abspath ../../../include/uapi) +CPPFLAGS += -I. -I.. -I$(UAPI_DIR) + +NDS_API_VER := $(shell sed -n 's/^[[:space:]]*#define[[:space:]]\+NDS_API_VERSION[[:space:]]\+\([0-9]\+\)u\?.*/\1/p' nds_api.h) +NDS_MAJOR := $(shell echo $$(($(NDS_API_VER) >> 8))) +NDS_MINOR := $(shell echo $$(($(NDS_API_VER) & 255))) +NDS_VERSION := $(NDS_MAJOR).$(NDS_MINOR) +NDS_LIB := libnds.so.$(NDS_VERSION) +NDS_SONAME := libnds.so.$(NDS_VERSION) + +NDS_SRCS := nds_api.c p2p_common.c +NDS_OBJS := $(NDS_SRCS:.c=.o) +NDS_DEPS := $(addprefix .,$(NDS_OBJS:.o=.d)) + +.PHONY: all binding clean + +all: $(NDS_LIB) + +binding: + python3 setup.py build_ext --inplace + +$(NDS_LIB): $(NDS_OBJS) + $(CC) -shared -Wl,-soname,$(NDS_SONAME) -o $@ $(NDS_OBJS) -lpthread + ln -sf libnds.so.$(NDS_VERSION) libnds.so + +%.o: %.c + $(CC) $(CPPFLAGS) $(CFLAGS) -fPIC -MMD -MP -MT $@ -MF .$*.d -c -o $@ $< + +-include $(NDS_DEPS) + +clean: + rm -f libnds.so libnds.so.* $(NDS_OBJS) $(NDS_DEPS) diff --git a/tools/lib/nds/file_p2p/file_p2p_api.c b/tools/lib/nds/file_p2p/file_p2p_api.c new file mode 100644 index 000000000000..e275c0c489e0 --- /dev/null +++ b/tools/lib/nds/file_p2p/file_p2p_api.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/ioctl.h> +#include <errno.h> +#include <limits.h> + +#include <linux/nds_p2p.h> +#include "p2p_common.h" +#include "file_p2p_api.h" + +int close_p2p_fd(int dev_fd) +{ + return close(dev_fd); +} + +int new_p2p_fd(void) +{ + return p2p_open_dev(); +} + +int check_fd(int dev_fd, int fd) +{ + struct p2p_check_fd_param param = { + .fd = fd, + .flags = 0, + .reserved = 0, + }; + + if (ioctl(dev_fd, IOCTL_CHECK_FD, ¶m) < 0) + return -errno; + return 0; +} + +int unregister_mem(int dev_fd, const struct p2p_mem_unregister_param *param) +{ + if (ioctl(dev_fd, IOCTL_UNREGISTER_MEM, param) < 0) + return -errno; + return 0; +} + +int register_mem(int dev_fd, struct p2p_mem_register_param *param) +{ + if (ioctl(dev_fd, IOCTL_REGISTER_MEM, param) < 0) + return -errno; + return 0; +} + +int rw_file(int dev_fd, const struct io_parameter *param) +{ + struct p2p_io_batch_param batch; + struct p2p_io_param *io; + unsigned long io_size; + const char *name; + int file_fd = -1; + int err = 0; + + if (!param) + return -EINVAL; + if (param->op != P2P_IO_READ && param->op != P2P_IO_WRITE) + return -EINVAL; + if (param->flags & ~P2P_IO_F_MASK) + return -EOPNOTSUPP; + if (param->flags & P2P_IO_F_REGISTERED_MEM) { + if (!param->mem_handle || param->host_pid) + return -EINVAL; + } else if (param->host_pid < 0) { + return -EINVAL; + } + + err = p2p_get_iov_size(param->iov, param->iov_nr, &io_size); + if (err) + return err; + if (ULONG_MAX - param->file_offset < io_size) + return -EOVERFLOW; + if (io_size > P2P_MAX_IO_BYTES) + return -EINVAL; + + name = param->file_name; + file_fd = open(name, (param->op == P2P_IO_WRITE ? O_WRONLY : O_RDONLY) | O_CLOEXEC); + if (file_fd < 0) { + err = -errno; + fprintf(stderr, "open %s failed, errno: %d\n", name, err); + return err; + } + + io = calloc(1, sizeof(*io)); + if (io == NULL) { + err = -ENOMEM; + fprintf(stderr, "calloc p2p_io_param failed, errno: %d\n", err); + goto close_fd_out; + } + + io->op = param->op; + io->file_fd = file_fd; + io->flags = param->flags; + if (param->flags & P2P_IO_F_REGISTERED_MEM) { + io->host_pid = 0; + io->mem_handle = param->mem_handle; + } else { + io->host_pid = param->host_pid ? param->host_pid : getpid(); + io->mem_handle = 0; + } + io->offset = param->file_offset; + io->iov = (uint64_t)(uintptr_t)param->iov; + io->iov_nr = param->iov_nr; + + batch.nr = 1; + batch.reserved = 0; + batch.items = (uint64_t)(uintptr_t)io; + err = ioctl(dev_fd, IOCTL_SUBMIT_IO, &batch); + if (err < 0) { + err = -errno; + fprintf(stderr, "%s file ioctl failed, errno: %d\n", + param->op == P2P_IO_WRITE ? "write" : "read", -err); + goto free_io_out; + } + if (err != 1) { + fprintf(stderr, "%s file ioctl accepted %d requests, expected 1\n", + param->op == P2P_IO_WRITE ? "write" : "read", err); + err = -EIO; + goto free_io_out; + } + err = 0; + +free_io_out: + free(io); +close_fd_out: + close(file_fd); + return err; +} diff --git a/tools/lib/nds/file_p2p/file_p2p_api.h b/tools/lib/nds/file_p2p/file_p2p_api.h new file mode 100644 index 000000000000..0526124b3b57 --- /dev/null +++ b/tools/lib/nds/file_p2p/file_p2p_api.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __FILE_P2P_API_H__ +#define __FILE_P2P_API_H__ + +#include <stdint.h> + +#include <linux/nds_p2p.h> + +struct io_parameter { + unsigned int op; + const char *file_name; + unsigned long file_offset; + struct p2p_iov *iov; + unsigned int iov_nr; + unsigned int flags; + int host_pid; + uint64_t mem_handle; +}; + +/* Open /dev/p2p_device. */ +int new_p2p_fd(void); +int close_p2p_fd(int dev_fd); +/* Validate target properties before submitting I/O. */ +int check_fd(int dev_fd, int fd); +/* Release a registration with unregister_mem() on its returned handle. */ +int register_mem(int dev_fd, struct p2p_mem_register_param *param); +int unregister_mem(int dev_fd, const struct p2p_mem_unregister_param *param); +/* + * file_offset is a byte offset; the kernel maps it to underlying blocks. + * Filesystem alignment and CPU-access errors arrive in completion events. + * Success means accepted. Collect results with IOCTL_GET_IO_EVENTS before + * reusing buffers. Final close waits for I/O and discards remaining events. + */ +int rw_file(int dev_fd, const struct io_parameter *param); + +#endif diff --git a/tools/lib/nds/file_p2p/nds_api.c b/tools/lib/nds/file_p2p/nds_api.c new file mode 100644 index 000000000000..2025ac34df72 --- /dev/null +++ b/tools/lib/nds/file_p2p/nds_api.c @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * NDS — a GDS-aligned userspace API for NPU HBM ↔ NVMe P2P I/O. + * + * NDS does not link against file_p2p_api; it shares p2p_common with file_p2p + * for device open only. The two userspace APIs evolve independently on top + * of the same kernel UAPI (linux/nds_p2p.h). + */ +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <string.h> +#include <errno.h> +#include <stddef.h> +#include <pthread.h> +#include <sys/types.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/ioctl.h> +#include <time.h> +#include <limits.h> +#include <stdint.h> + +#include "nds_api_internal.h" +#include <linux/nds_p2p.h> +#include "p2p_common.h" + +_Static_assert(sizeof(struct nds_io_vec) == sizeof(struct p2p_iov), + "nds_io_vec must match p2p_iov for zero-copy submission"); +_Static_assert(NDS_IO_MAX_IO_CNT <= P2P_MAX_IO_NR, + "NDS submit size must fit the kernel batch ioctl"); +_Static_assert(_Alignof(struct nds_io_vec) == _Alignof(struct p2p_iov), + "nds_io_vec alignment must match p2p_iov"); +_Static_assert(offsetof(struct nds_io_vec, buf_addr) == + offsetof(struct p2p_iov, addr) && + offsetof(struct nds_io_vec, buf_len) == + offsetof(struct p2p_iov, size) && + offsetof(struct nds_io_vec, reserved) == + offsetof(struct p2p_iov, reserved), + "nds_io_vec field layout must match p2p_iov"); +_Static_assert(sizeof(struct nds_io_event) == sizeof(struct p2p_io_event), + "nds_io_event must match p2p_io_event for zero-copy getevents"); +_Static_assert(sizeof(struct nds_io_event) == 48, + "nds_io_event / p2p_io_event are user_data + res + reserved[4]"); +_Static_assert(offsetof(struct nds_io_event, user_data) == + offsetof(struct p2p_io_event, user_data) && + offsetof(struct nds_io_event, res) == + offsetof(struct p2p_io_event, res) && + offsetof(struct nds_io_event, reserved) == + offsetof(struct p2p_io_event, reserved), + "nds_io_event field layout must match p2p_io_event"); + +_Static_assert(sizeof(struct nds_mem_range) == sizeof(struct p2p_mem_range) && + offsetof(struct nds_mem_range, addr) == + offsetof(struct p2p_mem_range, addr) && + offsetof(struct nds_mem_range, size) == + offsetof(struct p2p_mem_range, size), + "nds_mem_range must match p2p_mem_range for zero-copy registration"); +_Static_assert(NDS_MEM_MAX_RANGES <= P2P_MEM_MAX_RANGES, + "NDS group size must fit the kernel registration ioctl"); + +/* One registration: a single range or a group, in the caller's order. */ +struct nds_reg_entry { + struct nds_reg_entry *next; + uint64_t handle; + uint32_t nr; + struct nds_mem_range ranges[]; +}; + +/* Submit-side VA index entry; ranges never overlap across registrations. */ +struct nds_mem_ent { + uint64_t start; + uint64_t end; + uint64_t handle; +}; + +struct nds_state { + int book_fd; + /* Serializes register/unregister, including their ioctls. */ + pthread_mutex_t reg_lock; + struct nds_reg_entry *regs; + /* + * Sorted per-range index, used only to re-check overlap on the next + * register/unregister. Write-side bookkeeping: touched solely under + * reg_lock, never read by a submitter. + */ + struct nds_mem_ent *idx; + size_t idx_nr; + /* + * Submit-side lookup index. Runs of consecutive @idx entries (sorted + * by start) that share a handle are merged into one + * [first.start, last.end) entry, so one group with no neighboring + * registration — or any lone registration — is a single lookup entry + * and always hits the per-thread last-hit cache below. Writers hold + * reg_lock and replace the whole array under idx_lock; submitters + * normally avoid the lock entirely via a thread-local snapshot (see + * find_registered_mem). + */ + pthread_rwlock_t idx_lock; + struct nds_mem_ent *lookup_idx; + size_t lookup_nr; + /* + * Bumped (release) on every publish, add or remove. Readers load it + * (acquire) to invalidate both the last-hit cache and thread-local + * snapshots, so a removed range is never returned once the removing + * nds_unregister_mem*() call has returned on the same thread. + */ + uint64_t idx_gen; +}; + +static struct nds_state g_state = { + .book_fd = -1, + .reg_lock = PTHREAD_MUTEX_INITIALIZER, + .idx_lock = PTHREAD_RWLOCK_INITIALIZER, +}; + +/* + * Last index hit of this thread. A hit is reused only while idx_gen is + * unchanged, so it never outlives the unregister of its range. + */ +static __thread struct { + uint64_t gen; + struct nds_mem_ent ent; +} tls_mem_hit; + +/* + * Per-thread copy of the coalesced lookup index, refreshed under idx_lock + * only when stale (gen mismatch) and then binary-searched without any lock. + * idx/cap survive refreshes that do not grow nr, so steady-state misses never + * allocate. Only .idx is heap memory; it is freed at thread exit through + * snap_key below, since __thread storage itself is not otherwise notified of + * thread exit. + */ +struct nds_mem_snapshot { + uint64_t gen; + struct nds_mem_ent *idx; + size_t nr; + size_t cap; +}; + +static __thread struct nds_mem_snapshot tls_snap; +static pthread_key_t snap_key; +static pthread_once_t snap_key_once = PTHREAD_ONCE_INIT; + +/* Destructor for snap_key: frees this thread's snapshot buffer at exit. */ +static void nds_snap_destroy(void *p) +{ + struct nds_mem_snapshot *snap = p; + + free(snap->idx); +} + +static void nds_snap_key_create(void) +{ + pthread_key_create(&snap_key, nds_snap_destroy); +} + +static int is_init(void) +{ + return g_state.book_fd >= 0; +} + +/* ------------------------------------------------------------------ */ +/* Public NDS API */ +/* ------------------------------------------------------------------ */ + +int nds_init(struct nds_init_param *param) +{ + /* Not thread-safe: see nds_api.h. Concurrent init is undefined. */ + if (!param || param->flags || param->reserved1 || param->reserved2 || + is_init() || param->reserved3[0] || param->reserved3[1] || + param->reserved3[2] || param->reserved3[3]) + return -EINVAL; + + g_state.book_fd = p2p_open_dev(); + if (g_state.book_fd < 0) + return g_state.book_fd; + g_state.regs = NULL; + g_state.idx = NULL; + g_state.idx_nr = 0; + g_state.lookup_idx = NULL; + g_state.lookup_nr = 0; + param->version = NDS_API_VERSION; + return 0; +} + +int nds_register_fs(const struct nds_fs_desc *desc) +{ + uint32_t i; + + if (!desc || !desc->fs_fd || !desc->fs_fd_cnt || desc->reserved) + return -EINVAL; + for (i = 0; i < desc->fs_fd_cnt; i++) { + struct p2p_check_fd_param check = { .fd = desc->fs_fd[i] }; + + if (ioctl(g_state.book_fd, IOCTL_CHECK_FD, &check) < 0) + return -errno; + } + return 0; +} + +int nds_unregister_fs(const struct nds_fs_desc *desc) +{ + (void)desc; + return 0; +} + +/* + * Swap in a new lookup index and return the old one for the caller to free. + * Always bumps idx_gen (release), after the swap, whether ranges were added + * or removed: readers only ever consult the lookup index through this gen, + * so an added range must be visible exactly when a removed one disappears. + * @old_nr_out, if non-NULL, receives the old index's length, needed by a + * caller that may have to republish it on a later failure. + */ +static struct nds_mem_ent *nds_publish_index(struct nds_mem_ent *idx, size_t nr, + size_t *old_nr_out) +{ + struct nds_mem_ent *old; + + pthread_rwlock_wrlock(&g_state.idx_lock); + old = g_state.lookup_idx; + if (old_nr_out) + *old_nr_out = g_state.lookup_nr; + g_state.lookup_idx = idx; + g_state.lookup_nr = nr; + __atomic_add_fetch(&g_state.idx_gen, 1, __ATOMIC_RELEASE); + pthread_rwlock_unlock(&g_state.idx_lock); + return old; +} + +int nds_exit(void) +{ + struct nds_reg_entry *r, *next; + + pthread_mutex_lock(&g_state.reg_lock); + r = g_state.regs; + g_state.regs = NULL; + free(g_state.idx); + g_state.idx = NULL; + g_state.idx_nr = 0; + free(nds_publish_index(NULL, 0, NULL)); + pthread_mutex_unlock(&g_state.reg_lock); + + for (; r; r = next) { + struct p2p_mem_unregister_param up = { + .mem_handle = r->handle, + }; + next = r->next; + if (ioctl(g_state.book_fd, IOCTL_UNREGISTER_MEM, &up) < 0) + fprintf(stderr, + "nds: unregister memory addr=0x%llx size=%llu ranges=%u handle=%llu failed %d\n", + (unsigned long long)r->ranges[0].addr, + (unsigned long long)r->ranges[0].size, r->nr, + (unsigned long long)r->handle, -errno); + free(r); + } + + close(g_state.book_fd); + g_state.book_fd = -1; + return 0; +} + +static int nds_mem_ent_cmp(const void *a, const void *b) +{ + const struct nds_mem_ent *ea = a; + const struct nds_mem_ent *eb = b; + + if (ea->start < eb->start) + return -1; + return ea->start > eb->start; +} + +/* + * Build the current per-range index (g_state.idx) plus @ranges with handle 0. + * Ranges must be nonempty and must not overlap each other (-EINVAL) or + * another registration (-EEXIST). Called with reg_lock held, so g_state.idx + * is stable. This index only drives the overlap check above; submitters look + * up the coalesced copy built by nds_coalesce_fill() below. + */ +static int nds_index_add(const struct nds_mem_range *ranges, uint32_t nr, + struct nds_mem_ent **idx_out, size_t *nr_out) +{ + size_t total = g_state.idx_nr + nr; + struct nds_mem_ent *idx; + size_t i; + + idx = malloc(total * sizeof(*idx)); + if (!idx) + return -ENOMEM; + if (g_state.idx_nr) + memcpy(idx, g_state.idx, g_state.idx_nr * sizeof(*idx)); + for (i = 0; i < nr; i++) { + struct nds_mem_ent *ent = &idx[g_state.idx_nr + i]; + + if (!ranges[i].size) { + free(idx); + return -EINVAL; + } + if (ranges[i].addr > UINT64_MAX - ranges[i].size) { + free(idx); + return -EOVERFLOW; + } + ent->start = ranges[i].addr; + ent->end = ranges[i].addr + ranges[i].size; + ent->handle = 0; + } + + qsort(idx, total, sizeof(*idx), nds_mem_ent_cmp); + for (i = 1; i < total; i++) { + int ret; + + if (idx[i].start >= idx[i - 1].end) + continue; + fprintf(stderr, + "nds: register memory range 0x%llx+0x%llx overlaps 0x%llx+0x%llx\n", + (unsigned long long)idx[i].start, + (unsigned long long)(idx[i].end - idx[i].start), + (unsigned long long)idx[i - 1].start, + (unsigned long long)(idx[i - 1].end - idx[i - 1].start)); + /* New ranges still have handle 0. */ + ret = !idx[i].handle && !idx[i - 1].handle ? -EINVAL : -EEXIST; + free(idx); + return ret; + } + + *idx_out = idx; + *nr_out = total; + return 0; +} + +/* Build the current index without @handle. Called with reg_lock held. */ +static int nds_index_del(uint64_t handle, struct nds_mem_ent **idx_out, size_t *nr_out) +{ + struct nds_mem_ent *idx; + size_t nr = 0; + size_t i; + + idx = malloc((g_state.idx_nr ? g_state.idx_nr : 1) * sizeof(*idx)); + if (!idx) + return -ENOMEM; + for (i = 0; i < g_state.idx_nr; i++) { + if (g_state.idx[i].handle != handle) + idx[nr++] = g_state.idx[i]; + } + *idx_out = idx; + *nr_out = nr; + return 0; +} + +/* + * Merge consecutive entries of @idx (sorted by start, @nr of them) that + * share a handle into @out, whose capacity must be at least @nr: coalescing + * only ever shrinks. A gap inside a group has no entry of its own once + * merged, so an iov landing there now resolves to the group's handle instead + * of missing the index; the kernel still rejects it, with -ERANGE rather + * than the library's -EINVAL. Returns the number of entries written. + */ +static size_t nds_coalesce_fill(const struct nds_mem_ent *idx, size_t nr, + struct nds_mem_ent *out) +{ + size_t n = 0; + size_t i; + + for (i = 0; i < nr; i++) { + if (n && out[n - 1].handle == idx[i].handle) + out[n - 1].end = idx[i].end; + else + out[n++] = idx[i]; + } + return n; +} + +static int nds_register_ranges(const struct nds_mem_range *ranges, uint32_t nr) +{ + struct nds_reg_entry *r; + struct nds_mem_ent *idx; + struct nds_mem_ent *lookup; + struct nds_mem_ent *old_idx; + struct p2p_mem_register_param p = { 0 }; + size_t idx_nr; + size_t lookup_nr; + uint64_t handle; + size_t i; + int ret; + + r = malloc(sizeof(*r) + (size_t)nr * sizeof(r->ranges[0])); + if (!r) + return -ENOMEM; + memcpy(r->ranges, ranges, (size_t)nr * sizeof(r->ranges[0])); + r->nr = nr; + + pthread_mutex_lock(&g_state.reg_lock); + ret = nds_index_add(ranges, nr, &idx, &idx_nr); + if (ret) + goto unlock; + + /* + * Preallocate the coalesced buffer before the ioctl: once the kernel + * has accepted the registration, publishing the new indexes must not + * be able to fail. + */ + lookup = malloc((idx_nr ? idx_nr : 1) * sizeof(*lookup)); + if (!lookup) { + free(idx); + ret = -ENOMEM; + goto unlock; + } + + /* Register all ranges through the same ioctl. */ + p.ranges = (uint64_t)(uintptr_t)ranges; + p.nr = nr; + ret = ioctl(g_state.book_fd, IOCTL_REGISTER_MEM, &p); + handle = p.mem_handle; + if (ret < 0) { + ret = -errno; + fprintf(stderr, + "nds: register memory addr=0x%llx size=%llu ranges=%u failed %d\n", + (unsigned long long)ranges[0].addr, + (unsigned long long)ranges[0].size, nr, ret); + free(lookup); + free(idx); + goto unlock; + } + + for (i = 0; i < idx_nr; i++) { + if (!idx[i].handle) + idx[i].handle = handle; + } + lookup_nr = nds_coalesce_fill(idx, idx_nr, lookup); + r->handle = handle; + r->next = g_state.regs; + g_state.regs = r; + r = NULL; + old_idx = g_state.idx; + g_state.idx = idx; + g_state.idx_nr = idx_nr; + free(old_idx); + free(nds_publish_index(lookup, lookup_nr, NULL)); + ret = 0; + +unlock: + pthread_mutex_unlock(&g_state.reg_lock); + free(r); + return ret; +} + +/* + * Unregister the registration whose first range starts at ranges[0].addr. The + * caller must pass the same ranges in the same order as at registration. + */ +static int nds_unregister_ranges(const struct nds_mem_range *ranges, uint32_t nr) +{ + struct nds_reg_entry *r, **prev; + struct p2p_mem_unregister_param up = { 0 }; + struct nds_mem_ent *old_idx; + struct nds_mem_ent *old_lookup; + struct nds_mem_ent *idx; + struct nds_mem_ent *lookup; + size_t old_idx_nr; + size_t old_lookup_nr; + size_t idx_nr; + size_t lookup_nr; + int ret; + + pthread_mutex_lock(&g_state.reg_lock); + for (prev = &g_state.regs; *prev; prev = &(*prev)->next) { + if ((*prev)->ranges[0].addr == ranges[0].addr) + break; + } + if (!*prev) { + ret = -ENOENT; + goto unlock; + } + r = *prev; + if (r->nr != nr || memcmp(r->ranges, ranges, (size_t)nr * sizeof(ranges[0]))) { + fprintf(stderr, + "nds: unregister memory addr=0x%llx size=%llu ranges=%u does not match registered size=%llu ranges=%u\n", + (unsigned long long)ranges[0].addr, + (unsigned long long)ranges[0].size, nr, + (unsigned long long)r->ranges[0].size, r->nr); + ret = -EINVAL; + goto unlock; + } + + ret = nds_index_del(r->handle, &idx, &idx_nr); + if (ret) + goto unlock; + lookup = malloc((idx_nr ? idx_nr : 1) * sizeof(*lookup)); + if (!lookup) { + free(idx); + ret = -ENOMEM; + goto unlock; + } + lookup_nr = nds_coalesce_fill(idx, idx_nr, lookup); + + /* Hide the ranges before the kernel drops the handle. */ + old_idx = g_state.idx; + old_idx_nr = g_state.idx_nr; + g_state.idx = idx; + g_state.idx_nr = idx_nr; + old_lookup = nds_publish_index(lookup, lookup_nr, &old_lookup_nr); + + up.mem_handle = r->handle; + if (ioctl(g_state.book_fd, IOCTL_UNREGISTER_MEM, &up) < 0) { + ret = -errno; + fprintf(stderr, + "nds: unregister memory addr=0x%llx size=%llu ranges=%u handle=%llu failed %d\n", + (unsigned long long)r->ranges[0].addr, + (unsigned long long)r->ranges[0].size, r->nr, + (unsigned long long)r->handle, ret); + /* Keep the registration tracked so the caller can retry. */ + g_state.idx = old_idx; + g_state.idx_nr = old_idx_nr; + free(idx); + free(nds_publish_index(old_lookup, old_lookup_nr, NULL)); + goto unlock; + } + + *prev = r->next; + free(r); + free(old_idx); + free(old_lookup); + +unlock: + pthread_mutex_unlock(&g_state.reg_lock); + return ret; +} + +static int nds_mem_desc_valid(const struct nds_mem_desc *desc) +{ + return desc && desc->ranges && desc->nr && + desc->nr <= NDS_MEM_MAX_RANGES && !desc->flags && + !desc->reserved[0] && !desc->reserved[1] && + !desc->reserved[2] && !desc->reserved[3]; +} + +int nds_register_mem(const struct nds_mem_desc *desc) +{ + if (!nds_mem_desc_valid(desc)) + return -EINVAL; + return nds_register_ranges(desc->ranges, desc->nr); +} + +int nds_unregister_mem(const struct nds_mem_desc *desc) +{ + if (!nds_mem_desc_valid(desc)) + return -EINVAL; + return nds_unregister_ranges(desc->ranges, desc->nr); +} + +int nds_io_new_ctx(const struct nds_io_ctx_param *param, struct nds_io_ctx **ctx) +{ + struct nds_io_ctx *c; + int fd; + + if (!param || !ctx) + return -EINVAL; + if (!param->max_io_cnt || param->max_io_cnt > NDS_IO_MAX_IO_CNT || + param->flags || + param->reserved) + return -EINVAL; + + c = calloc(1, sizeof(*c)); + if (!c) + return -ENOMEM; + + fd = p2p_open_dev(); + if (fd < 0) { + free(c); + return fd; + } + c->p2p_fd = fd; + *ctx = c; + return 0; +} + +int nds_io_destroy_ctx(struct nds_io_ctx *ctx) +{ + int close_err = 0; + + if (!ctx) + return -EINVAL; + /* + * Catches a closed-but-not-yet-freed ctx. A second destroy after + * free() is still use-after-free (caller bug / UB). + */ + if (ctx->p2p_fd < 0) + return -EINVAL; + + /* The final close waits for outstanding I/O and releases its events. */ + if (close(ctx->p2p_fd) < 0) { + close_err = -errno; + fprintf(stderr, "nds: destroy_ctx close failed %d\n", close_err); + } + ctx->p2p_fd = -1; + free(ctx); + return close_err; +} + +/* Binary search a sorted nds_mem_ent array for the entry covering @addr. */ +static int nds_mem_ent_lookup(const struct nds_mem_ent *idx, size_t nr, uint64_t addr, + struct nds_mem_ent *ent_out) +{ + size_t lo = 0, hi = nr; + + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + + if (idx[mid].start <= addr) + lo = mid + 1; + else + hi = mid; + } + if (lo && addr < idx[lo - 1].end) { + *ent_out = idx[lo - 1]; + return 0; + } + return -EINVAL; +} + +/* + * Map the start of a registered request to its handle. Only the first IOV + * selects the registration; the kernel checks every IOV against it. + * + * Fast path: the per-thread last-hit cache. Slow path: refresh this thread's + * snapshot of the coalesced lookup index (under idx_lock) only when it is + * stale, then binary-search the snapshot with no lock at all. Because a + * group coalesces to one entry spanning its whole VA range, an address in a + * gap between a group's ranges now resolves to the group's handle instead of + * missing here; the kernel still rejects the actual iov with -ERANGE. + */ +static int find_registered_mem(uint64_t addr, uint64_t *handle_out) +{ + /* Read gen first so removal invalidates the cached index. */ + uint64_t gen = __atomic_load_n(&g_state.idx_gen, __ATOMIC_ACQUIRE); + struct nds_mem_snapshot *snap = &tls_snap; + struct nds_mem_ent ent; + int ret; + + if (tls_mem_hit.gen == gen && tls_mem_hit.ent.handle && + addr >= tls_mem_hit.ent.start && addr < tls_mem_hit.ent.end) { + *handle_out = tls_mem_hit.ent.handle; + return 0; + } + + if (snap->gen != gen) { + pthread_rwlock_rdlock(&g_state.idx_lock); + if (g_state.lookup_nr > snap->cap) { + struct nds_mem_ent *buf = malloc(g_state.lookup_nr * sizeof(*buf)); + bool first = !snap->cap; + + if (!buf) { + pthread_rwlock_unlock(&g_state.idx_lock); + return -ENOMEM; + } + free(snap->idx); + snap->idx = buf; + snap->cap = g_state.lookup_nr; + /* Register cleanup for this thread's buffer. */ + if (first) { + pthread_once(&snap_key_once, nds_snap_key_create); + pthread_setspecific(snap_key, snap); + } + } + if (g_state.lookup_nr) + memcpy(snap->idx, g_state.lookup_idx, + g_state.lookup_nr * sizeof(*snap->idx)); + snap->nr = g_state.lookup_nr; + /* + * Record the gen read inside the lock: it is what the copy + * above actually reflects, not the possibly staler gen loaded + * above. + */ + snap->gen = __atomic_load_n(&g_state.idx_gen, __ATOMIC_ACQUIRE); + pthread_rwlock_unlock(&g_state.idx_lock); + } + + ret = nds_mem_ent_lookup(snap->idx, snap->nr, addr, &ent); + if (ret) + return ret; + tls_mem_hit.gen = snap->gen; + tls_mem_hit.ent = ent; + *handle_out = ent.handle; + return 0; +} + +/* + * nds_io_vec is binary-identical to p2p_iov (see the _Static_assert block + * above), so the shared alignment/size validation of p2p_get_iov_size() + * applies directly; this also sums the iov into *io_size. + */ +static int validate_nds_iov(const struct nds_io_cb *cb, unsigned long *io_size) +{ + if (!cb->iov || !cb->iov_cnt || cb->iov_cnt > NDS_IO_MAX_IOV) + return -EINVAL; + return p2p_get_iov_size((const struct p2p_iov *)cb->iov, cb->iov_cnt, io_size); +} + +static int nds_iocb_to_io_param(const struct nds_io_cb *cb, int file_fd, struct p2p_io_param *io) +{ + unsigned long io_size; + uint64_t mem_handle = 0; + int err; + + if (cb->opcode > NDS_IO_OP_PWRITE || (cb->rw_flags & ~NDS_IO_F_MASK)) + return -EOPNOTSUPP; + + err = validate_nds_iov(cb, &io_size); + if (err) + return err; + if (io_size > P2P_MAX_IO_BYTES) + return -EINVAL; + if (cb->offset > ULONG_MAX || io_size > ULONG_MAX - (unsigned long)cb->offset) + return -EOVERFLOW; + + if (cb->rw_flags & NDS_IO_F_REGISTERED_MEM) { + if (cb->host_pid) + return -EINVAL; + err = find_registered_mem(cb->iov[0].buf_addr, &mem_handle); + if (err) + return err; + } else if (cb->host_pid < 0) { + return -EINVAL; + } + + memset(io, 0, sizeof(*io)); + io->op = (cb->opcode == NDS_IO_OP_PWRITE) ? P2P_IO_WRITE : P2P_IO_READ; + io->file_fd = file_fd; + io->user_data = cb->user_data; + io->offset = cb->offset; + io->iov = (uint64_t)(uintptr_t)cb->iov; + io->iov_nr = cb->iov_cnt; + + if (cb->rw_flags & NDS_IO_F_REGISTERED_MEM) { + io->flags = P2P_IO_F_REGISTERED_MEM; + io->host_pid = 0; + io->mem_handle = mem_handle; + } else { + io->flags = 0; + io->host_pid = cb->host_pid ? cb->host_pid : getpid(); + io->mem_handle = 0; + } + + return 0; +} + +static int nds_build_one_iocb(const struct nds_io_cb *cb, struct p2p_io_param *param) +{ + int file_fd = cb->obj.fd; + int ret; + + if (cb->reserved[0] || cb->reserved[1] || cb->obj.reserved || + file_fd < 0) + return -EINVAL; + + ret = nds_iocb_to_io_param(cb, file_fd, param); + if (ret) + fprintf(stderr, "nds: translate iocb failed %d\n", ret); + return ret; +} + +int nds_io_submit(struct nds_io_ctx *ctx, int nr, + const struct nds_io_cb *iocb) +{ + struct p2p_io_batch_param batch = { 0 }; + struct p2p_io_param *items; + unsigned int built = 0; + int build_err = 0; + int ret; + + if (!ctx || !iocb) + return -EINVAL; + if (nr < 0) + return -EINVAL; + if (!nr) + return 0; + if ((unsigned int)nr > NDS_IO_MAX_IO_CNT) + return -EINVAL; + + items = calloc((size_t)nr, sizeof(*items)); + if (!items) + return -ENOMEM; + + /* + * Build the valid prefix in iocb order, then submit its independent + * requests through one ioctl. Translation and kernel acceptance are + * both fail-stop: return the accepted prefix, or -errno if item zero + * fails. + */ + while (built < (unsigned int)nr) { + build_err = nds_build_one_iocb(&iocb[built], &items[built]); + if (build_err) + break; + built++; + } + if (!built) { + ret = build_err; + goto out; + } + + batch.nr = built; + batch.items = (uint64_t)(uintptr_t)items; + ret = ioctl(ctx->p2p_fd, IOCTL_SUBMIT_IO, &batch); + if (ret < 0) { + ret = -errno; + fprintf(stderr, "nds: submit ioctl failed %d\n", ret); + } else if (!ret) { + ret = -EIO; + } + +out: + free(items); + return ret; +} + +int nds_io_getevents(struct nds_io_ctx *ctx, int min_nr, + int nr, struct nds_io_event *events, + struct timespec *timeout) +{ + struct p2p_getevents_param param = { 0 }; + int ret; + + if (!ctx) + return -EINVAL; + if (min_nr < 0 || nr < 0 || min_nr > nr) + return -EINVAL; + if (!nr) + return 0; + if (!events) + return -EINVAL; + + /* + * nds_io_event is binary-identical to p2p_io_event (48B); pass the + * caller's buffer straight to the ioctl — no intermediate copy. + */ + param.min_nr = min_nr; + param.max_nr = nr; + if (timeout) { + if (timeout->tv_sec < 0 || timeout->tv_nsec < 0 || + timeout->tv_nsec >= 1000000000L || + (uint64_t)timeout->tv_sec > + (uint64_t)(INT64_MAX - timeout->tv_nsec) / + UINT64_C(1000000000)) + return -EINVAL; + param.timeout_ns = (__s64)timeout->tv_sec * 1000000000LL + + timeout->tv_nsec; + } else { + param.timeout_ns = -1; + } + param.events = (uint64_t)(uintptr_t)events; + + ret = ioctl(ctx->p2p_fd, IOCTL_GET_IO_EVENTS, ¶m); + if (ret < 0) { + ret = -errno; + fprintf(stderr, "nds: getevents ioctl failed %d\n", ret); + return ret; + } + return ret; +} diff --git a/tools/lib/nds/file_p2p/nds_api.h b/tools/lib/nds/file_p2p/nds_api.h new file mode 100644 index 000000000000..154ebc0b60b9 --- /dev/null +++ b/tools/lib/nds/file_p2p/nds_api.h @@ -0,0 +1,249 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NDS_API_H__ +#define __NDS_API_H__ + +#include <stdint.h> +#include <sys/types.h> +#include <time.h> /* struct timespec for nds_io_getevents */ + +/* + * NDS — userspace API for NPU HBM ↔ NVMe P2P I/O (GDS-like, async). + * + * Typical usage: + * + * 1. Initialize NDS, then register each filesystem as it becomes available: + * struct nds_init_param ip = { 0 }; + * int fds[] = { file_fd }; + * struct nds_fs_desc fs = { + * .fs_fd = fds, + * .fs_fd_cnt = 1, + * }; + * nds_init(&ip); + * assert(ip.version == NDS_API_VERSION); + * nds_register_fs(&fs); + * Call nds_register_fs() again to add filesystems after initialization. + * Each device fd learns its NPU from the first buffer used with that fd. + * + * 2. (Optional) Register HBM / CMB windows once, as a group (a single + * buffer is a group with nr == 1): + * struct nds_mem_desc mem = { .ranges = ranges, .nr = nr }; + * nds_register_mem(&mem); + * Later I/Os that use those VAs set NDS_IO_F_REGISTERED_MEM; the library + * looks up the handle — callers never see mem_handle. + * + * 3. Create an I/O context and submit work (Linux-AIO style): + * struct nds_io_ctx *ctx; + * nds_io_new_ctx(&(struct nds_io_ctx_param){ .max_io_cnt = N }, &ctx); + * struct nds_io_vec iov = { + * .buf_addr = ..., + * .buf_len = ..., + * }; + * + * struct nds_io_cb cb = { + * .opcode = NDS_IO_OP_PREAD, + * .obj = { .fd = file_fd }, + * .offset = ..., + * .iov = &iov, + * .iov_cnt = 1, + * .rw_flags = 0, // or NDS_IO_F_REGISTERED_MEM + * .host_pid = 0, // must be 0 with REGISTERED_MEM + * .user_data = cookie, // echoed in completion events + * }; + * nds_io_submit(ctx, 1, &cb); + * nds_io_getevents(ctx, 1, 1, events, NULL); + * + * 4. Tear down: destroy ctx → unregister mem → nds_exit(). + * + * Conventions: + * - All functions return 0 / positive count on success, or -errno on failure. + * - I/O is async only (no synchronous pread/pwrite helpers). + * - Targets are open file or block-device FDs. Paths are not accepted. + * - Offsets / lengths / buffer addresses must be sector-aligned (512 B); + * a regular-file write also needs filesystem-block alignment, and one + * logical I/O is limited to P2P_MAX_IO_BYTES. + * - reserved fields must be zero. + */ + +#define NDS_API_VERSION 2u +#define NDS_IO_MAX_IO_CNT 1024u +#define NDS_IO_MAX_IOV 65536u + +#define NDS_MEM_MAX_RANGES 8192u + +struct nds_io_ctx; + +struct nds_init_param { + uint32_t flags; /* must be 0 */ + uint32_t reserved1; /* must be 0 */ + uint32_t version; /* output: NDS_API_VERSION on success */ + uint32_t reserved2; /* must be 0 */ + uint64_t reserved3[4]; /* must be zero when unused */ +}; + +/* File, directory or block-device fds for filesystem and queue validation. */ +struct nds_fs_desc { + int32_t *fs_fd; + uint32_t fs_fd_cnt; + uint32_t reserved; +}; + +struct nds_mem_range { + uint64_t addr; + uint64_t size; +}; + +struct nds_mem_desc { + const struct nds_mem_range *ranges; + uint32_t nr; + uint32_t flags; /* must be 0 */ + uint64_t reserved[4]; /* must be zero */ +}; + +/* + * max_io_cnt: accepts 1..NDS_IO_MAX_IO_CNT as a queue-size hint but does + * not enforce an outstanding-I/O budget in the library. flags and reserved + * must be zero. + */ +struct nds_io_ctx_param { + uint32_t max_io_cnt; + uint32_t flags; + uint64_t reserved; +}; + +/* I/O target: an open file or block-device fd. */ +struct nds_io_obj { + uint64_t reserved; /* must be zero */ + union { + uint64_t _data; + int32_t fd; /* open file / block-device fd */ + }; +}; + +#define NDS_IO_OP_PREAD 0u /* read one or more buffers */ +#define NDS_IO_OP_PWRITE 1u /* write one or more buffers */ + +/* Buffer was previously registered with nds_register_mem(). */ +#define NDS_IO_F_REGISTERED_MEM (1u << 0) +#define NDS_IO_F_MASK NDS_IO_F_REGISTERED_MEM + +struct nds_io_vec { + uint64_t buf_addr; + uint32_t buf_len; + uint32_t reserved; /* must be zero; layout matches p2p_iov */ +}; + +/* + * One async I/O request. Fill an array of these and pass to nds_io_submit(). + * + * iov / iov_cnt describe the HBM (or CMB) VA windows for this transfer. + * With NDS_IO_F_REGISTERED_MEM, iov[0] selects a region previously passed to + * nds_register_mem(); the kernel requires every vector to fit that + * region or group. Otherwise buffers are one-shot. + * + * offset is the byte offset in obj.fd; the kernel maps it to the underlying + * blocks (no FIEMAP / fallocate step in userspace). Filesystem alignment, + * CPU-access fallbacks and short transfers report errors in completion events. + * Callers must exclusively manage files and keep mappings stable through I/O. + */ +struct nds_io_cb { + uint32_t opcode; /* NDS_IO_OP_* */ + uint32_t rw_flags; /* NDS_IO_F_* bitmask */ + struct nds_io_obj obj; + uint64_t offset; /* byte offset in the target object */ + uint64_t user_data; /* opaque cookie, returned in events */ + const struct nds_io_vec *iov; + uint32_t iov_cnt; + /* + * Owner of one-shot HBM VAs. 0 = current process. + * Must be 0 when NDS_IO_F_REGISTERED_MEM is set. + */ + int32_t host_pid; + uint64_t reserved[2]; /* must be zero */ +}; + +/* + * One completion. Layout matches kernel struct p2p_io_event (48B). + * res is success(==0) or -errno; reserved must be zero. + */ +struct nds_io_event { + uint64_t user_data; + int64_t res; + uint64_t reserved[4]; +}; + +/* + * Process-wide initialization and teardown. + * nds_init opens /dev/p2p_device and uses a separate fd for filesystem and + * memory-registration bookkeeping. It is not thread-safe or re-entrant; call + * it once from one thread. Do not call nds_exit concurrently with + * nds_register_fs() or any other NDS operation. + */ +int nds_init(struct nds_init_param *param); +int nds_exit(void); + +int nds_register_fs(const struct nds_fs_desc *desc); +int nds_unregister_fs(const struct nds_fs_desc *desc); + +/* + * Pin / unpin several separately allocated VA ranges (for example, KV-cache + * blocks) as one registration group. One NDS_IO_F_REGISTERED_MEM iocb may + * then scatter across any ranges of the group: iov[0] selects the group, and + * every vector must fit in one kernel translation segment (-ERANGE otherwise). + * Adjacent ranges merge into one segment only at a page-aligned boundary with + * the same devmm page size. Use separate vectors for different segments. + * + * desc->nr must be 1..NDS_MEM_MAX_RANGES. flags and reserved must be zero. + * Ranges need not be sorted but must be nonempty and must not overlap each + * other (-EINVAL) or any other registration (-EEXIST). A single buffer is a + * group with nr == 1; there is no separate single-range call. + * Unregister with the same ranges in the same order; a mismatch returns + * -EINVAL. There is no partial unregister. + * + * The library's lookup resolves iov[0] to the group as a whole, including any + * gap between its ranges that no other registration covers: an iov[0] placed + * in such a gap still selects the group, and the kernel then rejects the + * request with -ERANGE rather than the library returning -EINVAL. + */ +int nds_register_mem(const struct nds_mem_desc *desc); +int nds_unregister_mem(const struct nds_mem_desc *desc); + +/* + * Create / destroy a per-queue I/O context. Destroy closes the fd, waits for + * outstanding I/O and frees @ctx. The caller owns the pointer until destroy; + * double destroy (or any use after destroy) is undefined behavior. Do not call + * nds_io_getevents concurrently with destroy on the same ctx: the caller must + * stop all other uses of the context before destroy. + */ +int nds_io_new_ctx(const struct nds_io_ctx_param *param, + struct nds_io_ctx **ctx); +int nds_io_destroy_ctx(struct nds_io_ctx *ctx); + +/* + * Submit nr control blocks. Returns the number accepted, or -errno. + * + * Each iocb is one logical I/O: its iov[] is scatter-gather on a single + * obj.fd / offset and completes as one event (user_data). Different iocbs + * may target different files; the library submits the valid prefix as + * independent IOCTL_SUBMIT_IO items without cross-iocb merging. + * + * Fail-stop like Linux io_submit: on a mid-batch failure return the count + * of iocbs already accepted (positive); the failed iocb's errno is not + * returned. If nothing was accepted, return -errno. Accepted I/Os are not + * rolled back — reap them with nds_io_getevents(). + */ +int nds_io_submit(struct nds_io_ctx *ctx, int nr, + const struct nds_io_cb *iocb); + +/* + * Wait for at least min_nr completions, up to nr. + * timeout NULL = wait forever; {0,0} = non-blocking. + * On success returns the number of events filled (>= 0); on failure -errno. + * When no I/Os are outstanding, returns 0 immediately even if min_nr > 0 + * (does not block until timeout like Linux AIO on an empty ctx). + * Must not run concurrently with nds_io_destroy_ctx on the same ctx. + */ +int nds_io_getevents(struct nds_io_ctx *ctx, int min_nr, + int nr, struct nds_io_event *events, + struct timespec *timeout); + +#endif diff --git a/tools/lib/nds/file_p2p/nds_api_internal.h b/tools/lib/nds/file_p2p/nds_api_internal.h new file mode 100644 index 000000000000..a217c07d0541 --- /dev/null +++ b/tools/lib/nds/file_p2p/nds_api_internal.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef NDS_API_INTERNAL_H_ +#define NDS_API_INTERNAL_H_ + +#include "nds_api.h" + +/* Private layout shared by the library internals and white-box tests. */ +struct nds_io_ctx { + int p2p_fd; +}; + +#endif diff --git a/tools/lib/nds/file_p2p/p2p_common.c b/tools/lib/nds/file_p2p/p2p_common.c new file mode 100644 index 000000000000..39822040ca86 --- /dev/null +++ b/tools/lib/nds/file_p2p/p2p_common.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> +#include <limits.h> +#include <stdint.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/ioctl.h> + +#include <linux/nds_p2p.h> +#include "p2p_common.h" + +#define P2P_SECTOR_SIZE 512U +#define P2P_DEV_PATH "/dev/p2p_device" + +int p2p_open_dev(void) +{ + struct p2p_device_info info = { 0 }; + int dev_fd; + int err; + + dev_fd = open(P2P_DEV_PATH, O_RDWR | O_CLOEXEC); + if (dev_fd < 0) { + err = errno == ENOENT ? -ENODEV : -errno; + fprintf(stderr, "open %s failed, errno: %d\n", P2P_DEV_PATH, err); + return err; + } + + if (ioctl(dev_fd, IOCTL_GET_INFO, &info) < 0) { + err = errno == ENOTTY ? -EPROTONOSUPPORT : -errno; + goto close_fd; + } + if (((info.version >> 8) & 0xffU) != P2P_UAPI_VERSION_MAJOR) { + err = -EPROTONOSUPPORT; + goto close_fd; + } + return dev_fd; + +close_fd: + close(dev_fd); + return err; +} + +int p2p_get_iov_size(const struct p2p_iov *iov, unsigned int iov_nr, unsigned long *size_out) +{ + unsigned long size = 0; + unsigned int i; + + if (!iov || !iov_nr) + return -EINVAL; + + for (i = 0; i < iov_nr; i++) { + if (!iov[i].size || iov[i].reserved || + ((iov[i].addr | iov[i].size) & (P2P_SECTOR_SIZE - 1)) || + UINT64_MAX - iov[i].addr < iov[i].size || ULONG_MAX - size < iov[i].size) + return -EINVAL; + size += iov[i].size; + } + + *size_out = size; + return 0; +} diff --git a/tools/lib/nds/file_p2p/p2p_common.h b/tools/lib/nds/file_p2p/p2p_common.h new file mode 100644 index 000000000000..7ecaed36730a --- /dev/null +++ b/tools/lib/nds/file_p2p/p2p_common.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __P2P_COMMON_H__ +#define __P2P_COMMON_H__ + +#include <linux/nds_p2p.h> + +/* + * Open /dev/p2p_device O_RDWR|O_CLOEXEC. Returns fd or -errno (-ENODEV if + * absent, -EPROTONOSUPPORT for an incompatible UAPI). + */ +int p2p_open_dev(void); + +/* Sum iov[].size with sector-alignment checks. Returns 0 or -errno. */ +int p2p_get_iov_size(const struct p2p_iov *iov, unsigned int iov_nr, unsigned long *size_out); + +#endif diff --git a/tools/lib/nds/file_p2p/py_file_p2p_api.c b/tools/lib/nds/file_p2p/py_file_p2p_api.c new file mode 100644 index 000000000000..d0f989b05cea --- /dev/null +++ b/tools/lib/nds/file_p2p/py_file_p2p_api.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <limits.h> +#include <pthread.h> +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "file_p2p_api.h" + +static int parse_iovs(PyObject *object, struct p2p_iov **iov_out, + unsigned int *iov_nr_out) +{ + struct p2p_iov *iov; + PyObject *sequence; + Py_ssize_t count; + Py_ssize_t i; + + sequence = PySequence_Fast(object, "IOVs must be a sequence"); + if (!sequence) + return -1; + count = PySequence_Fast_GET_SIZE(sequence); + if (count <= 0 || count > UINT_MAX || + (size_t)count > SIZE_MAX / sizeof(*iov)) { + PyErr_SetString(PyExc_ValueError, + "IOVs must be a non-empty bounded sequence"); + Py_DECREF(sequence); + return -1; + } + + iov = calloc(count, sizeof(*iov)); + if (!iov) { + PyErr_NoMemory(); + Py_DECREF(sequence); + return -1; + } + + for (i = 0; i < count; i++) { + PyObject *entry; + PyObject **items; + unsigned long long addr; + unsigned long long size; + + entry = PySequence_Fast(PySequence_Fast_GET_ITEM(sequence, i), + "IOV must be (addr, size)"); + if (!entry) + goto error; + if (PySequence_Fast_GET_SIZE(entry) != 2) { + PyErr_SetString(PyExc_ValueError, + "IOV must be (addr, size)"); + Py_DECREF(entry); + goto error; + } + items = PySequence_Fast_ITEMS(entry); + addr = PyLong_AsUnsignedLongLong(items[0]); + size = PyLong_AsUnsignedLongLong(items[1]); + Py_DECREF(entry); + if (PyErr_Occurred()) + goto error; + if (size > UINT32_MAX) { + PyErr_SetString(PyExc_ValueError, + "IOV size exceeds the 32-bit UAPI limit"); + goto error; + } + iov[i].addr = addr; + iov[i].size = (uint32_t)size; + iov[i].reserved = 0; + } + + Py_DECREF(sequence); + *iov_out = iov; + *iov_nr_out = count; + return 0; + +error: + free(iov); + Py_DECREF(sequence); + return -1; +} + +static PyObject *py_rw_file(PyObject *Py_UNUSED(self), PyObject *args) +{ + int dev_fd = 0; + unsigned int op = 0; + const char *file_name = NULL; + unsigned long file_offset = 0; + struct p2p_iov *iov = NULL; + unsigned int iov_nr = 0; + PyObject *py_iovs = NULL; + long long mem_handle = 0; + unsigned int flags = 0; + int host_pid = 0; + int ret = 0; + + if (!PyArg_ParseTuple(args, "iIskOLI|i", &dev_fd, &op, &file_name, &file_offset, + &py_iovs, &mem_handle, &flags, &host_pid)) + return NULL; + if (parse_iovs(py_iovs, &iov, &iov_nr)) + return NULL; + + struct io_parameter param = { + .op = op, + .host_pid = host_pid, + .file_name = file_name, + .file_offset = file_offset, + .iov = iov, + .iov_nr = iov_nr, + .flags = flags, + .mem_handle = (uint64_t)mem_handle, + }; + + Py_BEGIN_ALLOW_THREADS + ret = rw_file(dev_fd, ¶m); + Py_END_ALLOW_THREADS + + free(iov); + return PyLong_FromLong((long)ret); +} + +static PyObject *py_check_fd(PyObject *Py_UNUSED(self), PyObject *args) +{ + int dev_fd; + int fd; + int ret; + + if (!PyArg_ParseTuple(args, "ii", &dev_fd, &fd)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = check_fd(dev_fd, fd); + Py_END_ALLOW_THREADS + return PyLong_FromLong((long)ret); +} + +static int parse_mem_ranges(PyObject *object, struct p2p_mem_range **ranges_out, + uint32_t *nr_out) +{ + struct p2p_mem_range *ranges; + PyObject *sequence; + Py_ssize_t count; + Py_ssize_t i; + + sequence = PySequence_Fast(object, "ranges must be a sequence of (addr, size)"); + if (!sequence) + return -1; + count = PySequence_Fast_GET_SIZE(sequence); + if (count <= 0 || count > P2P_MEM_MAX_RANGES) { + PyErr_SetString(PyExc_ValueError, "range count out of range"); + Py_DECREF(sequence); + return -1; + } + + ranges = calloc((size_t)count, sizeof(*ranges)); + if (!ranges) { + PyErr_NoMemory(); + Py_DECREF(sequence); + return -1; + } + + for (i = 0; i < count; i++) { + unsigned long long addr; + unsigned long long size; + + if (!PyArg_ParseTuple(PySequence_Fast_GET_ITEM(sequence, i), "KK", + &addr, &size)) { + free(ranges); + Py_DECREF(sequence); + return -1; + } + ranges[i].addr = addr; + ranges[i].size = size; + } + + Py_DECREF(sequence); + *ranges_out = ranges; + *nr_out = (uint32_t)count; + return 0; +} + +static PyObject *py_register_mem(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct p2p_mem_register_param param = { }; + struct p2p_mem_range *ranges; + PyObject *ranges_py; + int dev_fd; + int ret; + + if (!PyArg_ParseTuple(args, "iO", &dev_fd, &ranges_py)) + return NULL; + if (parse_mem_ranges(ranges_py, &ranges, ¶m.nr)) + return NULL; + param.ranges = (uint64_t)(uintptr_t)ranges; + + Py_BEGIN_ALLOW_THREADS + ret = register_mem(dev_fd, ¶m); + Py_END_ALLOW_THREADS + free(ranges); + + if (ret) + Py_RETURN_NONE; + return PyLong_FromLongLong((long long)param.mem_handle); +} + +static PyObject *py_unregister_mem(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct p2p_mem_unregister_param param = { }; + long long mem_handle; + int dev_fd; + int ret; + + if (!PyArg_ParseTuple(args, "iL", &dev_fd, &mem_handle)) + return NULL; + param.mem_handle = (uint64_t)mem_handle; + + Py_BEGIN_ALLOW_THREADS + ret = unregister_mem(dev_fd, ¶m); + Py_END_ALLOW_THREADS + + return PyLong_FromLong((long)ret); +} + +static PyObject *py_new_p2p_fd(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(ignored)) +{ + int ret = new_p2p_fd(); + + return PyLong_FromLong((long)ret); +} + +static PyObject *py_close_p2p_fd(PyObject *Py_UNUSED(self), PyObject *args) +{ + int dev_fd = 0; + + if (!PyArg_ParseTuple(args, "i", &dev_fd)) { + return NULL; + } + + close_p2p_fd(dev_fd); + + Py_RETURN_NONE; +} + +static PyMethodDef FileP2PMethods[] = { + { "rw_file", py_rw_file, METH_VARARGS, + "rw_file(dev_fd, op, file_name, file_offset, iovs, mem_handle, flags" + "[, host_pid]) -> int\n\n" + "Read from or write to a p2p I/O target.\n" + "\n" + "Parameters:\n" + " dev_fd (int): File descriptor of p2p device.\n" + " op (int): P2P_IO_READ or P2P_IO_WRITE.\n" + " file_name (str): Name of the I/O target.\n" + " file_offset (int): Starting file offset.\n" + " iovs (sequence): Device-memory (address, size) pairs.\n" + " mem_handle (int): Registered-memory handle, or zero.\n" + " flags (int): P2P I/O flags.\n" + " host_pid (int, optional): PID owning unregistered IOV addresses;\n" + " zero or omitted selects the current process. Must be zero for\n" + " registered memory.\n" + "\n" + "Returns:\n" + " int: 0 on success, non-zero on error.\n" }, + { "register_mem", py_register_mem, METH_VARARGS, + "register_mem(dev_fd, ranges) -> int | None\n\n" + "ranges: sequence of (addr, size) registered under one handle.\n" + "Return the memory handle; the handle may be negative.\n" + "Returns None on failure.\n" }, + { "unregister_mem", py_unregister_mem, METH_VARARGS, + "unregister_mem(dev_fd, mem_handle) -> int\n\n" + "Synchronously unregister a device-memory handle.\n" }, + { "new_p2p_fd", py_new_p2p_fd, METH_NOARGS, + "new_p2p_fd() -> int\n\n" + "New file descriptor on /dev/p2p_device.\n" + "\n" + "Returns:\n" + " int: File descriptor on success, negative errno on error.\n" }, + { "check_fd", py_check_fd, METH_VARARGS, + "check_fd(dev_fd, fd) -> int\n\n" + "Validate target properties before I/O. Returns 0 or a negative errno.\n" }, + { "close_p2p_fd", py_close_p2p_fd, METH_VARARGS, + "close_p2p_fd(dev_fd) -> None\n\n" + "Close p2p device file descriptor.\n" + "\n" + "Parameters:\n" + " dev_fd (int): File descriptor of p2p device.\n" + "\n" + "Returns:\n" + " None\n" }, + { NULL, NULL, 0, NULL } +}; + +static struct PyModuleDef file_p2p_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "file_p2p", + .m_doc = "p2p file access", + .m_size = -1, + .m_methods = FileP2PMethods, +}; + +PyMODINIT_FUNC PyInit_file_p2p(void) +{ + PyObject *module; + + module = PyModule_Create(&file_p2p_module); + if (!module) + return NULL; + if (PyModule_AddIntConstant(module, "P2P_IO_F_REGISTERED_MEM", + P2P_IO_F_REGISTERED_MEM)) { + Py_DECREF(module); + return NULL; + } + if (PyModule_AddIntConstant(module, "P2P_IO_READ", P2P_IO_READ) || + PyModule_AddIntConstant(module, "P2P_IO_WRITE", P2P_IO_WRITE)) { + Py_DECREF(module); + return NULL; + } + return module; +} diff --git a/tools/lib/nds/file_p2p/py_nds_api.c b/tools/lib/nds/file_p2p/py_nds_api.c new file mode 100644 index 000000000000..63627c3b627a --- /dev/null +++ b/tools/lib/nds/file_p2p/py_nds_api.c @@ -0,0 +1,576 @@ +// SPDX-License-Identifier: GPL-2.0 +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <math.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> +#include <time.h> + +#include "nds_api_internal.h" +#include "p2p_dev_uapi.h" + +/* + * Opaque ctx wrapper: Python holds the pointer as a PyCapsule named + * "nds_io_ctx". + */ +static const char *nds_ctx_capsule_name = "nds_io_ctx"; + +static void nds_ctx_capsule_destructor(PyObject *capsule) +{ + struct nds_io_ctx *ctx = PyCapsule_GetPointer(capsule, nds_ctx_capsule_name); + + if (ctx) + nds_io_destroy_ctx(ctx); +} + +static struct nds_io_ctx *nds_ctx_from_obj(PyObject *obj) +{ + struct nds_io_ctx *ctx; + + if (!PyCapsule_CheckExact(obj)) { + PyErr_SetString(PyExc_TypeError, "ctx must be an nds_io_ctx capsule"); + return NULL; + } + ctx = PyCapsule_GetPointer(obj, nds_ctx_capsule_name); + if (!ctx) + PyErr_SetString(PyExc_ValueError, "invalid or destroyed nds_io_ctx"); + return ctx; +} + +static int parse_nds_iov(PyObject *obj, struct nds_io_vec **iov_out, + uint32_t *iov_cnt_out) +{ + struct nds_io_vec *iov; + PyObject *seq; + Py_ssize_t count; + Py_ssize_t i; + + seq = PySequence_Fast(obj, "iov must be a sequence of (buf_addr, buf_len)"); + if (!seq) + return -1; + count = PySequence_Fast_GET_SIZE(seq); + if (count <= 0 || count > NDS_IO_MAX_IOV) { + Py_DECREF(seq); + PyErr_SetString(PyExc_ValueError, "iov count out of range"); + return -1; + } + iov = calloc((size_t)count, sizeof(*iov)); + if (!iov) { + Py_DECREF(seq); + PyErr_NoMemory(); + return -1; + } + for (i = 0; i < count; i++) { + unsigned long long addr; + unsigned int len; + + if (!PyArg_ParseTuple(PySequence_Fast_GET_ITEM(seq, i), "KI", + &addr, &len)) { + free(iov); + Py_DECREF(seq); + return -1; + } + iov[i].buf_addr = addr; + iov[i].buf_len = len; + } + Py_DECREF(seq); + *iov_out = iov; + *iov_cnt_out = (uint32_t)count; + return 0; +} + +static int py_parse_fs_desc(PyObject *fd_seq, struct nds_fs_desc *desc, + int32_t **fds_out) +{ + PyObject *seq = NULL; + int32_t *fds = NULL; + Py_ssize_t count; + Py_ssize_t i; + + seq = PySequence_Fast(fd_seq, "fs_fds must be a sequence of ints"); + if (!seq) + return -1; + count = PySequence_Fast_GET_SIZE(seq); + if (count < 0 || count > UINT32_MAX) { + Py_DECREF(seq); + PyErr_SetString(PyExc_ValueError, "fs_fds count out of range"); + return -1; + } + fds = calloc((size_t)(count ? count : 1), sizeof(*fds)); + if (!fds) { + Py_DECREF(seq); + PyErr_NoMemory(); + return -1; + } + for (i = 0; i < count; i++) { + long v = PyLong_AsLong(PySequence_Fast_GET_ITEM(seq, i)); + + if (PyErr_Occurred() || v < 0 || v > INT32_MAX) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_ValueError, + "invalid fd in fs_fds"); + free(fds); + Py_DECREF(seq); + return -1; + } + fds[i] = (int32_t)v; + } + Py_DECREF(seq); + + desc->fs_fd = fds; + desc->fs_fd_cnt = (uint32_t)count; + desc->reserved = 0; + *fds_out = fds; + return 0; +} + +static PyObject *py_nds_init(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct nds_init_param param = { }; + int ret; + + if (!PyArg_ParseTuple(args, "")) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = nds_init(¶m); + Py_END_ALLOW_THREADS + return PyLong_FromLong(ret); +} + +static PyObject *py_nds_register_fs(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct nds_fs_desc desc = { }; + PyObject *fd_seq; + int32_t *fds; + int ret; + + if (!PyArg_ParseTuple(args, "O", &fd_seq)) + return NULL; + if (py_parse_fs_desc(fd_seq, &desc, &fds)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = nds_register_fs(&desc); + Py_END_ALLOW_THREADS + free(fds); + return PyLong_FromLong(ret); +} + +static PyObject *py_nds_unregister_fs(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *ignored; + + if (!PyArg_ParseTuple(args, "O", &ignored)) + return NULL; + return PyLong_FromLong(nds_unregister_fs(NULL)); +} + +static PyObject *py_nds_exit(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(ignored)) +{ + int ret; + + Py_BEGIN_ALLOW_THREADS + ret = nds_exit(); + Py_END_ALLOW_THREADS + return PyLong_FromLong(ret); +} + +static int parse_nds_mem_ranges(PyObject *obj, struct nds_mem_range **ranges_out, + uint32_t *nr_out) +{ + struct nds_mem_range *ranges; + PyObject *seq; + Py_ssize_t count; + Py_ssize_t i; + + seq = PySequence_Fast(obj, "ranges must be a sequence of (addr, size)"); + if (!seq) + return -1; + count = PySequence_Fast_GET_SIZE(seq); + if (count <= 0 || count > NDS_MEM_MAX_RANGES) { + Py_DECREF(seq); + PyErr_SetString(PyExc_ValueError, "range count out of range"); + return -1; + } + ranges = calloc((size_t)count, sizeof(*ranges)); + if (!ranges) { + Py_DECREF(seq); + PyErr_NoMemory(); + return -1; + } + for (i = 0; i < count; i++) { + unsigned long long addr; + unsigned long long size; + + if (!PyArg_ParseTuple(PySequence_Fast_GET_ITEM(seq, i), "KK", + &addr, &size)) { + free(ranges); + Py_DECREF(seq); + return -1; + } + ranges[i].addr = addr; + ranges[i].size = size; + } + Py_DECREF(seq); + *ranges_out = ranges; + *nr_out = (uint32_t)count; + return 0; +} + +static PyObject *py_nds_register_mem(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct nds_mem_range *ranges; + PyObject *ranges_py; + uint32_t nr; + int flags = 0; + int ret; + + if (!PyArg_ParseTuple(args, "O|i", &ranges_py, &flags)) + return NULL; + if (parse_nds_mem_ranges(ranges_py, &ranges, &nr)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = nds_register_mem(&(struct nds_mem_desc){ + .ranges = ranges, .nr = nr, .flags = flags, + }); + Py_END_ALLOW_THREADS + free(ranges); + return PyLong_FromLong(ret); +} + +static PyObject *py_nds_unregister_mem(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct nds_mem_range *ranges; + PyObject *ranges_py; + uint32_t nr; + int flags = 0; + int ret; + + if (!PyArg_ParseTuple(args, "O|i", &ranges_py, &flags)) + return NULL; + if (parse_nds_mem_ranges(ranges_py, &ranges, &nr)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = nds_unregister_mem(&(struct nds_mem_desc){ + .ranges = ranges, .nr = nr, .flags = flags, + }); + Py_END_ALLOW_THREADS + free(ranges); + return PyLong_FromLong(ret); +} + +static PyObject *py_nds_io_new_ctx(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct nds_io_ctx_param param = { }; + struct nds_io_ctx *ctx = NULL; + unsigned int max_io_cnt; + unsigned int flags = 0; + int ret; + + if (!PyArg_ParseTuple(args, "I|I", &max_io_cnt, &flags)) + return NULL; + param.max_io_cnt = max_io_cnt; + param.flags = flags; + + Py_BEGIN_ALLOW_THREADS + ret = nds_io_new_ctx(¶m, &ctx); + Py_END_ALLOW_THREADS + if (ret) + return PyLong_FromLong(ret); + + return PyCapsule_New(ctx, nds_ctx_capsule_name, nds_ctx_capsule_destructor); +} + +static PyObject *py_nds_io_destroy_ctx(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *capsule; + struct nds_io_ctx *ctx; + int ret; + + if (!PyArg_ParseTuple(args, "O", &capsule)) + return NULL; + ctx = nds_ctx_from_obj(capsule); + if (!ctx) + return NULL; + + /* Disable auto-destroy; this call owns the teardown. */ + if (PyCapsule_SetDestructor(capsule, NULL) < 0) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = nds_io_destroy_ctx(ctx); + Py_END_ALLOW_THREADS + + /* Invalidate capsule so later use fails cleanly. */ + if (PyCapsule_SetName(capsule, "nds_io_ctx_destroyed") < 0) + return NULL; + return PyLong_FromLong(ret); +} + +static PyObject *py_nds_io_dup_fd_for_test(PyObject *Py_UNUSED(self), + PyObject *args) +{ + PyObject *capsule; + struct nds_io_ctx *ctx; + int fd; + + if (!PyArg_ParseTuple(args, "O", &capsule)) + return NULL; + ctx = nds_ctx_from_obj(capsule); + if (!ctx) + return NULL; + + fd = fcntl(ctx->p2p_fd, F_DUPFD_CLOEXEC, 0); + return PyLong_FromLong(fd < 0 ? -errno : fd); +} + +static PyObject *py_nds_io_submit(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *capsule; + PyObject *iocb_seq; + PyObject *seq; + struct nds_io_ctx *ctx; + struct nds_io_cb *cbs = NULL; + struct nds_io_vec **iov = NULL; + Py_ssize_t count; + Py_ssize_t i; + int ret; + + if (!PyArg_ParseTuple(args, "OO", &capsule, &iocb_seq)) + return NULL; + ctx = nds_ctx_from_obj(capsule); + if (!ctx) + return NULL; + + seq = PySequence_Fast(iocb_seq, "iocbs must be a sequence"); + if (!seq) + return NULL; + count = PySequence_Fast_GET_SIZE(seq); + if (count < 0 || count > INT_MAX) { + Py_DECREF(seq); + PyErr_SetString(PyExc_ValueError, "iocb count out of range"); + return NULL; + } + if (!count) { + Py_DECREF(seq); + return PyLong_FromLong(0); + } + + cbs = calloc((size_t)count, sizeof(*cbs)); + iov = calloc((size_t)count, sizeof(*iov)); + if (!cbs || !iov) { + free(cbs); + free(iov); + Py_DECREF(seq); + return PyErr_NoMemory(); + } + + for (i = 0; i < count; i++) { + PyObject *item = PySequence_Fast_GET_ITEM(seq, i); + PyObject *fd_py; + PyObject *iov_py; + unsigned int opcode; + unsigned int rw_flags = 0; + unsigned long long offset; + unsigned long long user_data = 0; + long long fd; + int host_pid = 0; + + /* + * iocb tuple: + * (opcode, fd, offset, iov + * [, rw_flags=0, host_pid=0, user_data=0]) + * iov: sequence of (buf_addr, buf_len) + */ + if (!PyArg_ParseTuple(item, "IOKO|IiK", + &opcode, &fd_py, &offset, &iov_py, + &rw_flags, &host_pid, + &user_data)) { + goto parse_error; + } + fd = PyLong_AsLongLong(fd_py); + if (PyErr_Occurred()) + goto parse_error; + if (fd < 0 || fd > INT32_MAX) { + PyErr_SetString(PyExc_ValueError, "fd out of range"); + goto parse_error; + } + if (parse_nds_iov(iov_py, &iov[i], &cbs[i].iov_cnt)) + goto parse_error; + cbs[i].opcode = opcode; + cbs[i].rw_flags = rw_flags; + cbs[i].obj.fd = (int32_t)fd; + cbs[i].offset = offset; + cbs[i].user_data = user_data; + cbs[i].iov = iov[i]; + cbs[i].host_pid = (int32_t)host_pid; + } + Py_DECREF(seq); + + Py_BEGIN_ALLOW_THREADS + ret = nds_io_submit(ctx, (int)count, cbs); + Py_END_ALLOW_THREADS + + for (i = 0; i < count; i++) + free(iov[i]); + free(iov); + free(cbs); + return PyLong_FromLong(ret); + +parse_error: + for (i = 0; i < count; i++) + free(iov[i]); + free(iov); + free(cbs); + Py_DECREF(seq); + return NULL; +} + +static PyObject *py_nds_io_getevents(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *capsule; + struct nds_io_ctx *ctx; + struct nds_io_event *events = NULL; + struct timespec ts_storage; + struct timespec *timeout = NULL; + int min_nr; + int nr; + double timeout_sec = -1.0; + PyObject *list; + int ret; + int i; + + if (!PyArg_ParseTuple(args, "Oii|d", &capsule, &min_nr, &nr, &timeout_sec)) + return NULL; + ctx = nds_ctx_from_obj(capsule); + if (!ctx) + return NULL; + if (min_nr < 0 || nr < 0 || min_nr > nr) + return PyLong_FromLong(-EINVAL); + if (!nr) + return PyList_New(0); + if (!isfinite(timeout_sec) || + (timeout_sec >= 0.0 && + timeout_sec > (double)INT64_MAX / 1000000000.0)) + return PyLong_FromLong(-EINVAL); + + events = calloc((size_t)nr, sizeof(*events)); + if (!events) + return PyErr_NoMemory(); + + if (timeout_sec >= 0.0) { + ts_storage.tv_sec = (time_t)timeout_sec; + ts_storage.tv_nsec = + (long)((timeout_sec - (double)ts_storage.tv_sec) * + 1000000000.0); + timeout = &ts_storage; + } + + Py_BEGIN_ALLOW_THREADS + ret = nds_io_getevents(ctx, min_nr, nr, events, timeout); + Py_END_ALLOW_THREADS + + if (ret < 0) { + free(events); + return PyLong_FromLong(ret); + } + + list = PyList_New(ret); + if (!list) { + free(events); + return NULL; + } + for (i = 0; i < ret; i++) { + PyObject *tup; + + tup = Py_BuildValue("(KL)", + (unsigned long long)events[i].user_data, + (long long)events[i].res); + if (!tup) { + Py_DECREF(list); + free(events); + return NULL; + } + PyList_SET_ITEM(list, i, tup); + } + free(events); + return list; +} + +static PyMethodDef NdsMethods[] = { + { "init", py_nds_init, METH_VARARGS, + "init() -> int\n" + "Open /dev/p2p_device. Each fd learns its NPU from its first buffer.\n" }, + { "exit", py_nds_exit, METH_NOARGS, + "exit() -> int\n" }, + { "register_fs", py_nds_register_fs, METH_VARARGS, + "register_fs(fs_fds) -> int\n" + "fs_fds: file/block fds to validate; no state is stored\n" }, + { "unregister_fs", py_nds_unregister_fs, METH_VARARGS, + "unregister_fs(fs_fds) -> int\n" + "Compatibility no-op.\n" }, + { "register_mem", py_nds_register_mem, METH_VARARGS, + "register_mem(ranges[, flags=0]) -> int\n" + "ranges: sequence of (addr, size) registered under one group\n" }, + { "unregister_mem", py_nds_unregister_mem, METH_VARARGS, + "unregister_mem(ranges[, flags=0]) -> int\n" + "ranges must match the registered group, in the same order.\n" }, + { "io_new_ctx", py_nds_io_new_ctx, METH_VARARGS, + "io_new_ctx(max_io_cnt[, flags=0]) -> ctx capsule | negative errno\n" }, + { "io_destroy_ctx", py_nds_io_destroy_ctx, METH_VARARGS, + "io_destroy_ctx(ctx) -> int\n" }, + { "_io_dup_fd_for_test", py_nds_io_dup_fd_for_test, METH_VARARGS, + "Internal test hook: duplicate the context fd.\n" }, + { "io_submit", py_nds_io_submit, METH_VARARGS, + "io_submit(ctx, iocbs) -> int\n\n" + "iocb: (opcode, fd, offset, iov" + "[, rw_flags=0, host_pid=0, user_data=0])\n" + "iov: sequence of (buf_addr, buf_len)\n" }, + { "io_getevents", py_nds_io_getevents, METH_VARARGS, + "io_getevents(ctx, min_nr, nr[, timeout_sec=-1]) -> list|int\n\n" + "On success returns [(user_data, res), ...].\n" + "On failure returns negative errno.\n" + "timeout_sec must be finite; < 0 waits forever and 0 is non-blocking.\n" }, + { NULL, NULL, 0, NULL } +}; + +static struct PyModuleDef nds_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "nds", + .m_doc = "NDS GDS-aligned P2P API", + .m_size = -1, + .m_methods = NdsMethods, +}; + +PyMODINIT_FUNC PyInit_nds(void) +{ + PyObject *module; + + module = PyModule_Create(&nds_module); + if (!module) + return NULL; + + if (PyModule_AddIntConstant(module, "NDS_API_VERSION", NDS_API_VERSION) || + PyModule_AddIntConstant(module, "NDS_IO_MAX_IO_CNT", + NDS_IO_MAX_IO_CNT) || + PyModule_AddIntConstant(module, "NDS_IO_MAX_IOV", NDS_IO_MAX_IOV) || + PyModule_AddIntConstant(module, "NDS_MEM_MAX_RANGES", + NDS_MEM_MAX_RANGES) || + PyModule_AddIntConstant(module, "NDS_IO_OP_PREAD", NDS_IO_OP_PREAD) || + PyModule_AddIntConstant(module, "NDS_IO_OP_PWRITE", NDS_IO_OP_PWRITE) || + PyModule_AddIntConstant(module, "NDS_IO_F_REGISTERED_MEM", + NDS_IO_F_REGISTERED_MEM)) { + Py_DECREF(module); + return NULL; + } + return module; +} diff --git a/tools/lib/nds/file_p2p/setup.py b/tools/lib/nds/file_p2p/setup.py new file mode 100644 index 000000000000..ebe2565943f5 --- /dev/null +++ b/tools/lib/nds/file_p2p/setup.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: GPL-2.0 +from setuptools import setup, Extension + +file_p2p_module = Extension( + "file_p2p", + sources=["file_p2p_api.c", "py_file_p2p_api.c", "p2p_common.c"], + depends=[ + "file_p2p_api.h", + "../../../include/uapi/linux/nds_p2p.h", + "p2p_common.h", + ], + extra_compile_args=["-Wall", "-Wextra", "-I../../../include/uapi"], +) + +nds_module = Extension( + "nds", + sources=["nds_api.c", "py_nds_api.c", "p2p_common.c"], + depends=[ + "nds_api.h", + "nds_api_internal.h", + "../../../include/uapi/linux/nds_p2p.h", + "p2p_common.h", + ], + extra_compile_args=["-Wall", "-Wextra", "-pthread", "-I../../../include/uapi"], + extra_link_args=["-pthread"], +) + +setup( + name="file_p2p", + version="1.0", + ext_modules=[file_p2p_module, nds_module], +) -- 2.52.0