ENABLE XDS Hou Tao (2): nds: add NVMe peer-to-peer I/O driver nds: add userspace library arch/arm64/configs/openeuler_defconfig | 6 + drivers/Kconfig | 2 + drivers/Makefile | 1 + drivers/nds/Kconfig | 16 + drivers/nds/Makefile | 6 + drivers/nds/debugfs.c | 188 ++ drivers/nds/debugfs.h | 22 + drivers/nds/dev.c | 1926 +++++++++++++++++++++ drivers/nds/mem.c | 65 + drivers/nds/mem.h | 17 + drivers/nds/mem_abi.h | 23 + drivers/nds/provider.c | 280 +++ drivers/nds/provider.h | 40 + include/uapi/linux/nds_p2p.h | 132 ++ tools/include/uapi/linux/nds_p2p.h | 132 ++ tools/lib/nds/Makefile | 13 + tools/lib/nds/file_p2p/.gitignore | 8 + tools/lib/nds/file_p2p/Makefile | 35 + tools/lib/nds/file_p2p/file_p2p_api.c | 136 ++ tools/lib/nds/file_p2p/file_p2p_api.h | 36 + tools/lib/nds/file_p2p/nds_api.c | 860 +++++++++ tools/lib/nds/file_p2p/nds_api.h | 249 +++ tools/lib/nds/file_p2p/nds_api_internal.h | 12 + tools/lib/nds/file_p2p/p2p_common.c | 69 + tools/lib/nds/file_p2p/p2p_common.h | 16 + tools/lib/nds/file_p2p/py_file_p2p_api.c | 320 ++++ tools/lib/nds/file_p2p/py_nds_api.c | 577 ++++++ tools/lib/nds/file_p2p/setup.py | 32 + 28 files changed, 5219 insertions(+) create mode 100644 drivers/nds/Kconfig create mode 100644 drivers/nds/Makefile create mode 100644 drivers/nds/debugfs.c create mode 100644 drivers/nds/debugfs.h create mode 100644 drivers/nds/dev.c create mode 100644 drivers/nds/mem.c create mode 100644 drivers/nds/mem.h create mode 100644 drivers/nds/mem_abi.h create mode 100644 drivers/nds/provider.c create mode 100644 drivers/nds/provider.h create mode 100644 include/uapi/linux/nds_p2p.h create mode 100644 tools/include/uapi/linux/nds_p2p.h create mode 100644 tools/lib/nds/Makefile create mode 100644 tools/lib/nds/file_p2p/.gitignore create mode 100644 tools/lib/nds/file_p2p/Makefile create mode 100644 tools/lib/nds/file_p2p/file_p2p_api.c create mode 100644 tools/lib/nds/file_p2p/file_p2p_api.h create mode 100644 tools/lib/nds/file_p2p/nds_api.c create mode 100644 tools/lib/nds/file_p2p/nds_api.h create mode 100644 tools/lib/nds/file_p2p/nds_api_internal.h create mode 100644 tools/lib/nds/file_p2p/p2p_common.c create mode 100644 tools/lib/nds/file_p2p/p2p_common.h create mode 100644 tools/lib/nds/file_p2p/py_file_p2p_api.c create mode 100644 tools/lib/nds/file_p2p/py_nds_api.c create mode 100644 tools/lib/nds/file_p2p/setup.py -- 2.52.0
From: Hou Tao <houtao1@huawei.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 -------------------------------- Introduce drivers/nds and the nds_p2p UAPI so userspace can submit asynchronous PCI P2PDMA I/O against registered device memory. The driver registers provider pages, maps them to PCI bus addresses through P2PDMA, and issues direct-I/O kiocbs to the backing block device. The driver is module-only and requires a P2PDMA-capable queue. Signed-off-by: Hou Tao <houtao1@huawei.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- arch/arm64/configs/openeuler_defconfig | 6 + drivers/Kconfig | 2 + drivers/Makefile | 1 + drivers/nds/Kconfig | 16 + drivers/nds/Makefile | 6 + drivers/nds/debugfs.c | 188 +++ drivers/nds/debugfs.h | 22 + drivers/nds/dev.c | 1926 ++++++++++++++++++++++++ drivers/nds/mem.c | 65 + drivers/nds/mem.h | 17 + drivers/nds/mem_abi.h | 23 + drivers/nds/provider.c | 280 ++++ drivers/nds/provider.h | 40 + include/uapi/linux/nds_p2p.h | 132 ++ 14 files changed, 2724 insertions(+) create mode 100644 drivers/nds/Kconfig create mode 100644 drivers/nds/Makefile create mode 100644 drivers/nds/debugfs.c create mode 100644 drivers/nds/debugfs.h create mode 100644 drivers/nds/dev.c create mode 100644 drivers/nds/mem.c create mode 100644 drivers/nds/mem.h create mode 100644 drivers/nds/mem_abi.h create mode 100644 drivers/nds/provider.c create mode 100644 drivers/nds/provider.h create mode 100644 include/uapi/linux/nds_p2p.h diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 1561eb379cc4..215013633a69 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -2517,10 +2517,16 @@ CONFIG_NVME_TARGET_FC=m CONFIG_NVME_TARGET_FCLOOP=m CONFIG_NVME_TARGET_TCP=m # CONFIG_NVME_TARGET_AUTH is not set # end of NVME Support +# +# NDS Support +# +CONFIG_NDS=m +# end of NDS Support + # # Misc devices # CONFIG_SENSORS_LIS3LV02D=m # CONFIG_AD525X_DPOT is not set diff --git a/drivers/Kconfig b/drivers/Kconfig index 35521e3200fa..de7dc87990be 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -36,10 +36,12 @@ source "drivers/pnp/Kconfig" source "drivers/block/Kconfig" source "drivers/nvme/Kconfig" +source "drivers/nds/Kconfig" + source "drivers/misc/Kconfig" source "drivers/scsi/Kconfig" source "drivers/ata/Kconfig" diff --git a/drivers/Makefile b/drivers/Makefile index 6f1408d617de..0754ee02bfc9 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -84,10 +84,11 @@ obj-$(CONFIG_DMA_SHARED_BUFFER) += dma-buf/ obj-$(CONFIG_NUBUS) += nubus/ obj-y += cxl/ obj-y += macintosh/ obj-y += scsi/ obj-y += nvme/ +obj-$(CONFIG_NDS) += nds/ obj-$(CONFIG_ATA) += ata/ obj-$(CONFIG_TARGET_CORE) += target/ obj-$(CONFIG_MTD) += mtd/ obj-$(CONFIG_SPI) += spi/ obj-$(CONFIG_SPMI) += spmi/ diff --git a/drivers/nds/Kconfig b/drivers/nds/Kconfig new file mode 100644 index 000000000000..3f0be3a1d8ea --- /dev/null +++ b/drivers/nds/Kconfig @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config NDS + tristate "NDS NVMe peer-to-peer I/O" + depends on m + depends on PCI_P2PDMA + depends on BLOCK && BLK_DEV_NVME + default m + help + NDS provides a character device and UAPI for PCI P2PDMA I/O + against registered device memory (e.g. NPU HBM). + + This driver can only be built as a module. The module will be + called p2p_dev. + + If unsure, say N. diff --git a/drivers/nds/Makefile b/drivers/nds/Makefile new file mode 100644 index 000000000000..94df04e04a05 --- /dev/null +++ b/drivers/nds/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# NDS / XDS NVMe peer-to-peer I/O + +obj-$(CONFIG_NDS) += p2p_dev.o +p2p_dev-y := dev.o provider.o debugfs.o mem.o diff --git a/drivers/nds/debugfs.c b/drivers/nds/debugfs.c new file mode 100644 index 000000000000..fabd9bbd6929 --- /dev/null +++ b/drivers/nds/debugfs.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/debugfs.h> +#include <linux/module.h> +#include <linux/percpu.h> +#include <linux/preempt.h> +#include <linux/seq_file.h> +#include <linux/string.h> +#include <asm/local.h> + +#include "debugfs.h" + +/* + * Per-CPU I/O counters, same pattern as block layer disk_stats / part_stat: + * Each field is indexed by P2P_IO_STATS_READ or P2P_IO_STATS_WRITE. Update + * this CPU under preempt_disable() (io_inflight via local_t); readers sum + * across CPUs. + */ +struct p2p_io_stats { + u64 io_issued[P2P_IO_STATS_NR]; + u64 io_bytes[P2P_IO_STATS_NR]; + local_t io_inflight[P2P_IO_STATS_NR]; + u64 io_failed[P2P_IO_STATS_NR]; + u64 io_issue_failed[P2P_IO_STATS_NR]; +}; + +struct p2p_io_stats_snapshot { + u64 io_issued[P2P_IO_STATS_NR]; + u64 io_bytes[P2P_IO_STATS_NR]; + s64 io_inflight[P2P_IO_STATS_NR]; + u64 io_failed[P2P_IO_STATS_NR]; + u64 io_issue_failed[P2P_IO_STATS_NR]; +}; + +static DEFINE_PER_CPU(struct p2p_io_stats, p2p_io_stats); +static struct dentry *p2p_debugfs_root; +static const char * const p2p_io_op_names[P2P_IO_STATS_NR] = { + [P2P_IO_STATS_READ] = "read", + [P2P_IO_STATS_WRITE] = "write", +}; + +static inline u32 p2p_io_stats_index(u32 op) +{ + return !!op; +} + +void p2p_stats_io_issued(u32 op, u64 bytes) +{ + struct p2p_io_stats *stats; + u32 index = p2p_io_stats_index(op); + + preempt_disable(); + stats = this_cpu_ptr(&p2p_io_stats); + stats->io_issued[index]++; + stats->io_bytes[index] += bytes; + local_inc(&stats->io_inflight[index]); + preempt_enable(); +} + +void p2p_stats_io_issue_failed(u32 op) +{ + struct p2p_io_stats *stats; + u32 index = p2p_io_stats_index(op); + + preempt_disable(); + stats = this_cpu_ptr(&p2p_io_stats); + stats->io_issue_failed[index]++; + preempt_enable(); +} + +void p2p_stats_io_complete(u32 op, int err) +{ + struct p2p_io_stats *stats; + u32 index = p2p_io_stats_index(op); + + preempt_disable(); + stats = this_cpu_ptr(&p2p_io_stats); + if (err) + stats->io_failed[index]++; + local_dec(&stats->io_inflight[index]); + preempt_enable(); +} + +static void p2p_io_stats_snapshot(struct p2p_io_stats_snapshot *out) +{ + unsigned int cpu; + u32 index; + + memset(out, 0, sizeof(*out)); + for_each_possible_cpu(cpu) { + struct p2p_io_stats *s = per_cpu_ptr(&p2p_io_stats, cpu); + + for (index = 0; index < P2P_IO_STATS_NR; index++) { + out->io_issued[index] += s->io_issued[index]; + out->io_bytes[index] += s->io_bytes[index]; + out->io_inflight[index] += local_read(&s->io_inflight[index]); + out->io_failed[index] += s->io_failed[index]; + out->io_issue_failed[index] += s->io_issue_failed[index]; + } + } +} + +static void p2p_summary_show_op(struct seq_file *seq, u32 index, + const struct p2p_io_stats_snapshot *snapshot) +{ + const char *name = p2p_io_op_names[index]; + + seq_printf(seq, "%s_io_issued: %llu\n", name, snapshot->io_issued[index]); + seq_printf(seq, "%s_io_bytes: %llu\n", name, snapshot->io_bytes[index]); + seq_printf(seq, "%s_io_inflight: %lld\n", name, snapshot->io_inflight[index]); + seq_printf(seq, "%s_io_failed: %llu\n", name, snapshot->io_failed[index]); + seq_printf(seq, "%s_io_issue_failed: %llu\n", name, snapshot->io_issue_failed[index]); +} + +static int p2p_summary_show(struct seq_file *seq, void *unused) +{ + struct p2p_io_stats_snapshot snapshot; + u32 index; + + p2p_io_stats_snapshot(&snapshot); + + for (index = 0; index < P2P_IO_STATS_NR; index++) + p2p_summary_show_op(seq, index, &snapshot); + return 0; +} + +static int p2p_summary_open(struct inode *inode, struct file *file) +{ + return single_open(file, p2p_summary_show, NULL); +} + +static ssize_t p2p_summary_write(struct file *file, + const char __user *buf, size_t count, + loff_t *ppos) +{ + unsigned int cpu; + u32 index; + + if (!count) + return 0; + + /* Reset cumulative counters only; leave io_inflight alone. */ + for_each_possible_cpu(cpu) { + struct p2p_io_stats *s = per_cpu_ptr(&p2p_io_stats, cpu); + + for (index = 0; index < P2P_IO_STATS_NR; index++) { + s->io_issued[index] = 0; + s->io_bytes[index] = 0; + s->io_failed[index] = 0; + s->io_issue_failed[index] = 0; + } + } + return count; +} + +static const struct file_operations p2p_summary_fops = { + .owner = THIS_MODULE, + .open = p2p_summary_open, + .read = seq_read, + .write = p2p_summary_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static int p2p_memory_open(struct inode *inode, struct file *file) +{ + return single_open(file, p2p_debugfs_memory_show, NULL); +} + +static const struct file_operations p2p_memory_fops = { + .owner = THIS_MODULE, + .open = p2p_memory_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void p2p_debugfs_init(void) +{ + p2p_debugfs_root = debugfs_create_dir("p2p_device", NULL); + debugfs_create_file("summary", 0600, p2p_debugfs_root, NULL, &p2p_summary_fops); + debugfs_create_file("memory", 0400, p2p_debugfs_root, NULL, &p2p_memory_fops); +} + +void p2p_debugfs_exit(void) +{ + debugfs_remove_recursive(p2p_debugfs_root); + p2p_debugfs_root = NULL; +} diff --git a/drivers/nds/debugfs.h b/drivers/nds/debugfs.h new file mode 100644 index 000000000000..419b31d699fb --- /dev/null +++ b/drivers/nds/debugfs.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef P2P_DEBUGFS_H_ +#define P2P_DEBUGFS_H_ + +#include <linux/types.h> + +#define P2P_IO_STATS_READ 0U +#define P2P_IO_STATS_WRITE 1U +#define P2P_IO_STATS_NR 2U + +struct seq_file; + +void p2p_debugfs_init(void); +void p2p_debugfs_exit(void); +int p2p_debugfs_memory_show(struct seq_file *seq, void *unused); + +void p2p_stats_io_issued(u32 op, u64 bytes); +void p2p_stats_io_issue_failed(u32 op); +/* One issue and one completion for each logical I/O; @err is 0 or -errno. */ +void p2p_stats_io_complete(u32 op, int err); + +#endif diff --git a/drivers/nds/dev.c b/drivers/nds/dev.c new file mode 100644 index 000000000000..e7c6043d3459 --- /dev/null +++ b/drivers/nds/dev.c @@ -0,0 +1,1926 @@ +// SPDX-License-Identifier: GPL-2.0 +#define pr_fmt(fmt) "p2p: " fmt + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/kdev_t.h> +#include <linux/fs.h> +#include <linux/cdev.h> +#include <linux/device.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <linux/ioctl.h> +#include <linux/vmalloc.h> +#include <linux/limits.h> +#include <linux/overflow.h> +#include <linux/mutex.h> +#include <linux/percpu-refcount.h> +#include <linux/pid.h> +#include <linux/random.h> +#include <linux/rcupdate.h> +#include <linux/seq_file.h> +#include <linux/sort.h> +#include <linux/string.h> +#include <linux/workqueue.h> +#include <linux/xarray.h> +#include <linux/wait.h> +#include <linux/ktime.h> +#ifdef CALC_CRC32 +#include <linux/crc32.h> +#endif + +#include <linux/blk-integrity.h> +#include <linux/blkdev.h> +#include <linux/bvec.h> +#include <linux/file.h> +#include <linux/fileattr.h> +#include <linux/magic.h> +#include <linux/pci.h> +#include <linux/pfn.h> +#include <linux/uio.h> + +#include <linux/nds_p2p.h> + +#include "mem.h" +#include "debugfs.h" +#include "provider.h" + +#define P2P_MAX_IOV 65536U +#define P2P_MAX_IOV_SIZE (2U << 30) +#define P2P_MIN_PAGE_SIZE (4U << 10) +#define P2P_MEM_COOKIE_SHIFT 48 +#define P2P_MEM_ID_MASK GENMASK_ULL(P2P_MEM_COOKIE_SHIFT - 1, 0) +/* Power-of-two CQ; caps in-flight and unharvested I/O per batch. */ +#define P2P_CQ_SIZE P2P_MAX_IO_NR +#define P2P_CQ_MASK (P2P_CQ_SIZE - 1) + +struct p2p_pa_iov { + u64 addr; + u64 len; +}; + +struct p2p_iov_map { + u64 aligned_addr; + u64 aligned_size; + u64 *pa_list; + u32 page_size; + u32 pa_num; +}; + +struct p2p_pinned_pa { + struct devmm_svm_process_id process_id; + struct p2p_iov_map *maps; + unsigned int pinned_map_nr; + u64 *pa_list; +}; + +struct p2p_batch; + +/* + * One translation segment of a registration. Adjacent ranges with the same + * page size and a page-aligned boundary share one segment. + */ +struct p2p_mem_seg { + u64 end; /* exclusive end of the registered bytes */ + u64 aligned_addr; /* devmm-aligned VA mapped by pa_list[0] */ + const u64 *pa_list; + u32 page_size; +}; + +struct p2p_registered_mem { + u64 handle; + /* Span of the registered ranges, reported by debugfs. */ + u64 start; + u64 end; + unsigned int nr_ranges; + /* Sorted segments; seg_starts holds their search keys. */ + unsigned int nr_segs; + u64 *seg_starts; + struct p2p_mem_seg *segs; + struct p2p_pinned_pa *pinned_pa; + /* Provider of every range; matches the owner fd. */ + struct p2p_provider *provider; + struct p2p_batch *owner; + struct pid *owner_tgid; + + struct percpu_ref io_refs; + struct completion io_zero; + struct rcu_head rcu; + struct list_head owner_node; +}; + +struct p2p_pinned_io_mem { + bool pinned; + bool reg_mem; + union { + struct p2p_pinned_pa *pinned_pa; + struct p2p_registered_mem *registered_mem; + }; +}; + +struct p2p_io_context; + +struct p2p_batch { + /* Cached from the first buffer; later requests reuse it. */ + struct p2p_provider *provider; + + /* Accepted CQ slots not yet retired; limited to P2P_CQ_SIZE. */ + atomic_t io_cnt; + + /* + * AIO-style completion ring (fs/aio.c shape): the producer publishes + * slots and advances cq_tail under completion_lock only; the consumer + * reads slots and advances cq_head under the ring_lock mutex only. + * Neither side takes the other's lock on the hot path. + */ + struct p2p_io_context **cq; + unsigned int cq_head; + unsigned int cq_tail; + /* Completed I/Os published to the CQ but not yet harvested. */ + atomic_t ready_events; + /* Accepted I/Os not yet harvested, including ready_events. */ + atomic_t outstanding_events; + spinlock_t completion_lock; + /* Serializes completion collection and final close. */ + struct mutex ring_lock; + + wait_queue_head_t wait; + struct list_head owned_mems; +}; + +struct p2p_io_context { + u32 op; + struct file *file; + struct kiocb kiocb; + /* The pages of pa_iov; one bvec for each physically contiguous run. */ + struct bio_vec *bvec; + unsigned int bvec_nr; + struct p2p_pa_iov *pa_iov; + unsigned int pa_iov_nr; + struct p2p_pinned_io_mem pinned_mem; + u64 data_size; + u64 user_data; + struct completion io_done; + /* One reference for the submitter and one for the VFS completion. */ + atomic_t io_ref; + int io_err; + struct p2p_batch *batch; +#if defined(CALC_CRC32) || defined(DUMP_CONTENT) + struct work_struct finalize_work; +#endif +}; + + +static dev_t dev; +static struct class *dev_class; +static struct cdev p2p_cdev; + +static DEFINE_XARRAY(registered_mems); +static DEFINE_MUTEX(registered_mem_lock); +static atomic64_t registered_mem_id = ATOMIC64_INIT(0); + +static int p2p_open(struct inode *inode, struct file *file); +static int p2p_release(struct inode *inode, struct file *file); +static long p2p_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +static void p2p_release_io(struct p2p_batch *batch); +static void p2p_revoke_all_registered_mem(struct p2p_batch *batch, + struct list_head *revoked); +static void p2p_destroy_registered_mem_list(struct list_head *revoked); +static void p2p_io_ctx_put(struct p2p_io_context *io_ctx); +static void p2p_publish_io_done(struct p2p_io_context *io_ctx); +#if defined(CALC_CRC32) || defined(DUMP_CONTENT) +static void p2p_finalize_io_work(struct work_struct *work); +#endif + +static const struct file_operations fops = { + .owner = THIS_MODULE, + .open = p2p_open, + .release = p2p_release, + .unlocked_ioctl = p2p_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +int p2p_debugfs_memory_show(struct seq_file *seq, void *unused) +{ + struct p2p_registered_mem *mem; + unsigned long index; + + rcu_read_lock(); + xa_for_each(®istered_mems, index, mem) { + /* RCU protects mem; the reference protects its arrays. */ + if (!percpu_ref_tryget_live(&mem->io_refs)) + continue; + seq_printf(seq, "handle: 0x%llx addr: 0x%llx size: 0x%llx", + mem->handle, mem->start, mem->end - mem->start); + if (mem->nr_ranges > 1) + seq_printf(seq, " ranges: %u segs: %u", mem->nr_ranges, mem->nr_segs); + seq_putc(seq, '\n'); + for (unsigned int i = 0; i < mem->nr_segs; i++) { + const struct p2p_mem_seg *seg = &mem->segs[i]; + + seq_printf(seq, " segment %u: va [0x%llx, 0x%llx) aligned 0x%llx page_size 0x%x\n", + i, mem->seg_starts[i], seg->end, + seg->aligned_addr, seg->page_size); + } + percpu_ref_put(&mem->io_refs); + } + rcu_read_unlock(); + + return 0; +} + +static int p2p_open(struct inode *inode, struct file *file) +{ + struct p2p_batch *batch; + + batch = kzalloc(sizeof(*batch), GFP_KERNEL); + if (!batch) + return -ENOMEM; + + batch->cq = kcalloc(P2P_CQ_SIZE, sizeof(*batch->cq), GFP_KERNEL); + if (!batch->cq) { + kfree(batch); + return -ENOMEM; + } + + spin_lock_init(&batch->completion_lock); + mutex_init(&batch->ring_lock); + init_waitqueue_head(&batch->wait); + INIT_LIST_HEAD(&batch->owned_mems); + atomic_set(&batch->io_cnt, 0); + /* Cross the unsigned counter wrap after eight completions. */ + batch->cq_head = -8U; + batch->cq_tail = -8U; + atomic_set(&batch->ready_events, 0); + atomic_set(&batch->outstanding_events, 0); + file->private_data = batch; + + return 0; +} + +static int p2p_release(struct inode *inode, struct file *file) +{ + struct p2p_batch *batch = file->private_data; + LIST_HEAD(revoked_mems); + + p2p_revoke_all_registered_mem(batch, &revoked_mems); + + if (atomic_read(&batch->io_cnt) > 0) + p2p_release_io(batch); + + p2p_destroy_registered_mem_list(&revoked_mems); + + kfree(batch->cq); + kfree(batch); + + file->private_data = NULL; + + return 0; +} + +/* + * devmm finds a host process by (host_pid, vm_id), and devid shares a union + * with vm_id. The id must stay 0 (the zeroed allocation): an NPU id there + * would name a VM, and every pin on an NPU other than 0 would fail. + */ +static void init_process_id(int host_pid, struct devmm_svm_process_id *process_id) +{ + rcu_read_lock(); + process_id->host_pid = pid_nr(find_vpid(host_pid)); + rcu_read_unlock(); +} + +static void init_current_process_id(struct devmm_svm_process_id *process_id) +{ + process_id->host_pid = task_tgid_nr(current); +} + +/* + * Select one provider for a request or registration. The first PA needs a + * lookup; later PAs normally need only a range check. @provider is shared by + * all maps, so the lookup cost does not grow with the IOV count. + */ +static int p2p_check_pa_list(struct p2p_provider **provider, const u64 *pa_list, + u32 pa_num, u32 page_size) +{ + u32 i; + + for (i = 0; i < pa_num; i++) { + struct p2p_provider *found; + + if (likely(*provider && + p2p_provider_contains(*provider, pa_list[i], page_size))) + continue; + found = p2p_provider_find(pa_list[i], page_size); + if (!found) { + pr_warn_ratelimited("PA 0x%llx is outside every HBM window\n", + pa_list[i]); + return -EFAULT; + } + if (*provider && found != *provider) { + pr_warn_ratelimited("PA 0x%llx belongs to %s, not %s\n", pa_list[i], + pci_name(found->pdev), pci_name((*provider)->pdev)); + return -EXDEV; + } + *provider = found; + } + return 0; +} + +static int p2p_bind_provider(struct p2p_batch *batch, struct p2p_provider *provider) +{ + struct p2p_provider *bound; + + /* Pointer comparison only; provider identity is already initialized. */ + bound = READ_ONCE(batch->provider); + if (likely(bound)) + return bound == provider ? 0 : -EXDEV; + bound = cmpxchg(&batch->provider, NULL, provider); + return !bound || bound == provider ? 0 : -EXDEV; +} + +static void p2p_put_pinned_pa(struct p2p_pinned_pa *pinned_pa) +{ + unsigned int i; + + for (i = 0; i < pinned_pa->pinned_map_nr; i++) { + struct p2p_iov_map *map = &pinned_pa->maps[i]; + + p2p_mem_put_pa_list(&pinned_pa->process_id, map->aligned_addr, + map->aligned_size, map->pa_list, + map->pa_num); + } + + kvfree(pinned_pa->pa_list); + kvfree(pinned_pa->maps); + kfree(pinned_pa); +} + +static void p2p_registered_mem_release(struct percpu_ref *ref) +{ + struct p2p_registered_mem *mem; + + mem = container_of(ref, struct p2p_registered_mem, io_refs); + pr_debug("registered memory handle 0x%llx released\n", mem->handle); + complete(&mem->io_zero); +} + +static void p2p_free_registered_mem_rcu(struct rcu_head *rcu) +{ + struct p2p_registered_mem *mem; + + mem = container_of(rcu, struct p2p_registered_mem, rcu); + percpu_ref_exit(&mem->io_refs); + kfree(mem); +} + +static void p2p_free_mem_segs(struct p2p_registered_mem *mem) +{ + kvfree(mem->seg_starts); + kvfree(mem->segs); +} + +static void p2p_destroy_registered_mem(struct p2p_registered_mem *mem) +{ + wait_for_completion(&mem->io_zero); + p2p_free_mem_segs(mem); + p2p_put_pinned_pa(mem->pinned_pa); + put_pid(mem->owner_tgid); + call_rcu(&mem->rcu, p2p_free_registered_mem_rcu); +} + +/* Pin each sorted range with its own devmm call; maps share one PA array. */ +static int p2p_pin_registered_pa(struct p2p_batch *batch, + const struct p2p_mem_range *ranges, unsigned int nr, + struct p2p_pinned_pa **pinned_pa_out, + struct p2p_provider **provider_out) +{ + struct p2p_provider *provider = READ_ONCE(batch->provider); + struct p2p_pinned_pa *pinned_pa; + u64 total_pa_num = 0; + unsigned int i; + int err; + + pinned_pa = kzalloc(sizeof(*pinned_pa), GFP_KERNEL); + if (!pinned_pa) + return -ENOMEM; + init_current_process_id(&pinned_pa->process_id); + + pinned_pa->maps = kvcalloc(nr, sizeof(*pinned_pa->maps), GFP_KERNEL); + if (!pinned_pa->maps) { + err = -ENOMEM; + goto put_pinned_pa; + } + + for (i = 0; i < nr; i++) { + struct p2p_iov_map *map = &pinned_pa->maps[i]; + u64 addr = ranges[i].addr; + u64 aligned_end; + u64 pa_num; + int page_size; + + page_size = p2p_mem_get_page_size(&pinned_pa->process_id, addr, ranges[i].size); + if (page_size < 0) { + err = page_size; + goto put_pinned_pa; + } + if (page_size < P2P_MIN_PAGE_SIZE || !is_power_of_2(page_size)) { + pr_err("invalid page size %d for registered range 0x%llx+0x%llx\n", + page_size, addr, ranges[i].size); + err = -EINVAL; + goto put_pinned_pa; + } + + if (check_add_overflow(addr + ranges[i].size, (u64)page_size - 1, &aligned_end)) { + err = -EOVERFLOW; + goto put_pinned_pa; + } + aligned_end &= ~((u64)page_size - 1); + + map->aligned_addr = addr & ~((u64)page_size - 1); + map->aligned_size = aligned_end - map->aligned_addr; + map->page_size = page_size; + pa_num = map->aligned_size / map->page_size; + if (!pa_num || pa_num > U32_MAX - total_pa_num) { + err = -E2BIG; + goto put_pinned_pa; + } + map->pa_num = pa_num; + total_pa_num += pa_num; + } + + pinned_pa->pa_list = kvmalloc_array(total_pa_num, sizeof(*pinned_pa->pa_list), GFP_KERNEL); + if (!pinned_pa->pa_list) { + err = -ENOMEM; + goto put_pinned_pa; + } + + total_pa_num = 0; + for (i = 0; i < nr; i++) { + struct p2p_iov_map *map = &pinned_pa->maps[i]; + + map->pa_list = pinned_pa->pa_list + total_pa_num; + err = p2p_mem_get_pa_list(&pinned_pa->process_id, map->aligned_addr, + map->aligned_size, map->pa_list, map->pa_num); + if (err) { + pr_err("registered PA list addr 0x%llx size 0x%llx num %u range %u err %d\n", + map->aligned_addr, map->aligned_size, map->pa_num, i, err); + goto put_pinned_pa; + } + pinned_pa->pinned_map_nr++; + err = p2p_check_pa_list(&provider, map->pa_list, map->pa_num, map->page_size); + if (err) + goto put_pinned_pa; + total_pa_num += map->pa_num; + } + + err = p2p_provider_prepare(provider); + if (err) + goto put_pinned_pa; + err = p2p_bind_provider(batch, provider); + if (err) + goto put_pinned_pa; + *pinned_pa_out = pinned_pa; + *provider_out = provider; + return 0; + +put_pinned_pa: + p2p_put_pinned_pa(pinned_pa); + return err; +} + +/* + * Build the lookup segments from the pinned maps. A range starting exactly at + * the page-aligned end of the previous segment with the same page size extends + * that segment: the maps are laid out in VA order in one PA array, so the + * previous segment's pa_list indexes straight into the next map. + */ +static int p2p_build_mem_segs(struct p2p_registered_mem *mem, const struct p2p_mem_range *ranges, + unsigned int nr) +{ + const struct p2p_iov_map *maps = mem->pinned_pa->maps; + unsigned int n = 0; + unsigned int i; + + mem->seg_starts = kvmalloc_array(nr, sizeof(*mem->seg_starts), GFP_KERNEL); + mem->segs = kvmalloc_array(nr, sizeof(*mem->segs), GFP_KERNEL); + if (!mem->seg_starts || !mem->segs) { + p2p_free_mem_segs(mem); + return -ENOMEM; + } + + for (i = 0; i < nr; i++) { + u64 end = ranges[i].addr + ranges[i].size; + + if (n && mem->segs[n - 1].end == ranges[i].addr && + mem->segs[n - 1].page_size == maps[i].page_size && + IS_ALIGNED(ranges[i].addr, maps[i].page_size)) { + mem->segs[n - 1].end = end; + continue; + } + mem->seg_starts[n] = ranges[i].addr; + mem->segs[n] = (struct p2p_mem_seg) { + .end = end, + .aligned_addr = maps[i].aligned_addr, + .pa_list = maps[i].pa_list, + .page_size = maps[i].page_size, + }; + n++; + } + + mem->nr_segs = n; + mem->nr_ranges = nr; + mem->start = ranges[0].addr; + mem->end = ranges[nr - 1].addr + ranges[nr - 1].size; + return 0; +} + +static u64 p2p_new_registered_mem_handle(void) +{ + u64 id = atomic64_inc_return(®istered_mem_id) & P2P_MEM_ID_MASK; + u64 cookie = get_random_u32() & U16_MAX; + + return cookie << P2P_MEM_COOKIE_SHIFT | id; +} + +static int p2p_publish_registered_mem(struct p2p_batch *batch, + struct p2p_registered_mem *mem) +{ + int err; + + mutex_lock(®istered_mem_lock); + mem->owner = batch; + err = xa_insert(®istered_mems, mem->handle, mem, GFP_KERNEL); + if (!err) { + list_add_tail(&mem->owner_node, &batch->owned_mems); + percpu_ref_reinit(&mem->io_refs); + } else { + pr_err("publish registered memory handle 0x%llx err %d\n", mem->handle, err); + } + mutex_unlock(®istered_mem_lock); + + return err; +} + +static int p2p_unpublish_registered_mem(struct p2p_batch *batch, u64 handle, + struct p2p_registered_mem **mem_out) +{ + struct p2p_registered_mem *mem; + int err = 0; + + mutex_lock(®istered_mem_lock); + rcu_read_lock(); + mem = xa_load(®istered_mems, handle); + if (!mem) { + err = -ENOENT; + goto unlock_rcu; + } + if (mem->owner != batch) { + pr_err("registered memory handle 0x%llx belongs to another p2p fd\n", handle); + err = -EPERM; + goto unlock_rcu; + } + if (mem->owner_tgid != task_tgid(current)) { + pr_err("registered memory handle 0x%llx belongs to another process\n", handle); + err = -EPERM; + goto unlock_rcu; + } + rcu_read_unlock(); + + xa_erase(®istered_mems, handle); + list_del_init(&mem->owner_node); + *mem_out = mem; + mutex_unlock(®istered_mem_lock); + return 0; + +unlock_rcu: + rcu_read_unlock(); + mutex_unlock(®istered_mem_lock); + return err; +} + +static void p2p_revoke_all_registered_mem(struct p2p_batch *batch, + struct list_head *revoked) +{ + struct p2p_registered_mem *mem, *next; + + if (list_empty(&batch->owned_mems)) + return; + + list_for_each_entry_safe(mem, next, &batch->owned_mems, owner_node) { + xa_erase(®istered_mems, mem->handle); + list_move_tail(&mem->owner_node, revoked); + pr_info("unregister memory handle 0x%llx through p2p fd release\n", + mem->handle); + } + + list_for_each_entry(mem, revoked, owner_node) + percpu_ref_kill(&mem->io_refs); +} + +static void p2p_destroy_registered_mem_list(struct list_head *revoked) +{ + struct p2p_registered_mem *mem; + struct p2p_registered_mem *next; + + list_for_each_entry_safe(mem, next, revoked, owner_node) { + list_del_init(&mem->owner_node); + p2p_destroy_registered_mem(mem); + } +} + +static int p2p_mem_range_cmp(const void *a, const void *b) +{ + const struct p2p_mem_range *ra = a; + const struct p2p_mem_range *rb = b; + + if (ra->addr < rb->addr) + return -1; + return ra->addr > rb->addr; +} + +/* Sort ranges by VA and reject empty, overflowing, or overlapping ones. */ +static int p2p_sort_mem_ranges(struct p2p_mem_range *ranges, unsigned int nr) +{ + unsigned int i; + u64 end; + + for (i = 0; i < nr; i++) { + if (!ranges[i].size) + return -EINVAL; + if (check_add_overflow(ranges[i].addr, ranges[i].size, &end)) + return -EOVERFLOW; + } + + sort(ranges, nr, sizeof(*ranges), p2p_mem_range_cmp, NULL); + for (i = 1; i < nr; i++) { + if (ranges[i].addr < ranges[i - 1].addr + ranges[i - 1].size) + return -EINVAL; + } + return 0; +} + +static int p2p_register_mem_ranges(struct p2p_batch *batch, struct p2p_mem_range *ranges, + unsigned int nr, u64 __user *handle_out) +{ + struct p2p_registered_mem *mem; + int err; + + err = p2p_sort_mem_ranges(ranges, nr); + if (err) + return err; + + mem = kzalloc(sizeof(*mem), GFP_KERNEL); + if (!mem) + return -ENOMEM; + mem->owner_tgid = get_task_pid(current, PIDTYPE_TGID); + init_completion(&mem->io_zero); + INIT_LIST_HEAD(&mem->owner_node); + + err = percpu_ref_init(&mem->io_refs, p2p_registered_mem_release, + PERCPU_REF_INIT_DEAD, GFP_KERNEL); + if (err) + goto put_owner_tgid; + + err = p2p_pin_registered_pa(batch, ranges, nr, &mem->pinned_pa, &mem->provider); + if (err) + goto exit_io_refs; + + err = p2p_build_mem_segs(mem, ranges, nr); + if (err) + goto put_pinned_pa; + + mem->handle = p2p_new_registered_mem_handle(); + if (put_user(mem->handle, handle_out)) { + err = -EFAULT; + goto free_segs; + } + + err = p2p_publish_registered_mem(batch, mem); + if (err) + goto free_segs; + + pr_info("registered memory handle 0x%llx addr 0x%llx size 0x%llx ranges %u segs %u\n", + mem->handle, mem->start, mem->end - mem->start, mem->nr_ranges, mem->nr_segs); + return 0; + +free_segs: + p2p_free_mem_segs(mem); +put_pinned_pa: + p2p_put_pinned_pa(mem->pinned_pa); +exit_io_refs: + percpu_ref_exit(&mem->io_refs); +put_owner_tgid: + put_pid(mem->owner_tgid); + kfree(mem); + return err; +} + +static int p2p_register_mem(struct p2p_batch *batch, void __user *arg) +{ + struct p2p_mem_register_param __user *user_param = arg; + struct p2p_mem_register_param param; + struct p2p_mem_range *ranges; + int err; + + if (copy_from_user(¶m, arg, sizeof(param))) + return -EFAULT; + if (param.flags || param.reserved[0] || param.reserved[1] || + param.reserved[2] || !param.ranges || + !param.nr || param.nr > P2P_MEM_MAX_RANGES) + return -EINVAL; + + ranges = kvmalloc_array(param.nr, sizeof(*ranges), GFP_KERNEL); + if (!ranges) + return -ENOMEM; + if (copy_from_user(ranges, u64_to_user_ptr(param.ranges), + array_size(param.nr, sizeof(*ranges)))) { + err = -EFAULT; + goto free_ranges; + } + err = p2p_register_mem_ranges(batch, ranges, param.nr, &user_param->mem_handle); +free_ranges: + kvfree(ranges); + return err; +} + +static int p2p_unregister_mem(struct p2p_batch *batch, void __user *arg) +{ + struct p2p_mem_unregister_param param; + struct p2p_registered_mem *mem; + int err; + + if (copy_from_user(¶m, arg, sizeof(param))) + return -EFAULT; + if (param.reserved[0] || param.reserved[1] || param.reserved[2] || + !param.mem_handle) + return -EINVAL; + + err = p2p_unpublish_registered_mem(batch, param.mem_handle, &mem); + if (err) { + pr_err("unregister memory handle 0x%llx err %d\n", param.mem_handle, err); + return err; + } + + percpu_ref_kill(&mem->io_refs); + p2p_destroy_registered_mem(mem); + + pr_info("unregistered memory handle 0x%llx\n", param.mem_handle); + return 0; +} + +#ifdef DUMP_CONTENT +static void dump_pa_content(u64 pa, u64 size) +{ + unsigned int data_len = min_t(u64, size, 16U << 10); + char prefix[64]; + void *addr; + + addr = ioremap(pa, data_len); + if (!addr) + return; + + scnprintf(prefix, sizeof(prefix), "p2p: PA 0x%llx content: ", pa); + print_hex_dump(KERN_INFO, prefix, DUMP_PREFIX_ADDRESS, 16, 1, addr, data_len, false); + + iounmap(addr); +} +#else +static inline void dump_pa_content(u64 pa, u64 size) +{ +} +#endif + +static int get_pa_iov(struct p2p_batch *batch, int host_pid, + const struct p2p_iov *iov, unsigned int iov_nr, + struct p2p_pa_iov **pa_iov, unsigned int *pa_iov_nr, + struct p2p_pinned_io_mem *pinned_mem) +{ + struct p2p_provider *provider = READ_ONCE(batch->provider); + struct p2p_pinned_pa *pinned_pa; + struct p2p_pa_iov *new_pa_iov; + struct p2p_iov_map *maps; + unsigned int new_pa_iov_nr; + unsigned int new_pa_iov_idx; + unsigned int i; + int err; + + pinned_pa = kzalloc(sizeof(*pinned_pa), GFP_KERNEL); + if (!pinned_pa) + return -ENOMEM; + init_process_id(host_pid, &pinned_pa->process_id); + pinned_pa->maps = kvcalloc(iov_nr, sizeof(*pinned_pa->maps), GFP_KERNEL); + if (!pinned_pa->maps) { + err = -ENOMEM; + goto put_pinned_pa; + } + maps = pinned_pa->maps; + + new_pa_iov_nr = 0; + for (i = 0; i < iov_nr; i++) { + u32 range_size; + u32 pa_num; + u32 offset; + int page_size; + + page_size = p2p_mem_get_page_size(&pinned_pa->process_id, iov[i].addr, iov[i].size); + if (page_size < P2P_MIN_PAGE_SIZE || !is_power_of_2(page_size)) { + pr_err_ratelimited("invalid page size %d for addr 0x%llx, iov %u\n", + page_size, iov[i].addr, i); + err = -EINVAL; + goto put_pinned_pa; + } + + offset = iov[i].addr & (page_size - 1); + range_size = offset + iov[i].size; + pa_num = DIV_ROUND_UP(range_size, (u32)page_size); + if (!pa_num || pa_num > UINT_MAX - new_pa_iov_nr) { + err = -E2BIG; + goto put_pinned_pa; + } + + maps[i].aligned_addr = iov[i].addr - offset; + maps[i].page_size = page_size; + maps[i].pa_num = pa_num; + maps[i].aligned_size = (u64)pa_num * page_size; + new_pa_iov_nr += maps[i].pa_num; + } + + pinned_pa->pa_list = kvmalloc_array(new_pa_iov_nr, sizeof(*pinned_pa->pa_list), GFP_KERNEL); + if (!pinned_pa->pa_list) { + err = -ENOMEM; + goto put_pinned_pa; + } + + new_pa_iov = kvmalloc_array(new_pa_iov_nr, sizeof(*new_pa_iov), GFP_KERNEL); + if (!new_pa_iov) { + err = -ENOMEM; + goto put_pinned_pa; + } + + new_pa_iov_idx = 0; + for (i = 0; i < iov_nr; i++) { + u32 remaining = iov[i].size; + u32 offset = iov[i].addr - maps[i].aligned_addr; + unsigned int j; + + maps[i].pa_list = pinned_pa->pa_list + new_pa_iov_idx; + err = p2p_mem_get_pa_list(&pinned_pa->process_id, + maps[i].aligned_addr, + maps[i].aligned_size, + maps[i].pa_list, maps[i].pa_num); + if (err) { + pr_err_ratelimited("PA list addr 0x%llx size 0x%llx num %u iov %u err %d\n", + maps[i].aligned_addr, maps[i].aligned_size, + maps[i].pa_num, i, err); + goto free_pa_iov; + } + pinned_pa->pinned_map_nr++; + err = p2p_check_pa_list(&provider, maps[i].pa_list, maps[i].pa_num, + maps[i].page_size); + if (err) + goto free_pa_iov; + + for (j = 0; j < maps[i].pa_num; j++) { + u32 len = min(remaining, maps[i].page_size - offset); + + new_pa_iov[new_pa_iov_idx].addr = maps[i].pa_list[j] + offset; + new_pa_iov[new_pa_iov_idx].len = len; + new_pa_iov_idx++; + remaining -= len; + offset = 0; + } + + if (remaining) { + pr_err_ratelimited("PA list leaves 0x%x bytes for iov %u\n", + remaining, i); + err = -EINVAL; + goto free_pa_iov; + } + } + + err = p2p_provider_prepare(provider); + if (err) + goto free_pa_iov; + err = p2p_bind_provider(batch, provider); + if (err) + goto free_pa_iov; + *pa_iov = new_pa_iov; + *pa_iov_nr = new_pa_iov_nr; + *pinned_mem = (struct p2p_pinned_io_mem) { + .pinned = true, + .pinned_pa = pinned_pa, + }; + return 0; + +free_pa_iov: + kvfree(new_pa_iov); +put_pinned_pa: + p2p_put_pinned_pa(pinned_pa); + return err; +} + +static int p2p_get_registered_mem(u64 handle, struct p2p_registered_mem **mem_out) +{ + struct p2p_registered_mem *mem; + int err; + + rcu_read_lock(); + mem = xa_load(®istered_mems, handle); + if (!mem) { + err = -ENOENT; + goto unlock; + } + if (!percpu_ref_tryget_live(&mem->io_refs)) { + err = -ESTALE; + goto unlock; + } + rcu_read_unlock(); + if (mem->owner_tgid != task_tgid(current)) { + percpu_ref_put(&mem->io_refs); + return -EPERM; + } + + *mem_out = mem; + return 0; + +unlock: + rcu_read_unlock(); + return err; +} + +/* + * Find the segment containing @addr. Try the caller's cursor and its successor + * before binary search: the IOVs of one request usually walk the group in VA + * order. The cursor is per request so concurrent submitters share nothing. + */ +static const struct p2p_mem_seg *p2p_mem_find_seg(const struct p2p_registered_mem *mem, u64 addr, + unsigned int *cursor) +{ + unsigned int lo = *cursor; + unsigned int hi; + + if (lo < mem->nr_segs && addr >= mem->seg_starts[lo]) { + if (addr < mem->segs[lo].end) + goto found; + lo++; + if (lo < mem->nr_segs && addr >= mem->seg_starts[lo] && addr < mem->segs[lo].end) + goto found; + } + + /* Find the last segment starting at or below addr. */ + lo = 0; + hi = mem->nr_segs; + while (lo < hi) { + unsigned int mid = lo + (hi - lo) / 2; + + if (mem->seg_starts[mid] <= addr) + lo = mid + 1; + else + hi = mid; + } + if (!lo || addr >= mem->segs[lo - 1].end) + return NULL; + lo--; +found: + *cursor = lo; + return &mem->segs[lo]; +} + +/* Translate or count one IOV. It must fit in one translation segment. */ +static int p2p_mem_map_iov(const struct p2p_registered_mem *mem, const struct p2p_iov *iov, + unsigned int *cursor, struct p2p_pa_iov *pa_iov, u64 *pa_iov_nr) +{ + const struct p2p_mem_seg *seg; + u64 seg_offset; + u64 page_idx; + u32 page_offset; + u32 remaining = iov->size; + + seg = p2p_mem_find_seg(mem, iov->addr, cursor); + if (!seg || remaining > seg->end - iov->addr) + return -ERANGE; + + seg_offset = iov->addr - seg->aligned_addr; + page_idx = seg_offset / seg->page_size; + page_offset = seg_offset % seg->page_size; + if (!pa_iov) { + *pa_iov_nr += DIV_ROUND_UP((u64)page_offset + remaining, seg->page_size); + return 0; + } + while (remaining) { + u32 len = min(remaining, seg->page_size - page_offset); + + pa_iov[*pa_iov_nr].addr = seg->pa_list[page_idx] + page_offset; + pa_iov[*pa_iov_nr].len = len; + (*pa_iov_nr)++; + remaining -= len; + page_idx++; + page_offset = 0; + } + return 0; +} + +static int get_registered_pa_iov(struct p2p_batch *batch, u64 handle, + const struct p2p_iov *iov, unsigned int iov_nr, + struct p2p_pa_iov **pa_iov_out, unsigned int *pa_iov_nr_out, + struct p2p_pinned_io_mem *pinned_mem) +{ + struct p2p_registered_mem *mem; + struct p2p_pa_iov *pa_iov; + unsigned int cursor = 0; + u64 pa_iov_nr = 0; + unsigned int i; + int err; + + err = p2p_get_registered_mem(handle, &mem); + if (err) + return err; + /* The registration already supplies its provider. */ + err = p2p_bind_provider(batch, mem->provider); + if (err) + goto put_mem; + + for (i = 0; i < iov_nr; i++) { + err = p2p_mem_map_iov(mem, &iov[i], &cursor, NULL, &pa_iov_nr); + if (err) + goto put_mem; + if (pa_iov_nr > UINT_MAX) { + err = -E2BIG; + goto put_mem; + } + } + + pa_iov = kvmalloc_array(pa_iov_nr, sizeof(*pa_iov), GFP_KERNEL); + if (!pa_iov) { + err = -ENOMEM; + goto put_mem; + } + + cursor = 0; + pa_iov_nr = 0; + for (i = 0; i < iov_nr; i++) { + err = p2p_mem_map_iov(mem, &iov[i], &cursor, pa_iov, &pa_iov_nr); + if (err) + goto free_pa_iov; + } + + *pa_iov_out = pa_iov; + *pa_iov_nr_out = pa_iov_nr; + *pinned_mem = (struct p2p_pinned_io_mem) { + .pinned = true, + .reg_mem = true, + .registered_mem = mem, + }; + return 0; + +free_pa_iov: + kvfree(pa_iov); +put_mem: + percpu_ref_put(&mem->io_refs); + return err; +} + +static void p2p_unpin_io_mem(struct p2p_pinned_io_mem *pinned_mem) +{ + if (!pinned_mem->pinned) + return; + + if (pinned_mem->reg_mem) + percpu_ref_put(&pinned_mem->registered_mem->io_refs); + else + p2p_put_pinned_pa(pinned_mem->pinned_pa); + pinned_mem->pinned = false; +} + +static void free_io_ctx(struct p2p_io_context *io_ctx) +{ + p2p_unpin_io_mem(&io_ctx->pinned_mem); + kvfree(io_ctx->bvec); + kvfree(io_ctx->pa_iov); + fput(io_ctx->file); + kfree(io_ctx); +} + +static struct p2p_io_context *new_io_ctx(u32 op, struct file *file, struct p2p_pa_iov *pa_iov, + unsigned int pa_iov_nr, struct bio_vec *bvec, + unsigned int bvec_nr, + struct p2p_pinned_io_mem *pinned_mem, u64 data_size, + u64 user_data) +{ + struct p2p_io_context *io_ctx; + + io_ctx = kzalloc(sizeof(*io_ctx), GFP_KERNEL); + if (!io_ctx) + return ERR_PTR(-ENOMEM); + + io_ctx->op = op; + io_ctx->file = file; + io_ctx->pa_iov = pa_iov; + io_ctx->pa_iov_nr = pa_iov_nr; + io_ctx->bvec = bvec; + io_ctx->bvec_nr = bvec_nr; + io_ctx->pinned_mem = *pinned_mem; + io_ctx->data_size = data_size; + io_ctx->user_data = user_data; + + atomic_set(&io_ctx->io_ref, 1); + init_completion(&io_ctx->io_done); +#if defined(CALC_CRC32) || defined(DUMP_CONTENT) + INIT_WORK(&io_ctx->finalize_work, p2p_finalize_io_work); +#endif + + pinned_mem->pinned = false; + return io_ctx; +} + +static void p2p_wake_batch_waiters(struct p2p_batch *batch) +{ + /* + * Pair condition stores (ready_events / outstanding_events / cq_tail + * under completion_lock, then unlock) with waitqueue_active: without + * this full barrier a waiter can observe !active, then miss the + * published condition and sleep. wait_event* still re-checks after + * prepare_to_wait. + */ + smp_mb(); + if (waitqueue_active(&batch->wait)) + wake_up(&batch->wait); +} + +static void p2p_publish_io_done(struct p2p_io_context *io_ctx) +{ + struct p2p_batch *batch = io_ctx->batch; + unsigned long flags; + + /* + * Producer hot path: completion_lock only (AIO's ctx->completion_lock + * shape). Always publish; final close waits + * for natural publish then harvests under ring_lock. + */ + spin_lock_irqsave(&batch->completion_lock, flags); + WARN_ON_ONCE(batch->cq_tail - READ_ONCE(batch->cq_head) >= P2P_CQ_SIZE); + batch->cq[batch->cq_tail & P2P_CQ_MASK] = io_ctx; + /* Slot stores must be visible before cq_tail advances. */ + smp_wmb(); + WRITE_ONCE(batch->cq_tail, batch->cq_tail + 1); + atomic_inc(&batch->ready_events); + spin_unlock_irqrestore(&batch->completion_lock, flags); + + p2p_wake_batch_waiters(batch); + /* + * Last touch of io_ctx: collection and close wait for io_done before + * freeing. + */ + complete(&io_ctx->io_done); +} + +static const char *p2p_io_op_name(u32 op) +{ + return op == P2P_IO_WRITE ? "write" : "read"; +} + +#ifdef CALC_CRC32 +static void dump_io_ctx_crc32(const struct p2p_io_context *io_ctx) +{ + u32 crc = ~0U; + unsigned int i; + void *buffer; + + /* + * Logical-I/O finalization observed every NVMe DMA completion before + * this helper. Order the following CPU reads after those completions. + */ + dma_rmb(); + + buffer = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buffer) { + pr_err("crc32 failed file=%pD length=0x%llx err=%d\n", + io_ctx->file, io_ctx->data_size, -ENOMEM); + return; + } + + for (i = 0; i < io_ctx->pa_iov_nr; i++) { + const struct p2p_pa_iov *iov = &io_ctx->pa_iov[i]; + void __iomem *addr; + u64 offset = 0; + + addr = ioremap(iov->addr, iov->len); + if (!addr) { + pr_err("crc32 failed file=%pD length=0x%llx err=%d\n", + io_ctx->file, io_ctx->data_size, -ENOMEM); + kfree(buffer); + return; + } + + while (offset < iov->len) { + size_t chunk = min_t(u64, iov->len - offset, PAGE_SIZE); + + memcpy_fromio(buffer, addr + offset, chunk); + crc = crc32_le(crc, buffer, chunk); + offset += chunk; + } + iounmap(addr); + } + + kfree(buffer); + crc ^= ~0U; + pr_info("crc32 file=%pD length=0x%llx crc32=0x%08x\n", + io_ctx->file, io_ctx->data_size, crc); +} +#else +static inline void dump_io_ctx_crc32(const struct p2p_io_context *io_ctx) +{ +} +#endif + +#if defined(CALC_CRC32) || defined(DUMP_CONTENT) +static void p2p_finalize_io_work(struct work_struct *work) +{ + struct p2p_io_context *io_ctx = + container_of(work, struct p2p_io_context, finalize_work); + + dump_io_ctx_crc32(io_ctx); + dump_pa_content(io_ctx->pa_iov[0].addr, + min_t(u64, io_ctx->data_size, io_ctx->pa_iov[0].len)); + /* + * Registered unpin here (process context). One-shot stays pinned until + * retire; pinned_mem itself is an embedded field and remains valid. + */ + if (io_ctx->pinned_mem.reg_mem) + p2p_unpin_io_mem(&io_ctx->pinned_mem); + p2p_publish_io_done(io_ctx); +} +#endif + +static void p2p_io_ctx_put(struct p2p_io_context *io_ctx) +{ + if (!atomic_dec_and_test(&io_ctx->io_ref)) + return; + +#if defined(CALC_CRC32) || defined(DUMP_CONTENT) + if (io_ctx->op == P2P_IO_READ && !io_ctx->io_err) { + schedule_work(&io_ctx->finalize_work); + return; + } +#endif + + /* + * Registered: drop the percpu_ref on the completion path (safe). + * One-shot: leave pinned until p2p_retire_cq_head so + * devmm_put_mem_pa_list never runs in interrupt context. + */ + if (io_ctx->pinned_mem.reg_mem) + p2p_unpin_io_mem(&io_ctx->pinned_mem); + + p2p_publish_io_done(io_ctx); +} + +/* + * Record the VFS result of one logical I/O and drop the completion reference. + * The event carries no byte count, so a short transfer is an error. Runs in + * the context of the VFS completion: hard interrupt, a workqueue, or the + * submitter when the VFS completed the I/O without queuing it. + */ +static void p2p_io_done(struct p2p_io_context *io_ctx, long res) +{ + int err = 0; + + if (res < 0) + err = res; + else if ((u64)res != io_ctx->data_size) + err = -EIO; + + if (err) + pr_err_ratelimited("%s file=%pD length=0x%llx result %ld err %d\n", + p2p_io_op_name(io_ctx->op), io_ctx->file, + io_ctx->data_size, res, err); + p2p_stats_io_complete(io_ctx->op, err); + io_ctx->io_err = err; + p2p_io_ctx_put(io_ctx); +} + +static void p2p_kiocb_complete(struct kiocb *kiocb, long res) +{ + struct p2p_io_context *io_ctx = container_of(kiocb, struct p2p_io_context, kiocb); + + /* Queued writes retain freeze protection until completion. */ + if (io_ctx->op == P2P_IO_WRITE) + kiocb_end_write(kiocb); + p2p_io_done(io_ctx, res); +} + +/* + * Issue one logical I/O as one direct-I/O kiocb over the HBM pages. The + * filesystem maps the file offset and the block layer splits and DMA-maps the + * pages as PCI P2PDMA memory. The caller holds its own io_ctx reference: + * p2p_kiocb_complete() can run before the VFS call returns. + */ +static void p2p_issue_io(struct p2p_io_context *io_ctx, loff_t offset) +{ + struct kiocb *kiocb = &io_ctx->kiocb; + struct iov_iter iter; + ssize_t ret; + + init_sync_kiocb(kiocb, io_ctx->file); + kiocb->ki_pos = offset; + /* The fd need not be O_DIRECT: the direct path is the only P2P path. */ + kiocb->ki_flags |= IOCB_DIRECT; + kiocb->ki_complete = p2p_kiocb_complete; + + atomic_inc(&io_ctx->io_ref); + p2p_stats_io_issued(io_ctx->op, io_ctx->data_size); + + if (io_ctx->op == P2P_IO_WRITE) { + kiocb->ki_flags |= IOCB_WRITE; + iov_iter_bvec(&iter, ITER_SOURCE, io_ctx->bvec, io_ctx->bvec_nr, + io_ctx->data_size); + ret = vfs_iocb_iter_write(io_ctx->file, kiocb, &iter); + } else { + iov_iter_bvec(&iter, ITER_DEST, io_ctx->bvec, io_ctx->bvec_nr, + io_ctx->data_size); + ret = vfs_iocb_iter_read(io_ctx->file, kiocb, &iter); + } + + /* The VFS completed this I/O; ki_complete will not run. */ + if (ret != -EIOCBQUEUED) + p2p_io_done(io_ctx, ret); +} + +static int validate_io_param(const struct p2p_io_param *param) +{ + if (param->op != P2P_IO_READ && param->op != P2P_IO_WRITE) + return -EINVAL; + if (param->flags & ~P2P_IO_F_MASK) + return -EOPNOTSUPP; + if (!param->iov_nr || param->iov_nr > P2P_MAX_IOV) + return -EINVAL; + if (param->reserved || param->reserved2[0] || param->reserved2[1] || param->reserved2[2]) + return -EINVAL; + if (param->flags & P2P_IO_F_REGISTERED_MEM) + return !param->mem_handle || param->host_pid ? -EINVAL : 0; + + return param->host_pid <= 0 ? -EINVAL : 0; +} + +/* XFS can keep the data of a realtime file on another device than s_bdev. */ +static bool p2p_is_xfs_realtime(struct file *file) +{ + struct fileattr fa = { }; + + if (vfs_fileattr_get(file->f_path.dentry, &fa)) + return true; + return fa.fsx_xflags & FS_XFLAG_REALTIME; +} + +/* + * Check the backing filesystem and queue for an fd. Filesystem checks may + * use a directory fd; I/O checks require a file or block device with direct + * I/O support. + * + * The data must reach an NVMe queue that takes P2PDMA pages, and no layer on + * the way may touch the pages with the CPU. The checks hold no state, so they + * run for every request. + */ +static int p2p_check_target(struct file *file, struct block_device **bdev_out, + bool for_io) +{ + struct inode *inode = file_inode(file); + struct block_device *bdev; + + if (S_ISBLK(inode->i_mode)) { + bdev = I_BDEV(file->f_mapping->host); + } else if (S_ISREG(inode->i_mode) || (!for_io && S_ISDIR(inode->i_mode))) { + /* + * Only filesystems with an iomap direct-I/O path that keeps all + * file data on s_bdev. + */ + switch (inode->i_sb->s_magic) { + case EXT4_SUPER_MAGIC: + break; + case XFS_SUPER_MAGIC: + if (S_ISREG(inode->i_mode) && p2p_is_xfs_realtime(file)) + return -EOPNOTSUPP; + break; + default: + return -EOPNOTSUPP; + } + bdev = inode->i_sb->s_bdev; + if (!bdev) + return -EOPNOTSUPP; + } else { + return -EINVAL; + } + + if (for_io && !(file->f_mode & FMODE_CAN_ODIRECT)) + return -EOPNOTSUPP; + /* + * A bvec iterator bypasses the block layer's own P2PDMA queue check. + * The flag is set on a PCIe NVMe namespace queue and on dm/md queues + * only when their supported targets and lower devices pass P2PDMA. + */ + if (!blk_queue_pci_p2pdma(bdev_get_queue(bdev))) + return -EOPNOTSUPP; + /* Protection information is generated and verified with the CPU. */ + if (blk_get_integrity(bdev->bd_disk)) + return -EOPNOTSUPP; + + *bdev_out = bdev; + return 0; +} + +/* Validate payload sizes, address bounds and target alignment in one pass. */ +static int p2p_validate_io_range(struct file *file, struct block_device *bdev, + const struct p2p_io_param *param, const struct p2p_iov *iov, + u64 *data_size) +{ + unsigned int lbs_mask = bdev_logical_block_size(bdev) - 1; + u64 offset = param->offset; + u64 size = 0; + unsigned int i; + + /* An append kiocb ignores the offset. */ + if (file->f_flags & O_APPEND || offset > S64_MAX || (offset & lbs_mask)) + return -EINVAL; + for (i = 0; i < param->iov_nr; i++) { + u64 end; + + if (!iov[i].size || iov[i].size > P2P_MAX_IOV_SIZE || iov[i].reserved || + ((iov[i].addr | iov[i].size) & lbs_mask) || + check_add_overflow((u64)iov[i].addr, (u64)iov[i].size, &end) || + check_add_overflow(size, (u64)iov[i].size, &size)) + return -EINVAL; + } + if (size > P2P_MAX_IO_BYTES || size > S64_MAX - offset) + return -EINVAL; + *data_size = size; + return 0; +} + +/* Check the actual iterator after contiguous PA segments have been merged. */ +static int p2p_check_bvec_align(const struct bio_vec *bvec, unsigned int nr, + u32 lbs) +{ + unsigned int i; + + for (i = 0; i < nr; i++) { + u64 addr = page_to_phys(bvec[i].bv_page) + bvec[i].bv_offset; + + if (addr % lbs || bvec[i].bv_len % lbs) + return -EINVAL; + } + return 0; +} + +/* + * Turn the PA segments of one request into a bvec array over the P2PDMA pages + * of the HBM window. Every PA is inside the window (p2p_check_pa_list()), so + * pfn_to_page() names a valid page. Adjacent segments that are physically + * contiguous share one multi-page bvec. + */ +static struct bio_vec *p2p_build_bvec(const struct p2p_pa_iov *pa_iov, unsigned int pa_iov_nr, + unsigned int *bvec_nr_out) +{ + struct bio_vec *bvec; + unsigned int n = 0; + unsigned int i; + u64 next_pa = 0; + + bvec = kvmalloc_array(pa_iov_nr, sizeof(*bvec), GFP_KERNEL); + if (!bvec) + return NULL; + + for (i = 0; i < pa_iov_nr; i++) { + u64 pa = pa_iov[i].addr; + u32 len = pa_iov[i].len; + + /* P2P_MAX_IO_BYTES caps the sum below UINT_MAX. */ + if (n && pa == next_pa) + bvec[n - 1].bv_len += len; + else + bvec_set_page(&bvec[n++], pfn_to_page(PHYS_PFN(pa)), len, + offset_in_page(pa)); + next_pa = pa + len; + } + + *bvec_nr_out = n; + return bvec; +} + +static int p2p_submit_one(struct p2p_batch *batch, const struct p2p_io_param *param) +{ + struct p2p_pa_iov *pa_iov = NULL; + struct bio_vec *bvec = NULL; + struct p2p_iov *iov; + struct file *file; + struct block_device *bdev; + struct p2p_io_context *io_ctx; + struct p2p_pinned_io_mem pinned_mem = { }; + u64 data_size; + unsigned int pa_iov_nr; + unsigned int bvec_nr; + int err; + + err = validate_io_param(param); + if (err) + return err; + + iov = kvmalloc_array(param->iov_nr, sizeof(*iov), GFP_KERNEL); + if (!iov) + return -ENOMEM; + if (copy_from_user(iov, u64_to_user_ptr(param->iov), + array_size(param->iov_nr, sizeof(*iov)))) { + err = -EFAULT; + goto free_iov; + } + + file = fget(param->file_fd); + if (!file) { + err = -EBADF; + goto free_iov; + } + if (!(file->f_mode & (param->op == P2P_IO_WRITE ? FMODE_WRITE : FMODE_READ))) { + err = -EBADF; + goto put_file; + } + err = p2p_check_target(file, &bdev, true); + if (err) + goto put_file; + err = p2p_validate_io_range(file, bdev, param, iov, &data_size); + if (err) + goto put_file; + + if (param->flags & P2P_IO_F_REGISTERED_MEM) + err = get_registered_pa_iov(batch, param->mem_handle, iov, param->iov_nr, + &pa_iov, &pa_iov_nr, &pinned_mem); + else + err = get_pa_iov(batch, param->host_pid, iov, param->iov_nr, + &pa_iov, &pa_iov_nr, &pinned_mem); + if (err) + goto put_file; + + bvec = p2p_build_bvec(pa_iov, pa_iov_nr, &bvec_nr); + if (!bvec) { + err = -ENOMEM; + goto unpin; + } + err = p2p_check_bvec_align(bvec, bvec_nr, + bdev_logical_block_size(bdev)); + if (err) + goto unpin; + + io_ctx = new_io_ctx(param->op, file, pa_iov, pa_iov_nr, bvec, bvec_nr, &pinned_mem, + data_size, param->user_data); + if (IS_ERR(io_ctx)) { + err = PTR_ERR(io_ctx); + goto unpin; + } + /* io_ctx owns the file, the pin and both arrays from here on. */ + kvfree(iov); + + io_ctx->batch = batch; + if (atomic_inc_return(&batch->io_cnt) > P2P_CQ_SIZE) { + if (atomic_dec_and_test(&batch->io_cnt)) + p2p_wake_batch_waiters(batch); + free_io_ctx(io_ctx); + return -EAGAIN; + } + atomic_inc(&batch->outstanding_events); + + /* The item is accepted: every result from here on is an event. */ + p2p_issue_io(io_ctx, param->offset); + p2p_io_ctx_put(io_ctx); + return 0; + +unpin: + p2p_unpin_io_mem(&pinned_mem); + kvfree(bvec); + kvfree(pa_iov); +put_file: + fput(file); +free_iov: + kvfree(iov); + return err; +} + +static int p2p_submit_io(struct p2p_batch *batch, void __user *arg) +{ + struct p2p_io_batch_param param; + struct p2p_io_param *items; + unsigned int accepted = 0; + unsigned int i; + int err; + + if (copy_from_user(¶m, arg, sizeof(param))) + return -EFAULT; + /* Every accepted item needs one slot in this fd's completion queue. */ + if (!param.nr || param.nr > P2P_MAX_IO_NR || param.reserved) + return -EINVAL; + + items = kvmalloc_array(param.nr, sizeof(*items), GFP_KERNEL); + if (!items) + return -ENOMEM; + if (copy_from_user(items, u64_to_user_ptr(param.items), + array_size(param.nr, sizeof(*items)))) { + err = -EFAULT; + goto out; + } + + for (i = 0; i < param.nr; i++) { + err = p2p_submit_one(batch, &items[i]); + if (err) + goto out; + accepted++; + } + err = accepted; +out: + kvfree(items); + return accepted ? accepted : err; +} + +/* + * Pop the CQ head slot and free its io_ctx. Caller holds ring_lock and must + * have finished delivering (or discarding) the slot's single logical event. + * Counts for remaining events were already adjusted by the caller. + */ +static void p2p_retire_cq_head(struct p2p_batch *batch) +{ + struct p2p_io_context *io_ctx; + + /* Pairs with the producer's smp_wmb() before cq_tail. */ + smp_rmb(); + io_ctx = batch->cq[batch->cq_head & P2P_CQ_MASK]; + WRITE_ONCE(batch->cq_head, batch->cq_head + 1); + + /* Publisher's complete() is its final field access before unpin. */ + wait_for_completion(&io_ctx->io_done); + /* Publish cq_head before returning its CQ slot to submitters. */ + smp_mb__before_atomic(); + atomic_dec(&batch->io_cnt); + free_io_ctx(io_ctx); +} + +/* + * Final close waits for each accepted I/O, then releases its CQ entry. + * No other fd operation can run here: the final file reference is gone. + */ +static void p2p_release_io(struct p2p_batch *batch) +{ + mutex_lock(&batch->ring_lock); + while (atomic_read(&batch->io_cnt)) { + /* + * Sleep with ring_lock held so getevents cannot interleave. + * Publish only takes completion_lock, so it can still wake us. + */ + wait_event(batch->wait, + batch->cq_head != READ_ONCE(batch->cq_tail) || + !atomic_read(&batch->io_cnt)); + + while (batch->cq_head != READ_ONCE(batch->cq_tail)) { + atomic_dec(&batch->ready_events); + atomic_dec(&batch->outstanding_events); + p2p_wake_batch_waiters(batch); + p2p_retire_cq_head(batch); + } + } + mutex_unlock(&batch->ring_lock); +} + +static void p2p_fill_io_event(struct p2p_io_context *io_ctx, struct p2p_io_event *out) +{ + out->user_data = io_ctx->user_data; + out->res = io_ctx->io_err; + out->reserved[0] = 0; + out->reserved[1] = 0; + out->reserved[2] = 0; + out->reserved[3] = 0; +} + +static bool p2p_cq_ready(struct p2p_batch *batch, unsigned int min_nr) +{ + unsigned int ready = atomic_read(&batch->ready_events); + unsigned int outstanding = atomic_read(&batch->outstanding_events); + + return ready >= min_nr || ready == outstanding; +} + +/* + * AIO-style getevents: wait on ready logical I/Os, harvest under the + * ring_lock mutex only, copy one event at a time (no kvmalloc intermediate). + * Each CQ slot is one logical I/O / one event. On copy_to_user fault the slot + * stays at head for the next call. + */ +static int p2p_get_io_events(struct p2p_batch *batch, void __user *arg) +{ + struct p2p_getevents_param param; + struct p2p_io_event __user *user_events; + struct p2p_io_context *io_ctx; + struct p2p_io_event event; + unsigned int total = 0; + int ret = 0; + + if (copy_from_user(¶m, arg, sizeof(param))) + return -EFAULT; + if (param.min_nr < 0 || param.max_nr <= 0 || + param.min_nr > param.max_nr || !param.events) + return -EINVAL; + + user_events = u64_to_user_ptr(param.events); + + if (param.timeout_ns) { + ktime_t until = param.timeout_ns < 0 ? KTIME_MAX : + ns_to_ktime(param.timeout_ns); + + ret = wait_event_interruptible_hrtimeout( + batch->wait, p2p_cq_ready(batch, param.min_nr), until); + if (ret == -ERESTARTSYS) + ret = -EINTR; + else if (ret == -ETIME) + ret = 0; + /* Fall through and harvest whatever is already on the CQ. */ + } + + mutex_lock(&batch->ring_lock); + while (total < (unsigned int)param.max_nr) { + if (batch->cq_head == READ_ONCE(batch->cq_tail)) + break; + /* Pairs with the producer's smp_wmb() before cq_tail. */ + smp_rmb(); + io_ctx = batch->cq[batch->cq_head & P2P_CQ_MASK]; + + p2p_fill_io_event(io_ctx, &event); + if (copy_to_user(&user_events[total], &event, sizeof(event))) { + ret = -EFAULT; + break; + } + total++; + p2p_retire_cq_head(batch); + } + + /* + * Single consumer under ring_lock: one atomic_sub for all successfully + * copied events instead of a per-event atomic_dec. + */ + if (total) { + atomic_sub(total, &batch->ready_events); + atomic_sub(total, &batch->outstanding_events); + p2p_wake_batch_waiters(batch); + } + mutex_unlock(&batch->ring_lock); + + if (total) + return (int)total; + return ret; +} + +static int p2p_check_fd(void __user *arg) +{ + struct p2p_check_fd_param param; + struct block_device *bdev; + struct file *file; + int err; + + if (copy_from_user(¶m, arg, sizeof(param))) + return -EFAULT; + if (param.flags || param.reserved) + return -EINVAL; + file = fget(param.fd); + if (!file) + return -EBADF; + err = p2p_check_target(file, &bdev, false); + fput(file); + return err; +} + +static int p2p_get_info(void __user *arg) +{ + const struct p2p_device_info info = { .version = P2P_UAPI_VERSION }; + + return copy_to_user(arg, &info, sizeof(info)) ? -EFAULT : 0; +} + +static long p2p_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct p2p_batch *batch = file->private_data; + + switch (cmd) { + case IOCTL_GET_INFO: + return p2p_get_info((void __user *)arg); + case IOCTL_CHECK_FD: + return p2p_check_fd((void __user *)arg); + case IOCTL_SUBMIT_IO: + return p2p_submit_io(batch, (void __user *)arg); + case IOCTL_REGISTER_MEM: + return p2p_register_mem(batch, (void __user *)arg); + case IOCTL_UNREGISTER_MEM: + return p2p_unregister_mem(batch, (void __user *)arg); + case IOCTL_GET_IO_EVENTS: + return p2p_get_io_events(batch, (void __user *)arg); + default: + return -ENOTTY; + } +} + +/* Root only by default; a udev rule can grant access to the XDS node. */ +static char *p2p_devnode(const struct device *device, umode_t *mode) +{ + if (mode) + *mode = 0600; + return NULL; +} + +static void p2p_destroy_devices(void) +{ + device_destroy(dev_class, dev); +} + +static int p2p_create_devices(void) +{ + struct device *device; + + device = device_create(dev_class, NULL, dev, NULL, "p2p_device"); + if (IS_ERR(device)) { + pr_err("create device err %ld\n", PTR_ERR(device)); + return PTR_ERR(device); + } + return 0; +} + +static int __init p2p_drv_init(void) +{ + int err; + + err = p2p_mem_init(); + if (err) + return err; + + err = p2p_providers_init(); + if (err) + goto exit_mem; + + p2p_debugfs_init(); + + err = alloc_chrdev_region(&dev, 0, 1, "p2p_device"); + if (err < 0) { + pr_err("allocate major number err %d\n", err); + goto exit_debugfs; + } + + cdev_init(&p2p_cdev, &fops); + err = cdev_add(&p2p_cdev, dev, 1); + if (err < 0) { + pr_err("add the device to the system err %d\n", err); + goto free_dev; + } + + dev_class = class_create("p2p_class"); + if (IS_ERR(dev_class)) { + err = PTR_ERR(dev_class); + pr_err("create class err %d\n", err); + goto del_cdev; + } + dev_class->devnode = p2p_devnode; + + err = p2p_create_devices(); + if (err) + goto del_cls; + + pr_info("driver inserted done, major %d\n", MAJOR(dev)); + return 0; + +del_cls: + class_destroy(dev_class); +del_cdev: + cdev_del(&p2p_cdev); +free_dev: + unregister_chrdev_region(dev, 1); +exit_debugfs: + p2p_debugfs_exit(); + p2p_providers_exit(); +exit_mem: + p2p_mem_exit(); + return err; +} + +static void __exit p2p_drv_exit(void) +{ + p2p_debugfs_exit(); + p2p_destroy_devices(); + class_destroy(dev_class); + cdev_del(&p2p_cdev); + unregister_chrdev_region(dev, 1); + WARN_ON_ONCE(!xa_empty(®istered_mems)); + xa_destroy(®istered_mems); + /* Drain the registered-memory RCU frees before the code goes away. */ + rcu_barrier(); + /* No fd is open any more: release the HBM pools and the PCI devices. */ + p2p_providers_exit(); + p2p_mem_exit(); + pr_info("driver removed done\n"); +} + +module_init(p2p_drv_init); +module_exit(p2p_drv_exit); +MODULE_DESCRIPTION("XDS NVMe peer-to-peer I/O driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/nds/mem.c b/drivers/nds/mem.c new file mode 100644 index 000000000000..f34f15d02ca4 --- /dev/null +++ b/drivers/nds/mem.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +#define pr_fmt(fmt) "p2p: " fmt + +#include <linux/errno.h> +#include <linux/module.h> + +#include "mem.h" + +static typeof(devmm_get_mem_pa_list) *get_mem_pa_list; +static typeof(devmm_put_mem_pa_list) *put_mem_pa_list; +static typeof(devmm_get_mem_page_size) *get_mem_page_size; + +int p2p_mem_init(void) +{ + get_mem_pa_list = symbol_get(devmm_get_mem_pa_list); + if (!get_mem_pa_list) { + pr_err("required symbol devmm_get_mem_pa_list is unavailable\n"); + return -ENODEV; + } + + put_mem_pa_list = symbol_get(devmm_put_mem_pa_list); + if (!put_mem_pa_list) { + pr_err("required symbol devmm_put_mem_pa_list is unavailable\n"); + goto put_get_mem_pa_list; + } + + get_mem_page_size = symbol_get(devmm_get_mem_page_size); + if (!get_mem_page_size) { + pr_err("required symbol devmm_get_mem_page_size is unavailable\n"); + goto put_put_mem_pa_list; + } + + return 0; + +put_put_mem_pa_list: + symbol_put(devmm_put_mem_pa_list); +put_get_mem_pa_list: + symbol_put(devmm_get_mem_pa_list); + return -ENODEV; +} + +int p2p_mem_get_pa_list(struct devmm_svm_process_id *process_id, u64 addr, + u64 size, u64 *pa_list, u32 pa_num) +{ + return get_mem_pa_list(process_id, addr, size, pa_list, pa_num); +} + +void p2p_mem_put_pa_list(struct devmm_svm_process_id *process_id, u64 addr, + u64 size, u64 *pa_list, u32 pa_num) +{ + put_mem_pa_list(process_id, addr, size, pa_list, pa_num); +} + +int p2p_mem_get_page_size(struct devmm_svm_process_id *process_id, u64 addr, + u64 size) +{ + return get_mem_page_size(process_id, addr, size); +} + +void p2p_mem_exit(void) +{ + symbol_put(devmm_get_mem_page_size); + symbol_put(devmm_put_mem_pa_list); + symbol_put(devmm_get_mem_pa_list); +} diff --git a/drivers/nds/mem.h b/drivers/nds/mem.h new file mode 100644 index 000000000000..617bf44a1346 --- /dev/null +++ b/drivers/nds/mem.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef P2P_MEM_H_ +#define P2P_MEM_H_ + +#include "mem_abi.h" + +int p2p_mem_get_pa_list(struct devmm_svm_process_id *process_id, u64 addr, + u64 size, u64 *pa_list, u32 pa_num); +void p2p_mem_put_pa_list(struct devmm_svm_process_id *process_id, u64 addr, + u64 size, u64 *pa_list, u32 pa_num); +int p2p_mem_get_page_size(struct devmm_svm_process_id *process_id, u64 addr, + u64 size); + +int p2p_mem_init(void); +void p2p_mem_exit(void); + +#endif diff --git a/drivers/nds/mem_abi.h b/drivers/nds/mem_abi.h new file mode 100644 index 000000000000..10c5a515c64c --- /dev/null +++ b/drivers/nds/mem_abi.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef P2P_MEM_ABI_H_ +#define P2P_MEM_ABI_H_ + +#include <linux/types.h> + +struct devmm_svm_process_id { + int32_t host_pid; + union { + uint16_t devid; + uint16_t vm_id; + }; + uint16_t vfid; +}; + +int devmm_get_mem_pa_list(struct devmm_svm_process_id *process_id, u64 addr, + u64 size, u64 *pa_list, u32 pa_num); +void devmm_put_mem_pa_list(struct devmm_svm_process_id *process_id, u64 addr, + u64 size, u64 *pa_list, u32 pa_num); +int devmm_get_mem_page_size(struct devmm_svm_process_id *process_id, u64 addr, + u64 size); + +#endif diff --git a/drivers/nds/provider.c b/drivers/nds/provider.c new file mode 100644 index 000000000000..b110ba3dbd9a --- /dev/null +++ b/drivers/nds/provider.c @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: GPL-2.0 +#define pr_fmt(fmt) "p2p: " fmt + +#include <linux/kernel.h> +#include <linux/memremap.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/pci.h> +#include <linux/pci-p2pdma.h> +#include <linux/pfn.h> +#include <linux/ratelimit.h> +#include <linux/sort.h> + +#include "provider.h" + +/* + * The PCI BAR that holds HBM. BAR 4 on the PCI device; the test stub backs + * "HBM" with an NVMe controller memory buffer, which QEMU puts in BAR 2. + */ +static int hbm_bar = 4; +module_param(hbm_bar, int, 0444); +MODULE_PARM_DESC(hbm_bar, "PCI BAR of the PCI device that holds HBM (default 4)"); + +static unsigned long pagemap_align; +module_param(pagemap_align, ulong, 0444); +MODULE_PARM_DESC(pagemap_align, "PCI HBM pagemap alignment: 0=auto, 2M or 1G"); + +static struct p2p_provider providers[P2P_MAX_DEVS]; +static unsigned int nr_providers; +static DEFINE_MUTEX(provider_lock); + +static int p2p_provider_cmp(const void *a, const void *b) +{ + const struct p2p_provider *pa = a; + const struct p2p_provider *pb = b; + + if (pa->start < pb->start) + return -1; + return pa->start > pb->start; +} + +struct p2p_provider *p2p_provider_find(u64 pa, u64 len) +{ + unsigned int lo = 0; + unsigned int hi = nr_providers; + + while (lo < hi) { + unsigned int mid = lo + (hi - lo) / 2; + struct p2p_provider *provider = &providers[mid]; + + if (pa < provider->start) + hi = mid; + else if (pa >= provider->end) + lo = mid + 1; + else + return p2p_provider_contains(provider, pa, len) ? provider : NULL; + } + return NULL; +} + +static const struct pci_device_id p2p_npu_device_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xd802) }, + { } +}; + +/* QEMU NVMe CMB used by stub.ko; it has no memory BAR 4. */ +static const struct pci_device_id p2p_stub_device_ids[] = { + { PCI_DEVICE(0x1b36, 0x0010) }, + { } +}; + +static bool p2p_provider_can_dedup(const struct p2p_provider *provider) +{ + return hbm_bar == 4 && pci_match_id(p2p_npu_device_ids, provider->pdev); +} + +static int p2p_provider_register_pages(struct p2p_provider *provider) +{ +#ifdef CONFIG_ARCH_WANT_OPTIMIZE_P2PDMA_VMEMMAP + size_t size, align; + + if (!p2p_provider_can_dedup(provider)) + return pci_p2pdma_add_resource(provider->pdev, hbm_bar, 0, 0); + + size = pci_resource_len(provider->pdev, hbm_bar); + align = pci_p2pdma_max_pagemap_align(provider->pdev, hbm_bar, size, 0); + if (pagemap_align) { + if (align < pagemap_align) { + pci_err(provider->pdev, + "requested pagemap alignment %#lx exceeds maximum %#zx\n", + pagemap_align, align); + return -EOPNOTSUPP; + } + align = pagemap_align; + } + if (align == PAGE_SIZE) + return pci_p2pdma_add_resource(provider->pdev, hbm_bar, 0, 0); + + return pci_p2pdma_add_resource_aligned(provider->pdev, hbm_bar, 0, align, 0); +#else + if (pagemap_align) { + pci_err(provider->pdev, + "pagemap alignment requires P2PDMA vmemmap support\n"); + return -EOPNOTSUPP; + } + return pci_p2pdma_add_resource(provider->pdev, hbm_bar, 0, 0); +#endif +} + +static void p2p_provider_remove_pages(struct p2p_provider *provider) +{ + if (!provider->page_resources) + return; + /* + * Run the devm_memremap_pages() unmap action, then free the P2PDMA + * pool and its metadata. Unmapping only the pages would leave a stale + * pool entry. Reused resources are outside this group. + */ + devres_release_group(&provider->pdev->dev, provider->page_resources); + provider->page_resources = NULL; +} + +/* Give the window struct pages, or reuse an exact existing P2PDMA mapping. */ +static int p2p_provider_add_pages(struct p2p_provider *provider) +{ + struct dev_pagemap *pgmap; + int err = 0; + + pgmap = get_dev_pagemap(PHYS_PFN(provider->start), NULL); + if (!pgmap) { + /* + * A new range in an existing pool cannot be removed separately: + * the PCI API has no operation to remove a gen_pool range. + * Only create a pool whose full lifetime we can manage. + */ + if (rcu_access_pointer(provider->pdev->p2pdma)) { + pci_err(provider->pdev, "cannot add HBM to an existing P2PDMA pool\n"); + return -EBUSY; + } + provider->page_resources = devres_open_group(&provider->pdev->dev, NULL, + GFP_KERNEL); + if (!provider->page_resources) + return -ENOMEM; + err = p2p_provider_register_pages(provider); + devres_close_group(&provider->pdev->dev, provider->page_resources); + if (err) + p2p_provider_remove_pages(provider); + return err; + } + + if (pgmap->type != MEMORY_DEVICE_PCI_P2PDMA || + pgmap->range.start != provider->start || + pgmap->range.end != provider->end - 1) { + pci_err(provider->pdev, "BAR %d is covered by a foreign pagemap\n", hbm_bar); + err = -EBUSY; + } else if (p2p_provider_can_dedup(provider) && pagemap_align && + pgmap->vmemmap_shift != ilog2(pagemap_align) - PAGE_SHIFT) { + pci_err(provider->pdev, + "HBM pagemap shift %lu does not match requested alignment %#lx\n", + pgmap->vmemmap_shift, pagemap_align); + err = -EBUSY; + } else { + pci_info(provider->pdev, "reusing HBM pagemap with vmemmap shift %lu\n", + pgmap->vmemmap_shift); + } + put_dev_pagemap(pgmap); + return err; +} + +int p2p_provider_prepare(struct p2p_provider *provider) +{ + size_t size = provider->end - provider->start; + int err = 0; + + /* Acquire the page mapping and pool published by the release store. */ + if (smp_load_acquire(&provider->ready)) + return 0; + + mutex_lock(&provider_lock); + if (provider->ready) + goto unlock; + + /* Keep PCI driver probe/unbind from changing devres during setup. */ + device_lock(&provider->pdev->dev); + err = p2p_provider_add_pages(provider); + if (err) + goto unlock_device; + + /* + * Take the whole pool, so nobody else gets HBM from it (sysfs + * p2pmem/allocate, in-kernel P2PDMA clients). The allocation also holds + * a pagemap reference: an early unbind of the PCI device blocks until + * module exit instead of freeing struct pages under in-flight I/O. + */ + provider->pool_addr = pci_alloc_p2pmem(provider->pdev, size); + if (!provider->pool_addr) { + pci_err(provider->pdev, "HBM p2pmem pool is in use\n"); + err = -EBUSY; + p2p_provider_remove_pages(provider); + goto unlock_device; + } + + /* Publish the page mapping and reserved pool to concurrent callers. */ + smp_store_release(&provider->ready, true); + pci_info(provider->pdev, "HBM window [0x%llx, 0x%llx) is ready\n", + provider->start, provider->end); +unlock_device: + device_unlock(&provider->pdev->dev); +unlock: + mutex_unlock(&provider_lock); + return err; +} + +int p2p_providers_init(void) +{ + struct pci_dev *pdev = NULL; + unsigned int found = 0; + + if (hbm_bar < 0 || hbm_bar >= PCI_STD_NUM_BARS) { + pr_err("invalid hbm_bar %d\n", hbm_bar); + return -EINVAL; + } + if (pagemap_align && pagemap_align != PMD_SIZE && pagemap_align != PUD_SIZE) { + pr_err("invalid pagemap_align %#lx; use 0, %#lx or %#lx\n", pagemap_align, + (unsigned long)PMD_SIZE, (unsigned long)PUD_SIZE); + return -EINVAL; + } +#ifndef CONFIG_ARCH_WANT_OPTIMIZE_P2PDMA_VMEMMAP + if (pagemap_align) { + pr_err("pagemap_align requires P2PDMA vmemmap support\n"); + return -EOPNOTSUPP; + } +#endif + + for_each_pci_dev(pdev) { + struct p2p_provider *provider; + + if (!pci_match_id(p2p_npu_device_ids, pdev) && + !pci_match_id(p2p_stub_device_ids, pdev)) + continue; + if (!(pci_resource_flags(pdev, hbm_bar) & IORESOURCE_MEM) || + !pci_resource_len(pdev, hbm_bar)) { + pci_warn(pdev, "no memory BAR %d, skipped\n", hbm_bar); + continue; + } + if (found == P2P_MAX_DEVS) { + pci_warn(pdev, "provider limit %u reached, skipped\n", P2P_MAX_DEVS); + continue; + } + + provider = &providers[found++]; + /* Hold a reference after for_each_pci_dev() drops its own. */ + provider->pdev = pci_dev_get(pdev); + provider->start = pci_resource_start(pdev, hbm_bar); + provider->end = provider->start + pci_resource_len(pdev, hbm_bar); + } + + sort(providers, found, sizeof(*providers), p2p_provider_cmp, NULL); + nr_providers = found; + pr_info("found %u PCI P2PDMA provider(s), HBM in BAR %d\n", found, hbm_bar); + return 0; +} + +void p2p_providers_exit(void) +{ + unsigned int i; + + for (i = 0; i < nr_providers; i++) { + struct p2p_provider *provider = &providers[i]; + + if (provider->ready) + pci_free_p2pmem(provider->pdev, provider->pool_addr, + provider->end - provider->start); + /* Return the pool before unmapping its pages. */ + p2p_provider_remove_pages(provider); + pci_dev_put(provider->pdev); + *provider = (struct p2p_provider) { }; + } + nr_providers = 0; +} diff --git a/drivers/nds/provider.h b/drivers/nds/provider.h new file mode 100644 index 000000000000..40a62748364c --- /dev/null +++ b/drivers/nds/provider.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef P2P_PROVIDER_H_ +#define P2P_PROVIDER_H_ + +#include <linux/types.h> + +struct pci_dev; + +/* One provider: the PCI device whose HBM BAR is a PCI P2PDMA provider. */ +struct p2p_provider { + struct pci_dev *pdev; + /* HBM window [start, end) in CPU physical address space. */ + u64 start; + u64 end; + /* Set once the window has struct pages and its pool is reserved. */ + bool ready; + void *pool_addr; + /* Owns new P2PDMA resources; NULL for reused mappings. */ + void *page_resources; +}; + +#define P2P_MAX_DEVS 32U + +int p2p_providers_init(void); +void p2p_providers_exit(void); + +/* Find the provider whose HBM window contains all of [@pa, @pa + @len). */ +struct p2p_provider *p2p_provider_find(u64 pa, u64 len); + +/* Make the HBM window a P2PDMA provider. Idempotent; may sleep. */ +int p2p_provider_prepare(struct p2p_provider *provider); + +static inline bool p2p_provider_contains(const struct p2p_provider *provider, + u64 pa, u64 len) +{ + return pa >= provider->start && pa < provider->end && + len <= provider->end - pa; +} + +#endif diff --git a/include/uapi/linux/nds_p2p.h b/include/uapi/linux/nds_p2p.h new file mode 100644 index 000000000000..f69bd610a41f --- /dev/null +++ b/include/uapi/linux/nds_p2p.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef NDS_P2P_H_ +#define NDS_P2P_H_ + +#include <linux/types.h> + +/* + * UAPI v2: PCI P2PDMA + asynchronous VFS I/O. + * + * All clients open /dev/p2p_device. An fd learns its provider from the first + * registered or one-shot buffer. Later buffers on that fd must use the same + * provider. No request or device-node name carries a PCI device id. + */ +/* GET_INFO uses bits 15:8 for major and bits 7:0 for minor. */ +#define P2P_UAPI_VERSION_MAJOR 2U +#define P2P_UAPI_VERSION_MINOR 0U +#define P2P_UAPI_VERSION ((P2P_UAPI_VERSION_MAJOR << 8) | P2P_UAPI_VERSION_MINOR) + +struct p2p_iov { + __u64 addr; /* HBM addr */ + __u32 size; + __u32 reserved; /* must be zero */ +}; + +#define P2P_IO_F_REGISTERED_MEM (1U << 0) +#define P2P_IO_F_MASK P2P_IO_F_REGISTERED_MEM + +#define P2P_IO_READ 0U +#define P2P_IO_WRITE 1U + +/* One logical I/O moves at most this many bytes (the kernel MAX_RW_COUNT). */ +#define P2P_MAX_IO_BYTES 0x7ffff000U + +/* + * Fixed-size request; iov is a userspace pointer. One batch item is one + * logical I/O at byte offset @offset in @file_fd, covering all iov segments + * and completing as one event carrying user_data. A transfer shorter than the + * iov total completes with -EIO. + */ +struct p2p_io_param { + unsigned int op; /* P2P_IO_READ / P2P_IO_WRITE */ + unsigned int flags; /* P2P_IO_F_* */ + int file_fd; /* regular file or block device; no O_APPEND */ + int host_pid; /* one-shot buffer owner; 0 if registered */ + __u64 mem_handle; /* from IOCTL_REGISTER_MEM, or 0 */ + __u64 user_data; /* echo'd in the completion event */ + __u64 offset; /* byte offset in file_fd */ + __u64 iov; /* userspace pointer to struct p2p_iov[] */ + unsigned int iov_nr; + unsigned int reserved; /* must be zero */ + __u64 reserved2[3]; /* must be zero */ +}; + +#define P2P_MAX_IO_NR 1024U + +/* + * Submit independent p2p_io_param records in array order. The ioctl stops at + * the first item that cannot be accepted and returns the accepted prefix, or + * a negative errno when the first item fails. Each accepted item produces one + * completion event. The ioctl may sleep in the filesystem. + */ +struct p2p_io_batch_param { + __u64 items; /* userspace pointer to p2p_io_param[] */ + __u32 nr; + __u32 reserved; +}; + +struct p2p_io_event { + __u64 user_data; + __s64 res; /* 0 on success or -errno */ + __u64 reserved[4]; +}; + +struct p2p_getevents_param { + __s32 min_nr; + __s32 max_nr; + __s64 timeout_ns; /* <0 infinite, 0 nonblock, >0 relative */ + __u64 events; /* userspace pointer to struct p2p_io_event[] */ +}; + +struct p2p_mem_unregister_param { + __u64 mem_handle; + __u64 reserved[3]; +}; + +struct p2p_mem_range { + __u64 addr; + __u64 size; +}; + +#define P2P_MEM_MAX_RANGES 8192U + +/* + * Register several non-overlapping VA ranges under one handle. Registered I/O + * with that handle may address any registered range. Each IOV must fit in one + * translation segment; crossing a segment boundary fails with -ERANGE. Adjacent + * ranges merge only at a page-aligned boundary with the same page size. Every + * range must be in one provider's window and match the fd's cached provider. + * A different provider fails with -EXDEV; an address outside every provider + * fails with -EFAULT. + * IOCTL_UNREGISTER_MEM releases the registration. + */ +struct p2p_mem_register_param { + __u64 ranges; /* userspace pointer to struct p2p_mem_range[] */ + __u32 nr; /* 1..P2P_MEM_MAX_RANGES */ + __u32 flags; /* must be zero */ + __u64 reserved[3]; /* must be zero */ + __u64 mem_handle; /* output */ +}; + +/* Validate a file, directory or block-device fd's filesystem and P2PDMA queue. */ +struct p2p_check_fd_param { + __s32 fd; + __u32 flags; /* must be zero */ + __u64 reserved; /* must be zero */ +}; + +/* Output only. Feature flags and reserved fields are zero for version 2.0. */ +struct p2p_device_info { + __u32 version; /* major in bits 15:8, minor in bits 7:0; upper bits zero */ + __u32 flags; + __u64 reserved[3]; +}; + +#define IOCTL_GET_INFO _IOR('k', 1, struct p2p_device_info) +#define IOCTL_CHECK_FD _IOW('k', 2, struct p2p_check_fd_param) +#define IOCTL_REGISTER_MEM _IOWR('k', 3, struct p2p_mem_register_param) +#define IOCTL_UNREGISTER_MEM _IOW('k', 4, struct p2p_mem_unregister_param) +#define IOCTL_SUBMIT_IO _IOWR('k', 5, struct p2p_io_batch_param) +#define IOCTL_GET_IO_EVENTS _IOWR('k', 6, struct p2p_getevents_param) + +#endif -- 2.52.0
From: Hou Tao <houtao1@huawei.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 -------------------------------- Add tools/lib/nds with the NDS userspace API. The library registers device memory, submits asynchronous P2P I/O, and harvests completion events through the nds_p2p UAPI. Signed-off-by: Hou Tao <houtao1@huawei.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- tools/include/uapi/linux/nds_p2p.h | 132 ++++ tools/lib/nds/Makefile | 13 + tools/lib/nds/file_p2p/.gitignore | 8 + tools/lib/nds/file_p2p/Makefile | 35 + tools/lib/nds/file_p2p/file_p2p_api.c | 136 ++++ tools/lib/nds/file_p2p/file_p2p_api.h | 36 + tools/lib/nds/file_p2p/nds_api.c | 860 ++++++++++++++++++++++ tools/lib/nds/file_p2p/nds_api.h | 249 +++++++ tools/lib/nds/file_p2p/nds_api_internal.h | 12 + tools/lib/nds/file_p2p/p2p_common.c | 69 ++ tools/lib/nds/file_p2p/p2p_common.h | 16 + tools/lib/nds/file_p2p/py_file_p2p_api.c | 320 ++++++++ tools/lib/nds/file_p2p/py_nds_api.c | 577 +++++++++++++++ tools/lib/nds/file_p2p/setup.py | 32 + 14 files changed, 2495 insertions(+) create mode 100644 tools/include/uapi/linux/nds_p2p.h create mode 100644 tools/lib/nds/Makefile create mode 100644 tools/lib/nds/file_p2p/.gitignore create mode 100644 tools/lib/nds/file_p2p/Makefile create mode 100644 tools/lib/nds/file_p2p/file_p2p_api.c create mode 100644 tools/lib/nds/file_p2p/file_p2p_api.h create mode 100644 tools/lib/nds/file_p2p/nds_api.c create mode 100644 tools/lib/nds/file_p2p/nds_api.h create mode 100644 tools/lib/nds/file_p2p/nds_api_internal.h create mode 100644 tools/lib/nds/file_p2p/p2p_common.c create mode 100644 tools/lib/nds/file_p2p/p2p_common.h create mode 100644 tools/lib/nds/file_p2p/py_file_p2p_api.c create mode 100644 tools/lib/nds/file_p2p/py_nds_api.c create mode 100644 tools/lib/nds/file_p2p/setup.py diff --git a/tools/include/uapi/linux/nds_p2p.h b/tools/include/uapi/linux/nds_p2p.h new file mode 100644 index 000000000000..f69bd610a41f --- /dev/null +++ b/tools/include/uapi/linux/nds_p2p.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef NDS_P2P_H_ +#define NDS_P2P_H_ + +#include <linux/types.h> + +/* + * UAPI v2: PCI P2PDMA + asynchronous VFS I/O. + * + * All clients open /dev/p2p_device. An fd learns its provider from the first + * registered or one-shot buffer. Later buffers on that fd must use the same + * provider. No request or device-node name carries a PCI device id. + */ +/* GET_INFO uses bits 15:8 for major and bits 7:0 for minor. */ +#define P2P_UAPI_VERSION_MAJOR 2U +#define P2P_UAPI_VERSION_MINOR 0U +#define P2P_UAPI_VERSION ((P2P_UAPI_VERSION_MAJOR << 8) | P2P_UAPI_VERSION_MINOR) + +struct p2p_iov { + __u64 addr; /* HBM addr */ + __u32 size; + __u32 reserved; /* must be zero */ +}; + +#define P2P_IO_F_REGISTERED_MEM (1U << 0) +#define P2P_IO_F_MASK P2P_IO_F_REGISTERED_MEM + +#define P2P_IO_READ 0U +#define P2P_IO_WRITE 1U + +/* One logical I/O moves at most this many bytes (the kernel MAX_RW_COUNT). */ +#define P2P_MAX_IO_BYTES 0x7ffff000U + +/* + * Fixed-size request; iov is a userspace pointer. One batch item is one + * logical I/O at byte offset @offset in @file_fd, covering all iov segments + * and completing as one event carrying user_data. A transfer shorter than the + * iov total completes with -EIO. + */ +struct p2p_io_param { + unsigned int op; /* P2P_IO_READ / P2P_IO_WRITE */ + unsigned int flags; /* P2P_IO_F_* */ + int file_fd; /* regular file or block device; no O_APPEND */ + int host_pid; /* one-shot buffer owner; 0 if registered */ + __u64 mem_handle; /* from IOCTL_REGISTER_MEM, or 0 */ + __u64 user_data; /* echo'd in the completion event */ + __u64 offset; /* byte offset in file_fd */ + __u64 iov; /* userspace pointer to struct p2p_iov[] */ + unsigned int iov_nr; + unsigned int reserved; /* must be zero */ + __u64 reserved2[3]; /* must be zero */ +}; + +#define P2P_MAX_IO_NR 1024U + +/* + * Submit independent p2p_io_param records in array order. The ioctl stops at + * the first item that cannot be accepted and returns the accepted prefix, or + * a negative errno when the first item fails. Each accepted item produces one + * completion event. The ioctl may sleep in the filesystem. + */ +struct p2p_io_batch_param { + __u64 items; /* userspace pointer to p2p_io_param[] */ + __u32 nr; + __u32 reserved; +}; + +struct p2p_io_event { + __u64 user_data; + __s64 res; /* 0 on success or -errno */ + __u64 reserved[4]; +}; + +struct p2p_getevents_param { + __s32 min_nr; + __s32 max_nr; + __s64 timeout_ns; /* <0 infinite, 0 nonblock, >0 relative */ + __u64 events; /* userspace pointer to struct p2p_io_event[] */ +}; + +struct p2p_mem_unregister_param { + __u64 mem_handle; + __u64 reserved[3]; +}; + +struct p2p_mem_range { + __u64 addr; + __u64 size; +}; + +#define P2P_MEM_MAX_RANGES 8192U + +/* + * Register several non-overlapping VA ranges under one handle. Registered I/O + * with that handle may address any registered range. Each IOV must fit in one + * translation segment; crossing a segment boundary fails with -ERANGE. Adjacent + * ranges merge only at a page-aligned boundary with the same page size. Every + * range must be in one provider's window and match the fd's cached provider. + * A different provider fails with -EXDEV; an address outside every provider + * fails with -EFAULT. + * IOCTL_UNREGISTER_MEM releases the registration. + */ +struct p2p_mem_register_param { + __u64 ranges; /* userspace pointer to struct p2p_mem_range[] */ + __u32 nr; /* 1..P2P_MEM_MAX_RANGES */ + __u32 flags; /* must be zero */ + __u64 reserved[3]; /* must be zero */ + __u64 mem_handle; /* output */ +}; + +/* Validate a file, directory or block-device fd's filesystem and P2PDMA queue. */ +struct p2p_check_fd_param { + __s32 fd; + __u32 flags; /* must be zero */ + __u64 reserved; /* must be zero */ +}; + +/* Output only. Feature flags and reserved fields are zero for version 2.0. */ +struct p2p_device_info { + __u32 version; /* major in bits 15:8, minor in bits 7:0; upper bits zero */ + __u32 flags; + __u64 reserved[3]; +}; + +#define IOCTL_GET_INFO _IOR('k', 1, struct p2p_device_info) +#define IOCTL_CHECK_FD _IOW('k', 2, struct p2p_check_fd_param) +#define IOCTL_REGISTER_MEM _IOWR('k', 3, struct p2p_mem_register_param) +#define IOCTL_UNREGISTER_MEM _IOW('k', 4, struct p2p_mem_unregister_param) +#define IOCTL_SUBMIT_IO _IOWR('k', 5, struct p2p_io_batch_param) +#define IOCTL_GET_IO_EVENTS _IOWR('k', 6, struct p2p_getevents_param) + +#endif diff --git a/tools/lib/nds/Makefile b/tools/lib/nds/Makefile new file mode 100644 index 000000000000..dd691a8312f4 --- /dev/null +++ b/tools/lib/nds/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# NDS userspace library (tools/lib/nds) + +.PHONY: all clean lib + +all: lib + +lib: + $(MAKE) -C file_p2p + +clean: + $(MAKE) -C file_p2p clean diff --git a/tools/lib/nds/file_p2p/.gitignore b/tools/lib/nds/file_p2p/.gitignore new file mode 100644 index 000000000000..997fbf047228 --- /dev/null +++ b/tools/lib/nds/file_p2p/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +build/ +file_p2p*.so +nds*.so +libfile_p2p.so +libnds.so* +.*.d +/nds_example diff --git a/tools/lib/nds/file_p2p/Makefile b/tools/lib/nds/file_p2p/Makefile new file mode 100644 index 000000000000..ddb716d63221 --- /dev/null +++ b/tools/lib/nds/file_p2p/Makefile @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: GPL-2.0 +CC ?= cc +CFLAGS ?= -O2 -Wall -Wextra -Werror -g -ggdb -fno-omit-frame-pointer +UAPI_DIR ?= $(abspath ../../../include/uapi) +CPPFLAGS += -I. -I.. -I$(UAPI_DIR) + +NDS_API_VER := $(shell sed -n 's/^[[:space:]]*#define[[:space:]]\+NDS_API_VERSION[[:space:]]\+\([0-9]\+\)u\?.*/\1/p' nds_api.h) +NDS_MAJOR := $(shell echo $$(($(NDS_API_VER) >> 8))) +NDS_MINOR := $(shell echo $$(($(NDS_API_VER) & 255))) +NDS_VERSION := $(NDS_MAJOR).$(NDS_MINOR) +NDS_LIB := libnds.so.$(NDS_VERSION) +NDS_SONAME := libnds.so.$(NDS_VERSION) + +NDS_SRCS := nds_api.c p2p_common.c +NDS_OBJS := $(NDS_SRCS:.c=.o) +NDS_DEPS := $(addprefix .,$(NDS_OBJS:.o=.d)) + +.PHONY: all binding clean + +all: $(NDS_LIB) + +binding: + python3 setup.py build_ext --inplace + +$(NDS_LIB): $(NDS_OBJS) + $(CC) -shared -Wl,-soname,$(NDS_SONAME) -o $@ $(NDS_OBJS) -lpthread + ln -sf libnds.so.$(NDS_VERSION) libnds.so + +%.o: %.c + $(CC) $(CPPFLAGS) $(CFLAGS) -fPIC -MMD -MP -MT $@ -MF .$*.d -c -o $@ $< + +-include $(NDS_DEPS) + +clean: + rm -f libnds.so libnds.so.* $(NDS_OBJS) $(NDS_DEPS) diff --git a/tools/lib/nds/file_p2p/file_p2p_api.c b/tools/lib/nds/file_p2p/file_p2p_api.c new file mode 100644 index 000000000000..e275c0c489e0 --- /dev/null +++ b/tools/lib/nds/file_p2p/file_p2p_api.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/ioctl.h> +#include <errno.h> +#include <limits.h> + +#include <linux/nds_p2p.h> +#include "p2p_common.h" +#include "file_p2p_api.h" + +int close_p2p_fd(int dev_fd) +{ + return close(dev_fd); +} + +int new_p2p_fd(void) +{ + return p2p_open_dev(); +} + +int check_fd(int dev_fd, int fd) +{ + struct p2p_check_fd_param param = { + .fd = fd, + .flags = 0, + .reserved = 0, + }; + + if (ioctl(dev_fd, IOCTL_CHECK_FD, ¶m) < 0) + return -errno; + return 0; +} + +int unregister_mem(int dev_fd, const struct p2p_mem_unregister_param *param) +{ + if (ioctl(dev_fd, IOCTL_UNREGISTER_MEM, param) < 0) + return -errno; + return 0; +} + +int register_mem(int dev_fd, struct p2p_mem_register_param *param) +{ + if (ioctl(dev_fd, IOCTL_REGISTER_MEM, param) < 0) + return -errno; + return 0; +} + +int rw_file(int dev_fd, const struct io_parameter *param) +{ + struct p2p_io_batch_param batch; + struct p2p_io_param *io; + unsigned long io_size; + const char *name; + int file_fd = -1; + int err = 0; + + if (!param) + return -EINVAL; + if (param->op != P2P_IO_READ && param->op != P2P_IO_WRITE) + return -EINVAL; + if (param->flags & ~P2P_IO_F_MASK) + return -EOPNOTSUPP; + if (param->flags & P2P_IO_F_REGISTERED_MEM) { + if (!param->mem_handle || param->host_pid) + return -EINVAL; + } else if (param->host_pid < 0) { + return -EINVAL; + } + + err = p2p_get_iov_size(param->iov, param->iov_nr, &io_size); + if (err) + return err; + if (ULONG_MAX - param->file_offset < io_size) + return -EOVERFLOW; + if (io_size > P2P_MAX_IO_BYTES) + return -EINVAL; + + name = param->file_name; + file_fd = open(name, (param->op == P2P_IO_WRITE ? O_WRONLY : O_RDONLY) | O_CLOEXEC); + if (file_fd < 0) { + err = -errno; + fprintf(stderr, "open %s failed, errno: %d\n", name, err); + return err; + } + + io = calloc(1, sizeof(*io)); + if (io == NULL) { + err = -ENOMEM; + fprintf(stderr, "calloc p2p_io_param failed, errno: %d\n", err); + goto close_fd_out; + } + + io->op = param->op; + io->file_fd = file_fd; + io->flags = param->flags; + if (param->flags & P2P_IO_F_REGISTERED_MEM) { + io->host_pid = 0; + io->mem_handle = param->mem_handle; + } else { + io->host_pid = param->host_pid ? param->host_pid : getpid(); + io->mem_handle = 0; + } + io->offset = param->file_offset; + io->iov = (uint64_t)(uintptr_t)param->iov; + io->iov_nr = param->iov_nr; + + batch.nr = 1; + batch.reserved = 0; + batch.items = (uint64_t)(uintptr_t)io; + err = ioctl(dev_fd, IOCTL_SUBMIT_IO, &batch); + if (err < 0) { + err = -errno; + fprintf(stderr, "%s file ioctl failed, errno: %d\n", + param->op == P2P_IO_WRITE ? "write" : "read", -err); + goto free_io_out; + } + if (err != 1) { + fprintf(stderr, "%s file ioctl accepted %d requests, expected 1\n", + param->op == P2P_IO_WRITE ? "write" : "read", err); + err = -EIO; + goto free_io_out; + } + err = 0; + +free_io_out: + free(io); +close_fd_out: + close(file_fd); + return err; +} diff --git a/tools/lib/nds/file_p2p/file_p2p_api.h b/tools/lib/nds/file_p2p/file_p2p_api.h new file mode 100644 index 000000000000..0526124b3b57 --- /dev/null +++ b/tools/lib/nds/file_p2p/file_p2p_api.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __FILE_P2P_API_H__ +#define __FILE_P2P_API_H__ + +#include <stdint.h> + +#include <linux/nds_p2p.h> + +struct io_parameter { + unsigned int op; + const char *file_name; + unsigned long file_offset; + struct p2p_iov *iov; + unsigned int iov_nr; + unsigned int flags; + int host_pid; + uint64_t mem_handle; +}; + +/* Open /dev/p2p_device. */ +int new_p2p_fd(void); +int close_p2p_fd(int dev_fd); +/* Validate target properties before submitting I/O. */ +int check_fd(int dev_fd, int fd); +/* Release a registration with unregister_mem() on its returned handle. */ +int register_mem(int dev_fd, struct p2p_mem_register_param *param); +int unregister_mem(int dev_fd, const struct p2p_mem_unregister_param *param); +/* + * file_offset is a byte offset; the kernel maps it to underlying blocks. + * Filesystem alignment and CPU-access errors arrive in completion events. + * Success means accepted. Collect results with IOCTL_GET_IO_EVENTS before + * reusing buffers. Final close waits for I/O and discards remaining events. + */ +int rw_file(int dev_fd, const struct io_parameter *param); + +#endif diff --git a/tools/lib/nds/file_p2p/nds_api.c b/tools/lib/nds/file_p2p/nds_api.c new file mode 100644 index 000000000000..2025ac34df72 --- /dev/null +++ b/tools/lib/nds/file_p2p/nds_api.c @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * NDS — a GDS-aligned userspace API for NPU HBM ↔ NVMe P2P I/O. + * + * NDS does not link against file_p2p_api; it shares p2p_common with file_p2p + * for device open only. The two userspace APIs evolve independently on top + * of the same kernel UAPI (linux/nds_p2p.h). + */ +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <string.h> +#include <errno.h> +#include <stddef.h> +#include <pthread.h> +#include <sys/types.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/ioctl.h> +#include <time.h> +#include <limits.h> +#include <stdint.h> + +#include "nds_api_internal.h" +#include <linux/nds_p2p.h> +#include "p2p_common.h" + +_Static_assert(sizeof(struct nds_io_vec) == sizeof(struct p2p_iov), + "nds_io_vec must match p2p_iov for zero-copy submission"); +_Static_assert(NDS_IO_MAX_IO_CNT <= P2P_MAX_IO_NR, + "NDS submit size must fit the kernel batch ioctl"); +_Static_assert(_Alignof(struct nds_io_vec) == _Alignof(struct p2p_iov), + "nds_io_vec alignment must match p2p_iov"); +_Static_assert(offsetof(struct nds_io_vec, buf_addr) == + offsetof(struct p2p_iov, addr) && + offsetof(struct nds_io_vec, buf_len) == + offsetof(struct p2p_iov, size) && + offsetof(struct nds_io_vec, reserved) == + offsetof(struct p2p_iov, reserved), + "nds_io_vec field layout must match p2p_iov"); +_Static_assert(sizeof(struct nds_io_event) == sizeof(struct p2p_io_event), + "nds_io_event must match p2p_io_event for zero-copy getevents"); +_Static_assert(sizeof(struct nds_io_event) == 48, + "nds_io_event / p2p_io_event are user_data + res + reserved[4]"); +_Static_assert(offsetof(struct nds_io_event, user_data) == + offsetof(struct p2p_io_event, user_data) && + offsetof(struct nds_io_event, res) == + offsetof(struct p2p_io_event, res) && + offsetof(struct nds_io_event, reserved) == + offsetof(struct p2p_io_event, reserved), + "nds_io_event field layout must match p2p_io_event"); + +_Static_assert(sizeof(struct nds_mem_range) == sizeof(struct p2p_mem_range) && + offsetof(struct nds_mem_range, addr) == + offsetof(struct p2p_mem_range, addr) && + offsetof(struct nds_mem_range, size) == + offsetof(struct p2p_mem_range, size), + "nds_mem_range must match p2p_mem_range for zero-copy registration"); +_Static_assert(NDS_MEM_MAX_RANGES <= P2P_MEM_MAX_RANGES, + "NDS group size must fit the kernel registration ioctl"); + +/* One registration: a single range or a group, in the caller's order. */ +struct nds_reg_entry { + struct nds_reg_entry *next; + uint64_t handle; + uint32_t nr; + struct nds_mem_range ranges[]; +}; + +/* Submit-side VA index entry; ranges never overlap across registrations. */ +struct nds_mem_ent { + uint64_t start; + uint64_t end; + uint64_t handle; +}; + +struct nds_state { + int book_fd; + /* Serializes register/unregister, including their ioctls. */ + pthread_mutex_t reg_lock; + struct nds_reg_entry *regs; + /* + * Sorted per-range index, used only to re-check overlap on the next + * register/unregister. Write-side bookkeeping: touched solely under + * reg_lock, never read by a submitter. + */ + struct nds_mem_ent *idx; + size_t idx_nr; + /* + * Submit-side lookup index. Runs of consecutive @idx entries (sorted + * by start) that share a handle are merged into one + * [first.start, last.end) entry, so one group with no neighboring + * registration — or any lone registration — is a single lookup entry + * and always hits the per-thread last-hit cache below. Writers hold + * reg_lock and replace the whole array under idx_lock; submitters + * normally avoid the lock entirely via a thread-local snapshot (see + * find_registered_mem). + */ + pthread_rwlock_t idx_lock; + struct nds_mem_ent *lookup_idx; + size_t lookup_nr; + /* + * Bumped (release) on every publish, add or remove. Readers load it + * (acquire) to invalidate both the last-hit cache and thread-local + * snapshots, so a removed range is never returned once the removing + * nds_unregister_mem*() call has returned on the same thread. + */ + uint64_t idx_gen; +}; + +static struct nds_state g_state = { + .book_fd = -1, + .reg_lock = PTHREAD_MUTEX_INITIALIZER, + .idx_lock = PTHREAD_RWLOCK_INITIALIZER, +}; + +/* + * Last index hit of this thread. A hit is reused only while idx_gen is + * unchanged, so it never outlives the unregister of its range. + */ +static __thread struct { + uint64_t gen; + struct nds_mem_ent ent; +} tls_mem_hit; + +/* + * Per-thread copy of the coalesced lookup index, refreshed under idx_lock + * only when stale (gen mismatch) and then binary-searched without any lock. + * idx/cap survive refreshes that do not grow nr, so steady-state misses never + * allocate. Only .idx is heap memory; it is freed at thread exit through + * snap_key below, since __thread storage itself is not otherwise notified of + * thread exit. + */ +struct nds_mem_snapshot { + uint64_t gen; + struct nds_mem_ent *idx; + size_t nr; + size_t cap; +}; + +static __thread struct nds_mem_snapshot tls_snap; +static pthread_key_t snap_key; +static pthread_once_t snap_key_once = PTHREAD_ONCE_INIT; + +/* Destructor for snap_key: frees this thread's snapshot buffer at exit. */ +static void nds_snap_destroy(void *p) +{ + struct nds_mem_snapshot *snap = p; + + free(snap->idx); +} + +static void nds_snap_key_create(void) +{ + pthread_key_create(&snap_key, nds_snap_destroy); +} + +static int is_init(void) +{ + return g_state.book_fd >= 0; +} + +/* ------------------------------------------------------------------ */ +/* Public NDS API */ +/* ------------------------------------------------------------------ */ + +int nds_init(struct nds_init_param *param) +{ + /* Not thread-safe: see nds_api.h. Concurrent init is undefined. */ + if (!param || param->flags || param->reserved1 || param->reserved2 || + is_init() || param->reserved3[0] || param->reserved3[1] || + param->reserved3[2] || param->reserved3[3]) + return -EINVAL; + + g_state.book_fd = p2p_open_dev(); + if (g_state.book_fd < 0) + return g_state.book_fd; + g_state.regs = NULL; + g_state.idx = NULL; + g_state.idx_nr = 0; + g_state.lookup_idx = NULL; + g_state.lookup_nr = 0; + param->version = NDS_API_VERSION; + return 0; +} + +int nds_register_fs(const struct nds_fs_desc *desc) +{ + uint32_t i; + + if (!desc || !desc->fs_fd || !desc->fs_fd_cnt || desc->reserved) + return -EINVAL; + for (i = 0; i < desc->fs_fd_cnt; i++) { + struct p2p_check_fd_param check = { .fd = desc->fs_fd[i] }; + + if (ioctl(g_state.book_fd, IOCTL_CHECK_FD, &check) < 0) + return -errno; + } + return 0; +} + +int nds_unregister_fs(const struct nds_fs_desc *desc) +{ + (void)desc; + return 0; +} + +/* + * Swap in a new lookup index and return the old one for the caller to free. + * Always bumps idx_gen (release), after the swap, whether ranges were added + * or removed: readers only ever consult the lookup index through this gen, + * so an added range must be visible exactly when a removed one disappears. + * @old_nr_out, if non-NULL, receives the old index's length, needed by a + * caller that may have to republish it on a later failure. + */ +static struct nds_mem_ent *nds_publish_index(struct nds_mem_ent *idx, size_t nr, + size_t *old_nr_out) +{ + struct nds_mem_ent *old; + + pthread_rwlock_wrlock(&g_state.idx_lock); + old = g_state.lookup_idx; + if (old_nr_out) + *old_nr_out = g_state.lookup_nr; + g_state.lookup_idx = idx; + g_state.lookup_nr = nr; + __atomic_add_fetch(&g_state.idx_gen, 1, __ATOMIC_RELEASE); + pthread_rwlock_unlock(&g_state.idx_lock); + return old; +} + +int nds_exit(void) +{ + struct nds_reg_entry *r, *next; + + pthread_mutex_lock(&g_state.reg_lock); + r = g_state.regs; + g_state.regs = NULL; + free(g_state.idx); + g_state.idx = NULL; + g_state.idx_nr = 0; + free(nds_publish_index(NULL, 0, NULL)); + pthread_mutex_unlock(&g_state.reg_lock); + + for (; r; r = next) { + struct p2p_mem_unregister_param up = { + .mem_handle = r->handle, + }; + next = r->next; + if (ioctl(g_state.book_fd, IOCTL_UNREGISTER_MEM, &up) < 0) + fprintf(stderr, + "nds: unregister memory addr=0x%llx size=%llu ranges=%u handle=%llu failed %d\n", + (unsigned long long)r->ranges[0].addr, + (unsigned long long)r->ranges[0].size, r->nr, + (unsigned long long)r->handle, -errno); + free(r); + } + + close(g_state.book_fd); + g_state.book_fd = -1; + return 0; +} + +static int nds_mem_ent_cmp(const void *a, const void *b) +{ + const struct nds_mem_ent *ea = a; + const struct nds_mem_ent *eb = b; + + if (ea->start < eb->start) + return -1; + return ea->start > eb->start; +} + +/* + * Build the current per-range index (g_state.idx) plus @ranges with handle 0. + * Ranges must be nonempty and must not overlap each other (-EINVAL) or + * another registration (-EEXIST). Called with reg_lock held, so g_state.idx + * is stable. This index only drives the overlap check above; submitters look + * up the coalesced copy built by nds_coalesce_fill() below. + */ +static int nds_index_add(const struct nds_mem_range *ranges, uint32_t nr, + struct nds_mem_ent **idx_out, size_t *nr_out) +{ + size_t total = g_state.idx_nr + nr; + struct nds_mem_ent *idx; + size_t i; + + idx = malloc(total * sizeof(*idx)); + if (!idx) + return -ENOMEM; + if (g_state.idx_nr) + memcpy(idx, g_state.idx, g_state.idx_nr * sizeof(*idx)); + for (i = 0; i < nr; i++) { + struct nds_mem_ent *ent = &idx[g_state.idx_nr + i]; + + if (!ranges[i].size) { + free(idx); + return -EINVAL; + } + if (ranges[i].addr > UINT64_MAX - ranges[i].size) { + free(idx); + return -EOVERFLOW; + } + ent->start = ranges[i].addr; + ent->end = ranges[i].addr + ranges[i].size; + ent->handle = 0; + } + + qsort(idx, total, sizeof(*idx), nds_mem_ent_cmp); + for (i = 1; i < total; i++) { + int ret; + + if (idx[i].start >= idx[i - 1].end) + continue; + fprintf(stderr, + "nds: register memory range 0x%llx+0x%llx overlaps 0x%llx+0x%llx\n", + (unsigned long long)idx[i].start, + (unsigned long long)(idx[i].end - idx[i].start), + (unsigned long long)idx[i - 1].start, + (unsigned long long)(idx[i - 1].end - idx[i - 1].start)); + /* New ranges still have handle 0. */ + ret = !idx[i].handle && !idx[i - 1].handle ? -EINVAL : -EEXIST; + free(idx); + return ret; + } + + *idx_out = idx; + *nr_out = total; + return 0; +} + +/* Build the current index without @handle. Called with reg_lock held. */ +static int nds_index_del(uint64_t handle, struct nds_mem_ent **idx_out, size_t *nr_out) +{ + struct nds_mem_ent *idx; + size_t nr = 0; + size_t i; + + idx = malloc((g_state.idx_nr ? g_state.idx_nr : 1) * sizeof(*idx)); + if (!idx) + return -ENOMEM; + for (i = 0; i < g_state.idx_nr; i++) { + if (g_state.idx[i].handle != handle) + idx[nr++] = g_state.idx[i]; + } + *idx_out = idx; + *nr_out = nr; + return 0; +} + +/* + * Merge consecutive entries of @idx (sorted by start, @nr of them) that + * share a handle into @out, whose capacity must be at least @nr: coalescing + * only ever shrinks. A gap inside a group has no entry of its own once + * merged, so an iov landing there now resolves to the group's handle instead + * of missing the index; the kernel still rejects it, with -ERANGE rather + * than the library's -EINVAL. Returns the number of entries written. + */ +static size_t nds_coalesce_fill(const struct nds_mem_ent *idx, size_t nr, + struct nds_mem_ent *out) +{ + size_t n = 0; + size_t i; + + for (i = 0; i < nr; i++) { + if (n && out[n - 1].handle == idx[i].handle) + out[n - 1].end = idx[i].end; + else + out[n++] = idx[i]; + } + return n; +} + +static int nds_register_ranges(const struct nds_mem_range *ranges, uint32_t nr) +{ + struct nds_reg_entry *r; + struct nds_mem_ent *idx; + struct nds_mem_ent *lookup; + struct nds_mem_ent *old_idx; + struct p2p_mem_register_param p = { 0 }; + size_t idx_nr; + size_t lookup_nr; + uint64_t handle; + size_t i; + int ret; + + r = malloc(sizeof(*r) + (size_t)nr * sizeof(r->ranges[0])); + if (!r) + return -ENOMEM; + memcpy(r->ranges, ranges, (size_t)nr * sizeof(r->ranges[0])); + r->nr = nr; + + pthread_mutex_lock(&g_state.reg_lock); + ret = nds_index_add(ranges, nr, &idx, &idx_nr); + if (ret) + goto unlock; + + /* + * Preallocate the coalesced buffer before the ioctl: once the kernel + * has accepted the registration, publishing the new indexes must not + * be able to fail. + */ + lookup = malloc((idx_nr ? idx_nr : 1) * sizeof(*lookup)); + if (!lookup) { + free(idx); + ret = -ENOMEM; + goto unlock; + } + + /* Register all ranges through the same ioctl. */ + p.ranges = (uint64_t)(uintptr_t)ranges; + p.nr = nr; + ret = ioctl(g_state.book_fd, IOCTL_REGISTER_MEM, &p); + handle = p.mem_handle; + if (ret < 0) { + ret = -errno; + fprintf(stderr, + "nds: register memory addr=0x%llx size=%llu ranges=%u failed %d\n", + (unsigned long long)ranges[0].addr, + (unsigned long long)ranges[0].size, nr, ret); + free(lookup); + free(idx); + goto unlock; + } + + for (i = 0; i < idx_nr; i++) { + if (!idx[i].handle) + idx[i].handle = handle; + } + lookup_nr = nds_coalesce_fill(idx, idx_nr, lookup); + r->handle = handle; + r->next = g_state.regs; + g_state.regs = r; + r = NULL; + old_idx = g_state.idx; + g_state.idx = idx; + g_state.idx_nr = idx_nr; + free(old_idx); + free(nds_publish_index(lookup, lookup_nr, NULL)); + ret = 0; + +unlock: + pthread_mutex_unlock(&g_state.reg_lock); + free(r); + return ret; +} + +/* + * Unregister the registration whose first range starts at ranges[0].addr. The + * caller must pass the same ranges in the same order as at registration. + */ +static int nds_unregister_ranges(const struct nds_mem_range *ranges, uint32_t nr) +{ + struct nds_reg_entry *r, **prev; + struct p2p_mem_unregister_param up = { 0 }; + struct nds_mem_ent *old_idx; + struct nds_mem_ent *old_lookup; + struct nds_mem_ent *idx; + struct nds_mem_ent *lookup; + size_t old_idx_nr; + size_t old_lookup_nr; + size_t idx_nr; + size_t lookup_nr; + int ret; + + pthread_mutex_lock(&g_state.reg_lock); + for (prev = &g_state.regs; *prev; prev = &(*prev)->next) { + if ((*prev)->ranges[0].addr == ranges[0].addr) + break; + } + if (!*prev) { + ret = -ENOENT; + goto unlock; + } + r = *prev; + if (r->nr != nr || memcmp(r->ranges, ranges, (size_t)nr * sizeof(ranges[0]))) { + fprintf(stderr, + "nds: unregister memory addr=0x%llx size=%llu ranges=%u does not match registered size=%llu ranges=%u\n", + (unsigned long long)ranges[0].addr, + (unsigned long long)ranges[0].size, nr, + (unsigned long long)r->ranges[0].size, r->nr); + ret = -EINVAL; + goto unlock; + } + + ret = nds_index_del(r->handle, &idx, &idx_nr); + if (ret) + goto unlock; + lookup = malloc((idx_nr ? idx_nr : 1) * sizeof(*lookup)); + if (!lookup) { + free(idx); + ret = -ENOMEM; + goto unlock; + } + lookup_nr = nds_coalesce_fill(idx, idx_nr, lookup); + + /* Hide the ranges before the kernel drops the handle. */ + old_idx = g_state.idx; + old_idx_nr = g_state.idx_nr; + g_state.idx = idx; + g_state.idx_nr = idx_nr; + old_lookup = nds_publish_index(lookup, lookup_nr, &old_lookup_nr); + + up.mem_handle = r->handle; + if (ioctl(g_state.book_fd, IOCTL_UNREGISTER_MEM, &up) < 0) { + ret = -errno; + fprintf(stderr, + "nds: unregister memory addr=0x%llx size=%llu ranges=%u handle=%llu failed %d\n", + (unsigned long long)r->ranges[0].addr, + (unsigned long long)r->ranges[0].size, r->nr, + (unsigned long long)r->handle, ret); + /* Keep the registration tracked so the caller can retry. */ + g_state.idx = old_idx; + g_state.idx_nr = old_idx_nr; + free(idx); + free(nds_publish_index(old_lookup, old_lookup_nr, NULL)); + goto unlock; + } + + *prev = r->next; + free(r); + free(old_idx); + free(old_lookup); + +unlock: + pthread_mutex_unlock(&g_state.reg_lock); + return ret; +} + +static int nds_mem_desc_valid(const struct nds_mem_desc *desc) +{ + return desc && desc->ranges && desc->nr && + desc->nr <= NDS_MEM_MAX_RANGES && !desc->flags && + !desc->reserved[0] && !desc->reserved[1] && + !desc->reserved[2] && !desc->reserved[3]; +} + +int nds_register_mem(const struct nds_mem_desc *desc) +{ + if (!nds_mem_desc_valid(desc)) + return -EINVAL; + return nds_register_ranges(desc->ranges, desc->nr); +} + +int nds_unregister_mem(const struct nds_mem_desc *desc) +{ + if (!nds_mem_desc_valid(desc)) + return -EINVAL; + return nds_unregister_ranges(desc->ranges, desc->nr); +} + +int nds_io_new_ctx(const struct nds_io_ctx_param *param, struct nds_io_ctx **ctx) +{ + struct nds_io_ctx *c; + int fd; + + if (!param || !ctx) + return -EINVAL; + if (!param->max_io_cnt || param->max_io_cnt > NDS_IO_MAX_IO_CNT || + param->flags || + param->reserved) + return -EINVAL; + + c = calloc(1, sizeof(*c)); + if (!c) + return -ENOMEM; + + fd = p2p_open_dev(); + if (fd < 0) { + free(c); + return fd; + } + c->p2p_fd = fd; + *ctx = c; + return 0; +} + +int nds_io_destroy_ctx(struct nds_io_ctx *ctx) +{ + int close_err = 0; + + if (!ctx) + return -EINVAL; + /* + * Catches a closed-but-not-yet-freed ctx. A second destroy after + * free() is still use-after-free (caller bug / UB). + */ + if (ctx->p2p_fd < 0) + return -EINVAL; + + /* The final close waits for outstanding I/O and releases its events. */ + if (close(ctx->p2p_fd) < 0) { + close_err = -errno; + fprintf(stderr, "nds: destroy_ctx close failed %d\n", close_err); + } + ctx->p2p_fd = -1; + free(ctx); + return close_err; +} + +/* Binary search a sorted nds_mem_ent array for the entry covering @addr. */ +static int nds_mem_ent_lookup(const struct nds_mem_ent *idx, size_t nr, uint64_t addr, + struct nds_mem_ent *ent_out) +{ + size_t lo = 0, hi = nr; + + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + + if (idx[mid].start <= addr) + lo = mid + 1; + else + hi = mid; + } + if (lo && addr < idx[lo - 1].end) { + *ent_out = idx[lo - 1]; + return 0; + } + return -EINVAL; +} + +/* + * Map the start of a registered request to its handle. Only the first IOV + * selects the registration; the kernel checks every IOV against it. + * + * Fast path: the per-thread last-hit cache. Slow path: refresh this thread's + * snapshot of the coalesced lookup index (under idx_lock) only when it is + * stale, then binary-search the snapshot with no lock at all. Because a + * group coalesces to one entry spanning its whole VA range, an address in a + * gap between a group's ranges now resolves to the group's handle instead of + * missing here; the kernel still rejects the actual iov with -ERANGE. + */ +static int find_registered_mem(uint64_t addr, uint64_t *handle_out) +{ + /* Read gen first so removal invalidates the cached index. */ + uint64_t gen = __atomic_load_n(&g_state.idx_gen, __ATOMIC_ACQUIRE); + struct nds_mem_snapshot *snap = &tls_snap; + struct nds_mem_ent ent; + int ret; + + if (tls_mem_hit.gen == gen && tls_mem_hit.ent.handle && + addr >= tls_mem_hit.ent.start && addr < tls_mem_hit.ent.end) { + *handle_out = tls_mem_hit.ent.handle; + return 0; + } + + if (snap->gen != gen) { + pthread_rwlock_rdlock(&g_state.idx_lock); + if (g_state.lookup_nr > snap->cap) { + struct nds_mem_ent *buf = malloc(g_state.lookup_nr * sizeof(*buf)); + bool first = !snap->cap; + + if (!buf) { + pthread_rwlock_unlock(&g_state.idx_lock); + return -ENOMEM; + } + free(snap->idx); + snap->idx = buf; + snap->cap = g_state.lookup_nr; + /* Register cleanup for this thread's buffer. */ + if (first) { + pthread_once(&snap_key_once, nds_snap_key_create); + pthread_setspecific(snap_key, snap); + } + } + if (g_state.lookup_nr) + memcpy(snap->idx, g_state.lookup_idx, + g_state.lookup_nr * sizeof(*snap->idx)); + snap->nr = g_state.lookup_nr; + /* + * Record the gen read inside the lock: it is what the copy + * above actually reflects, not the possibly staler gen loaded + * above. + */ + snap->gen = __atomic_load_n(&g_state.idx_gen, __ATOMIC_ACQUIRE); + pthread_rwlock_unlock(&g_state.idx_lock); + } + + ret = nds_mem_ent_lookup(snap->idx, snap->nr, addr, &ent); + if (ret) + return ret; + tls_mem_hit.gen = snap->gen; + tls_mem_hit.ent = ent; + *handle_out = ent.handle; + return 0; +} + +/* + * nds_io_vec is binary-identical to p2p_iov (see the _Static_assert block + * above), so the shared alignment/size validation of p2p_get_iov_size() + * applies directly; this also sums the iov into *io_size. + */ +static int validate_nds_iov(const struct nds_io_cb *cb, unsigned long *io_size) +{ + if (!cb->iov || !cb->iov_cnt || cb->iov_cnt > NDS_IO_MAX_IOV) + return -EINVAL; + return p2p_get_iov_size((const struct p2p_iov *)cb->iov, cb->iov_cnt, io_size); +} + +static int nds_iocb_to_io_param(const struct nds_io_cb *cb, int file_fd, struct p2p_io_param *io) +{ + unsigned long io_size; + uint64_t mem_handle = 0; + int err; + + if (cb->opcode > NDS_IO_OP_PWRITE || (cb->rw_flags & ~NDS_IO_F_MASK)) + return -EOPNOTSUPP; + + err = validate_nds_iov(cb, &io_size); + if (err) + return err; + if (io_size > P2P_MAX_IO_BYTES) + return -EINVAL; + if (cb->offset > ULONG_MAX || io_size > ULONG_MAX - (unsigned long)cb->offset) + return -EOVERFLOW; + + if (cb->rw_flags & NDS_IO_F_REGISTERED_MEM) { + if (cb->host_pid) + return -EINVAL; + err = find_registered_mem(cb->iov[0].buf_addr, &mem_handle); + if (err) + return err; + } else if (cb->host_pid < 0) { + return -EINVAL; + } + + memset(io, 0, sizeof(*io)); + io->op = (cb->opcode == NDS_IO_OP_PWRITE) ? P2P_IO_WRITE : P2P_IO_READ; + io->file_fd = file_fd; + io->user_data = cb->user_data; + io->offset = cb->offset; + io->iov = (uint64_t)(uintptr_t)cb->iov; + io->iov_nr = cb->iov_cnt; + + if (cb->rw_flags & NDS_IO_F_REGISTERED_MEM) { + io->flags = P2P_IO_F_REGISTERED_MEM; + io->host_pid = 0; + io->mem_handle = mem_handle; + } else { + io->flags = 0; + io->host_pid = cb->host_pid ? cb->host_pid : getpid(); + io->mem_handle = 0; + } + + return 0; +} + +static int nds_build_one_iocb(const struct nds_io_cb *cb, struct p2p_io_param *param) +{ + int file_fd = cb->obj.fd; + int ret; + + if (cb->reserved[0] || cb->reserved[1] || cb->obj.reserved || + file_fd < 0) + return -EINVAL; + + ret = nds_iocb_to_io_param(cb, file_fd, param); + if (ret) + fprintf(stderr, "nds: translate iocb failed %d\n", ret); + return ret; +} + +int nds_io_submit(struct nds_io_ctx *ctx, int nr, + const struct nds_io_cb *iocb) +{ + struct p2p_io_batch_param batch = { 0 }; + struct p2p_io_param *items; + unsigned int built = 0; + int build_err = 0; + int ret; + + if (!ctx || !iocb) + return -EINVAL; + if (nr < 0) + return -EINVAL; + if (!nr) + return 0; + if ((unsigned int)nr > NDS_IO_MAX_IO_CNT) + return -EINVAL; + + items = calloc((size_t)nr, sizeof(*items)); + if (!items) + return -ENOMEM; + + /* + * Build the valid prefix in iocb order, then submit its independent + * requests through one ioctl. Translation and kernel acceptance are + * both fail-stop: return the accepted prefix, or -errno if item zero + * fails. + */ + while (built < (unsigned int)nr) { + build_err = nds_build_one_iocb(&iocb[built], &items[built]); + if (build_err) + break; + built++; + } + if (!built) { + ret = build_err; + goto out; + } + + batch.nr = built; + batch.items = (uint64_t)(uintptr_t)items; + ret = ioctl(ctx->p2p_fd, IOCTL_SUBMIT_IO, &batch); + if (ret < 0) { + ret = -errno; + fprintf(stderr, "nds: submit ioctl failed %d\n", ret); + } else if (!ret) { + ret = -EIO; + } + +out: + free(items); + return ret; +} + +int nds_io_getevents(struct nds_io_ctx *ctx, int min_nr, + int nr, struct nds_io_event *events, + struct timespec *timeout) +{ + struct p2p_getevents_param param = { 0 }; + int ret; + + if (!ctx) + return -EINVAL; + if (min_nr < 0 || nr < 0 || min_nr > nr) + return -EINVAL; + if (!nr) + return 0; + if (!events) + return -EINVAL; + + /* + * nds_io_event is binary-identical to p2p_io_event (48B); pass the + * caller's buffer straight to the ioctl — no intermediate copy. + */ + param.min_nr = min_nr; + param.max_nr = nr; + if (timeout) { + if (timeout->tv_sec < 0 || timeout->tv_nsec < 0 || + timeout->tv_nsec >= 1000000000L || + (uint64_t)timeout->tv_sec > + (uint64_t)(INT64_MAX - timeout->tv_nsec) / + UINT64_C(1000000000)) + return -EINVAL; + param.timeout_ns = (__s64)timeout->tv_sec * 1000000000LL + + timeout->tv_nsec; + } else { + param.timeout_ns = -1; + } + param.events = (uint64_t)(uintptr_t)events; + + ret = ioctl(ctx->p2p_fd, IOCTL_GET_IO_EVENTS, ¶m); + if (ret < 0) { + ret = -errno; + fprintf(stderr, "nds: getevents ioctl failed %d\n", ret); + return ret; + } + return ret; +} diff --git a/tools/lib/nds/file_p2p/nds_api.h b/tools/lib/nds/file_p2p/nds_api.h new file mode 100644 index 000000000000..154ebc0b60b9 --- /dev/null +++ b/tools/lib/nds/file_p2p/nds_api.h @@ -0,0 +1,249 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NDS_API_H__ +#define __NDS_API_H__ + +#include <stdint.h> +#include <sys/types.h> +#include <time.h> /* struct timespec for nds_io_getevents */ + +/* + * NDS — userspace API for NPU HBM ↔ NVMe P2P I/O (GDS-like, async). + * + * Typical usage: + * + * 1. Initialize NDS, then register each filesystem as it becomes available: + * struct nds_init_param ip = { 0 }; + * int fds[] = { file_fd }; + * struct nds_fs_desc fs = { + * .fs_fd = fds, + * .fs_fd_cnt = 1, + * }; + * nds_init(&ip); + * assert(ip.version == NDS_API_VERSION); + * nds_register_fs(&fs); + * Call nds_register_fs() again to add filesystems after initialization. + * Each device fd learns its NPU from the first buffer used with that fd. + * + * 2. (Optional) Register HBM / CMB windows once, as a group (a single + * buffer is a group with nr == 1): + * struct nds_mem_desc mem = { .ranges = ranges, .nr = nr }; + * nds_register_mem(&mem); + * Later I/Os that use those VAs set NDS_IO_F_REGISTERED_MEM; the library + * looks up the handle — callers never see mem_handle. + * + * 3. Create an I/O context and submit work (Linux-AIO style): + * struct nds_io_ctx *ctx; + * nds_io_new_ctx(&(struct nds_io_ctx_param){ .max_io_cnt = N }, &ctx); + * struct nds_io_vec iov = { + * .buf_addr = ..., + * .buf_len = ..., + * }; + * + * struct nds_io_cb cb = { + * .opcode = NDS_IO_OP_PREAD, + * .obj = { .fd = file_fd }, + * .offset = ..., + * .iov = &iov, + * .iov_cnt = 1, + * .rw_flags = 0, // or NDS_IO_F_REGISTERED_MEM + * .host_pid = 0, // must be 0 with REGISTERED_MEM + * .user_data = cookie, // echoed in completion events + * }; + * nds_io_submit(ctx, 1, &cb); + * nds_io_getevents(ctx, 1, 1, events, NULL); + * + * 4. Tear down: destroy ctx → unregister mem → nds_exit(). + * + * Conventions: + * - All functions return 0 / positive count on success, or -errno on failure. + * - I/O is async only (no synchronous pread/pwrite helpers). + * - Targets are open file or block-device FDs. Paths are not accepted. + * - Offsets / lengths / buffer addresses must be sector-aligned (512 B); + * a regular-file write also needs filesystem-block alignment, and one + * logical I/O is limited to P2P_MAX_IO_BYTES. + * - reserved fields must be zero. + */ + +#define NDS_API_VERSION 2u +#define NDS_IO_MAX_IO_CNT 1024u +#define NDS_IO_MAX_IOV 65536u + +#define NDS_MEM_MAX_RANGES 8192u + +struct nds_io_ctx; + +struct nds_init_param { + uint32_t flags; /* must be 0 */ + uint32_t reserved1; /* must be 0 */ + uint32_t version; /* output: NDS_API_VERSION on success */ + uint32_t reserved2; /* must be 0 */ + uint64_t reserved3[4]; /* must be zero when unused */ +}; + +/* File, directory or block-device fds for filesystem and queue validation. */ +struct nds_fs_desc { + int32_t *fs_fd; + uint32_t fs_fd_cnt; + uint32_t reserved; +}; + +struct nds_mem_range { + uint64_t addr; + uint64_t size; +}; + +struct nds_mem_desc { + const struct nds_mem_range *ranges; + uint32_t nr; + uint32_t flags; /* must be 0 */ + uint64_t reserved[4]; /* must be zero */ +}; + +/* + * max_io_cnt: accepts 1..NDS_IO_MAX_IO_CNT as a queue-size hint but does + * not enforce an outstanding-I/O budget in the library. flags and reserved + * must be zero. + */ +struct nds_io_ctx_param { + uint32_t max_io_cnt; + uint32_t flags; + uint64_t reserved; +}; + +/* I/O target: an open file or block-device fd. */ +struct nds_io_obj { + uint64_t reserved; /* must be zero */ + union { + uint64_t _data; + int32_t fd; /* open file / block-device fd */ + }; +}; + +#define NDS_IO_OP_PREAD 0u /* read one or more buffers */ +#define NDS_IO_OP_PWRITE 1u /* write one or more buffers */ + +/* Buffer was previously registered with nds_register_mem(). */ +#define NDS_IO_F_REGISTERED_MEM (1u << 0) +#define NDS_IO_F_MASK NDS_IO_F_REGISTERED_MEM + +struct nds_io_vec { + uint64_t buf_addr; + uint32_t buf_len; + uint32_t reserved; /* must be zero; layout matches p2p_iov */ +}; + +/* + * One async I/O request. Fill an array of these and pass to nds_io_submit(). + * + * iov / iov_cnt describe the HBM (or CMB) VA windows for this transfer. + * With NDS_IO_F_REGISTERED_MEM, iov[0] selects a region previously passed to + * nds_register_mem(); the kernel requires every vector to fit that + * region or group. Otherwise buffers are one-shot. + * + * offset is the byte offset in obj.fd; the kernel maps it to the underlying + * blocks (no FIEMAP / fallocate step in userspace). Filesystem alignment, + * CPU-access fallbacks and short transfers report errors in completion events. + * Callers must exclusively manage files and keep mappings stable through I/O. + */ +struct nds_io_cb { + uint32_t opcode; /* NDS_IO_OP_* */ + uint32_t rw_flags; /* NDS_IO_F_* bitmask */ + struct nds_io_obj obj; + uint64_t offset; /* byte offset in the target object */ + uint64_t user_data; /* opaque cookie, returned in events */ + const struct nds_io_vec *iov; + uint32_t iov_cnt; + /* + * Owner of one-shot HBM VAs. 0 = current process. + * Must be 0 when NDS_IO_F_REGISTERED_MEM is set. + */ + int32_t host_pid; + uint64_t reserved[2]; /* must be zero */ +}; + +/* + * One completion. Layout matches kernel struct p2p_io_event (48B). + * res is success(==0) or -errno; reserved must be zero. + */ +struct nds_io_event { + uint64_t user_data; + int64_t res; + uint64_t reserved[4]; +}; + +/* + * Process-wide initialization and teardown. + * nds_init opens /dev/p2p_device and uses a separate fd for filesystem and + * memory-registration bookkeeping. It is not thread-safe or re-entrant; call + * it once from one thread. Do not call nds_exit concurrently with + * nds_register_fs() or any other NDS operation. + */ +int nds_init(struct nds_init_param *param); +int nds_exit(void); + +int nds_register_fs(const struct nds_fs_desc *desc); +int nds_unregister_fs(const struct nds_fs_desc *desc); + +/* + * Pin / unpin several separately allocated VA ranges (for example, KV-cache + * blocks) as one registration group. One NDS_IO_F_REGISTERED_MEM iocb may + * then scatter across any ranges of the group: iov[0] selects the group, and + * every vector must fit in one kernel translation segment (-ERANGE otherwise). + * Adjacent ranges merge into one segment only at a page-aligned boundary with + * the same devmm page size. Use separate vectors for different segments. + * + * desc->nr must be 1..NDS_MEM_MAX_RANGES. flags and reserved must be zero. + * Ranges need not be sorted but must be nonempty and must not overlap each + * other (-EINVAL) or any other registration (-EEXIST). A single buffer is a + * group with nr == 1; there is no separate single-range call. + * Unregister with the same ranges in the same order; a mismatch returns + * -EINVAL. There is no partial unregister. + * + * The library's lookup resolves iov[0] to the group as a whole, including any + * gap between its ranges that no other registration covers: an iov[0] placed + * in such a gap still selects the group, and the kernel then rejects the + * request with -ERANGE rather than the library returning -EINVAL. + */ +int nds_register_mem(const struct nds_mem_desc *desc); +int nds_unregister_mem(const struct nds_mem_desc *desc); + +/* + * Create / destroy a per-queue I/O context. Destroy closes the fd, waits for + * outstanding I/O and frees @ctx. The caller owns the pointer until destroy; + * double destroy (or any use after destroy) is undefined behavior. Do not call + * nds_io_getevents concurrently with destroy on the same ctx: the caller must + * stop all other uses of the context before destroy. + */ +int nds_io_new_ctx(const struct nds_io_ctx_param *param, + struct nds_io_ctx **ctx); +int nds_io_destroy_ctx(struct nds_io_ctx *ctx); + +/* + * Submit nr control blocks. Returns the number accepted, or -errno. + * + * Each iocb is one logical I/O: its iov[] is scatter-gather on a single + * obj.fd / offset and completes as one event (user_data). Different iocbs + * may target different files; the library submits the valid prefix as + * independent IOCTL_SUBMIT_IO items without cross-iocb merging. + * + * Fail-stop like Linux io_submit: on a mid-batch failure return the count + * of iocbs already accepted (positive); the failed iocb's errno is not + * returned. If nothing was accepted, return -errno. Accepted I/Os are not + * rolled back — reap them with nds_io_getevents(). + */ +int nds_io_submit(struct nds_io_ctx *ctx, int nr, + const struct nds_io_cb *iocb); + +/* + * Wait for at least min_nr completions, up to nr. + * timeout NULL = wait forever; {0,0} = non-blocking. + * On success returns the number of events filled (>= 0); on failure -errno. + * When no I/Os are outstanding, returns 0 immediately even if min_nr > 0 + * (does not block until timeout like Linux AIO on an empty ctx). + * Must not run concurrently with nds_io_destroy_ctx on the same ctx. + */ +int nds_io_getevents(struct nds_io_ctx *ctx, int min_nr, + int nr, struct nds_io_event *events, + struct timespec *timeout); + +#endif diff --git a/tools/lib/nds/file_p2p/nds_api_internal.h b/tools/lib/nds/file_p2p/nds_api_internal.h new file mode 100644 index 000000000000..a217c07d0541 --- /dev/null +++ b/tools/lib/nds/file_p2p/nds_api_internal.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef NDS_API_INTERNAL_H_ +#define NDS_API_INTERNAL_H_ + +#include "nds_api.h" + +/* Private layout shared by the library internals and white-box tests. */ +struct nds_io_ctx { + int p2p_fd; +}; + +#endif diff --git a/tools/lib/nds/file_p2p/p2p_common.c b/tools/lib/nds/file_p2p/p2p_common.c new file mode 100644 index 000000000000..39822040ca86 --- /dev/null +++ b/tools/lib/nds/file_p2p/p2p_common.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <fcntl.h> +#include <unistd.h> +#include <limits.h> +#include <stdint.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/ioctl.h> + +#include <linux/nds_p2p.h> +#include "p2p_common.h" + +#define P2P_SECTOR_SIZE 512U +#define P2P_DEV_PATH "/dev/p2p_device" + +int p2p_open_dev(void) +{ + struct p2p_device_info info = { 0 }; + int dev_fd; + int err; + + dev_fd = open(P2P_DEV_PATH, O_RDWR | O_CLOEXEC); + if (dev_fd < 0) { + err = errno == ENOENT ? -ENODEV : -errno; + fprintf(stderr, "open %s failed, errno: %d\n", P2P_DEV_PATH, err); + return err; + } + + if (ioctl(dev_fd, IOCTL_GET_INFO, &info) < 0) { + err = errno == ENOTTY ? -EPROTONOSUPPORT : -errno; + goto close_fd; + } + if (((info.version >> 8) & 0xffU) != P2P_UAPI_VERSION_MAJOR) { + err = -EPROTONOSUPPORT; + goto close_fd; + } + return dev_fd; + +close_fd: + close(dev_fd); + return err; +} + +int p2p_get_iov_size(const struct p2p_iov *iov, unsigned int iov_nr, unsigned long *size_out) +{ + unsigned long size = 0; + unsigned int i; + + if (!iov || !iov_nr) + return -EINVAL; + + for (i = 0; i < iov_nr; i++) { + if (!iov[i].size || iov[i].reserved || + ((iov[i].addr | iov[i].size) & (P2P_SECTOR_SIZE - 1)) || + UINT64_MAX - iov[i].addr < iov[i].size || ULONG_MAX - size < iov[i].size) + return -EINVAL; + size += iov[i].size; + } + + *size_out = size; + return 0; +} diff --git a/tools/lib/nds/file_p2p/p2p_common.h b/tools/lib/nds/file_p2p/p2p_common.h new file mode 100644 index 000000000000..7ecaed36730a --- /dev/null +++ b/tools/lib/nds/file_p2p/p2p_common.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __P2P_COMMON_H__ +#define __P2P_COMMON_H__ + +#include <linux/nds_p2p.h> + +/* + * Open /dev/p2p_device O_RDWR|O_CLOEXEC. Returns fd or -errno (-ENODEV if + * absent, -EPROTONOSUPPORT for an incompatible UAPI). + */ +int p2p_open_dev(void); + +/* Sum iov[].size with sector-alignment checks. Returns 0 or -errno. */ +int p2p_get_iov_size(const struct p2p_iov *iov, unsigned int iov_nr, unsigned long *size_out); + +#endif diff --git a/tools/lib/nds/file_p2p/py_file_p2p_api.c b/tools/lib/nds/file_p2p/py_file_p2p_api.c new file mode 100644 index 000000000000..d0f989b05cea --- /dev/null +++ b/tools/lib/nds/file_p2p/py_file_p2p_api.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <limits.h> +#include <pthread.h> +#include <stdio.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "file_p2p_api.h" + +static int parse_iovs(PyObject *object, struct p2p_iov **iov_out, + unsigned int *iov_nr_out) +{ + struct p2p_iov *iov; + PyObject *sequence; + Py_ssize_t count; + Py_ssize_t i; + + sequence = PySequence_Fast(object, "IOVs must be a sequence"); + if (!sequence) + return -1; + count = PySequence_Fast_GET_SIZE(sequence); + if (count <= 0 || count > UINT_MAX || + (size_t)count > SIZE_MAX / sizeof(*iov)) { + PyErr_SetString(PyExc_ValueError, + "IOVs must be a non-empty bounded sequence"); + Py_DECREF(sequence); + return -1; + } + + iov = calloc(count, sizeof(*iov)); + if (!iov) { + PyErr_NoMemory(); + Py_DECREF(sequence); + return -1; + } + + for (i = 0; i < count; i++) { + PyObject *entry; + PyObject **items; + unsigned long long addr; + unsigned long long size; + + entry = PySequence_Fast(PySequence_Fast_GET_ITEM(sequence, i), + "IOV must be (addr, size)"); + if (!entry) + goto error; + if (PySequence_Fast_GET_SIZE(entry) != 2) { + PyErr_SetString(PyExc_ValueError, + "IOV must be (addr, size)"); + Py_DECREF(entry); + goto error; + } + items = PySequence_Fast_ITEMS(entry); + addr = PyLong_AsUnsignedLongLong(items[0]); + size = PyLong_AsUnsignedLongLong(items[1]); + Py_DECREF(entry); + if (PyErr_Occurred()) + goto error; + if (size > UINT32_MAX) { + PyErr_SetString(PyExc_ValueError, + "IOV size exceeds the 32-bit UAPI limit"); + goto error; + } + iov[i].addr = addr; + iov[i].size = (uint32_t)size; + iov[i].reserved = 0; + } + + Py_DECREF(sequence); + *iov_out = iov; + *iov_nr_out = count; + return 0; + +error: + free(iov); + Py_DECREF(sequence); + return -1; +} + +static PyObject *py_rw_file(PyObject *Py_UNUSED(self), PyObject *args) +{ + int dev_fd = 0; + unsigned int op = 0; + const char *file_name = NULL; + unsigned long file_offset = 0; + struct p2p_iov *iov = NULL; + unsigned int iov_nr = 0; + PyObject *py_iovs = NULL; + long long mem_handle = 0; + unsigned int flags = 0; + int host_pid = 0; + int ret = 0; + + if (!PyArg_ParseTuple(args, "iIskOLI|i", &dev_fd, &op, &file_name, &file_offset, + &py_iovs, &mem_handle, &flags, &host_pid)) + return NULL; + if (parse_iovs(py_iovs, &iov, &iov_nr)) + return NULL; + + struct io_parameter param = { + .op = op, + .host_pid = host_pid, + .file_name = file_name, + .file_offset = file_offset, + .iov = iov, + .iov_nr = iov_nr, + .flags = flags, + .mem_handle = (uint64_t)mem_handle, + }; + + Py_BEGIN_ALLOW_THREADS + ret = rw_file(dev_fd, ¶m); + Py_END_ALLOW_THREADS + + free(iov); + return PyLong_FromLong((long)ret); +} + +static PyObject *py_check_fd(PyObject *Py_UNUSED(self), PyObject *args) +{ + int dev_fd; + int fd; + int ret; + + if (!PyArg_ParseTuple(args, "ii", &dev_fd, &fd)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = check_fd(dev_fd, fd); + Py_END_ALLOW_THREADS + return PyLong_FromLong((long)ret); +} + +static int parse_mem_ranges(PyObject *object, struct p2p_mem_range **ranges_out, + uint32_t *nr_out) +{ + struct p2p_mem_range *ranges; + PyObject *sequence; + Py_ssize_t count; + Py_ssize_t i; + + sequence = PySequence_Fast(object, "ranges must be a sequence of (addr, size)"); + if (!sequence) + return -1; + count = PySequence_Fast_GET_SIZE(sequence); + if (count <= 0 || count > P2P_MEM_MAX_RANGES) { + PyErr_SetString(PyExc_ValueError, "range count out of range"); + Py_DECREF(sequence); + return -1; + } + + ranges = calloc((size_t)count, sizeof(*ranges)); + if (!ranges) { + PyErr_NoMemory(); + Py_DECREF(sequence); + return -1; + } + + for (i = 0; i < count; i++) { + unsigned long long addr; + unsigned long long size; + + if (!PyArg_ParseTuple(PySequence_Fast_GET_ITEM(sequence, i), "KK", + &addr, &size)) { + free(ranges); + Py_DECREF(sequence); + return -1; + } + ranges[i].addr = addr; + ranges[i].size = size; + } + + Py_DECREF(sequence); + *ranges_out = ranges; + *nr_out = (uint32_t)count; + return 0; +} + +static PyObject *py_register_mem(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct p2p_mem_register_param param = { }; + struct p2p_mem_range *ranges; + PyObject *ranges_py; + int dev_fd; + int ret; + + if (!PyArg_ParseTuple(args, "iO", &dev_fd, &ranges_py)) + return NULL; + if (parse_mem_ranges(ranges_py, &ranges, ¶m.nr)) + return NULL; + param.ranges = (uint64_t)(uintptr_t)ranges; + + Py_BEGIN_ALLOW_THREADS + ret = register_mem(dev_fd, ¶m); + Py_END_ALLOW_THREADS + free(ranges); + + if (ret) + Py_RETURN_NONE; + return PyLong_FromLongLong((long long)param.mem_handle); +} + +static PyObject *py_unregister_mem(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct p2p_mem_unregister_param param = { }; + long long mem_handle; + int dev_fd; + int ret; + + if (!PyArg_ParseTuple(args, "iL", &dev_fd, &mem_handle)) + return NULL; + param.mem_handle = (uint64_t)mem_handle; + + Py_BEGIN_ALLOW_THREADS + ret = unregister_mem(dev_fd, ¶m); + Py_END_ALLOW_THREADS + + return PyLong_FromLong((long)ret); +} + +static PyObject *py_new_p2p_fd(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(ignored)) +{ + int ret = new_p2p_fd(); + + return PyLong_FromLong((long)ret); +} + +static PyObject *py_close_p2p_fd(PyObject *Py_UNUSED(self), PyObject *args) +{ + int dev_fd = 0; + + if (!PyArg_ParseTuple(args, "i", &dev_fd)) { + return NULL; + } + + close_p2p_fd(dev_fd); + + Py_RETURN_NONE; +} + +static PyMethodDef FileP2PMethods[] = { + { "rw_file", py_rw_file, METH_VARARGS, + "rw_file(dev_fd, op, file_name, file_offset, iovs, mem_handle, flags" + "[, host_pid]) -> int\n\n" + "Read from or write to a p2p I/O target.\n" + "\n" + "Parameters:\n" + " dev_fd (int): File descriptor of p2p device.\n" + " op (int): P2P_IO_READ or P2P_IO_WRITE.\n" + " file_name (str): Name of the I/O target.\n" + " file_offset (int): Starting file offset.\n" + " iovs (sequence): Device-memory (address, size) pairs.\n" + " mem_handle (int): Registered-memory handle, or zero.\n" + " flags (int): P2P I/O flags.\n" + " host_pid (int, optional): PID owning unregistered IOV addresses;\n" + " zero or omitted selects the current process. Must be zero for\n" + " registered memory.\n" + "\n" + "Returns:\n" + " int: 0 on success, non-zero on error.\n" }, + { "register_mem", py_register_mem, METH_VARARGS, + "register_mem(dev_fd, ranges) -> int | None\n\n" + "ranges: sequence of (addr, size) registered under one handle.\n" + "Return the memory handle; the handle may be negative.\n" + "Returns None on failure.\n" }, + { "unregister_mem", py_unregister_mem, METH_VARARGS, + "unregister_mem(dev_fd, mem_handle) -> int\n\n" + "Synchronously unregister a device-memory handle.\n" }, + { "new_p2p_fd", py_new_p2p_fd, METH_NOARGS, + "new_p2p_fd() -> int\n\n" + "New file descriptor on /dev/p2p_device.\n" + "\n" + "Returns:\n" + " int: File descriptor on success, negative errno on error.\n" }, + { "check_fd", py_check_fd, METH_VARARGS, + "check_fd(dev_fd, fd) -> int\n\n" + "Validate target properties before I/O. Returns 0 or a negative errno.\n" }, + { "close_p2p_fd", py_close_p2p_fd, METH_VARARGS, + "close_p2p_fd(dev_fd) -> None\n\n" + "Close p2p device file descriptor.\n" + "\n" + "Parameters:\n" + " dev_fd (int): File descriptor of p2p device.\n" + "\n" + "Returns:\n" + " None\n" }, + { NULL, NULL, 0, NULL } +}; + +static struct PyModuleDef file_p2p_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "file_p2p", + .m_doc = "p2p file access", + .m_size = -1, + .m_methods = FileP2PMethods, +}; + +PyMODINIT_FUNC PyInit_file_p2p(void) +{ + PyObject *module; + + module = PyModule_Create(&file_p2p_module); + if (!module) + return NULL; + if (PyModule_AddIntConstant(module, "P2P_IO_F_REGISTERED_MEM", + P2P_IO_F_REGISTERED_MEM)) { + Py_DECREF(module); + return NULL; + } + if (PyModule_AddIntConstant(module, "P2P_IO_READ", P2P_IO_READ) || + PyModule_AddIntConstant(module, "P2P_IO_WRITE", P2P_IO_WRITE)) { + Py_DECREF(module); + return NULL; + } + return module; +} diff --git a/tools/lib/nds/file_p2p/py_nds_api.c b/tools/lib/nds/file_p2p/py_nds_api.c new file mode 100644 index 000000000000..7d8000579db2 --- /dev/null +++ b/tools/lib/nds/file_p2p/py_nds_api.c @@ -0,0 +1,577 @@ +// SPDX-License-Identifier: GPL-2.0 +#define PY_SSIZE_T_CLEAN +#include <Python.h> +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <math.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> +#include <time.h> + +#include "nds_api_internal.h" +#include "p2p_dev_uapi.h" + +/* + * Opaque ctx wrapper: Python holds the pointer as a PyCapsule named + * "nds_io_ctx". + */ +static const char *nds_ctx_capsule_name = "nds_io_ctx"; + +static void nds_ctx_capsule_destructor(PyObject *capsule) +{ + struct nds_io_ctx *ctx = PyCapsule_GetPointer(capsule, nds_ctx_capsule_name); + + if (ctx) + nds_io_destroy_ctx(ctx); +} + +static struct nds_io_ctx *nds_ctx_from_obj(PyObject *obj) +{ + struct nds_io_ctx *ctx; + + if (!PyCapsule_CheckExact(obj)) { + PyErr_SetString(PyExc_TypeError, "ctx must be an nds_io_ctx capsule"); + return NULL; + } + ctx = PyCapsule_GetPointer(obj, nds_ctx_capsule_name); + if (!ctx) + PyErr_SetString(PyExc_ValueError, "invalid or destroyed nds_io_ctx"); + return ctx; +} + +static int parse_nds_iov(PyObject *obj, struct nds_io_vec **iov_out, + uint32_t *iov_cnt_out) +{ + struct nds_io_vec *iov; + PyObject *seq; + Py_ssize_t count; + Py_ssize_t i; + + seq = PySequence_Fast(obj, "iov must be a sequence of (buf_addr, buf_len)"); + if (!seq) + return -1; + count = PySequence_Fast_GET_SIZE(seq); + if (count <= 0 || count > NDS_IO_MAX_IOV) { + Py_DECREF(seq); + PyErr_SetString(PyExc_ValueError, "iov count out of range"); + return -1; + } + iov = calloc((size_t)count, sizeof(*iov)); + if (!iov) { + Py_DECREF(seq); + PyErr_NoMemory(); + return -1; + } + for (i = 0; i < count; i++) { + unsigned long long addr; + unsigned int len; + + if (!PyArg_ParseTuple(PySequence_Fast_GET_ITEM(seq, i), "KI", + &addr, &len)) { + free(iov); + Py_DECREF(seq); + return -1; + } + iov[i].buf_addr = addr; + iov[i].buf_len = len; + } + Py_DECREF(seq); + *iov_out = iov; + *iov_cnt_out = (uint32_t)count; + return 0; +} + +static int py_parse_fs_desc(PyObject *fd_seq, struct nds_fs_desc *desc, + int32_t **fds_out) +{ + PyObject *seq = NULL; + int32_t *fds = NULL; + Py_ssize_t count; + Py_ssize_t i; + + seq = PySequence_Fast(fd_seq, "fs_fds must be a sequence of ints"); + if (!seq) + return -1; + count = PySequence_Fast_GET_SIZE(seq); + if (count < 0 || count > UINT32_MAX) { + Py_DECREF(seq); + PyErr_SetString(PyExc_ValueError, "fs_fds count out of range"); + return -1; + } + fds = calloc((size_t)(count ? count : 1), sizeof(*fds)); + if (!fds) { + Py_DECREF(seq); + PyErr_NoMemory(); + return -1; + } + for (i = 0; i < count; i++) { + long v = PyLong_AsLong(PySequence_Fast_GET_ITEM(seq, i)); + + if (PyErr_Occurred() || v < 0 || v > INT32_MAX) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_ValueError, + "invalid fd in fs_fds"); + free(fds); + Py_DECREF(seq); + return -1; + } + fds[i] = (int32_t)v; + } + Py_DECREF(seq); + + desc->fs_fd = fds; + desc->fs_fd_cnt = (uint32_t)count; + desc->reserved = 0; + *fds_out = fds; + return 0; +} + +static PyObject *py_nds_init(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct nds_init_param param = { }; + int ret; + + if (!PyArg_ParseTuple(args, "")) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = nds_init(¶m); + Py_END_ALLOW_THREADS + return PyLong_FromLong(ret); +} + +static PyObject *py_nds_register_fs(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct nds_fs_desc desc = { }; + PyObject *fd_seq; + int32_t *fds; + int ret; + + if (!PyArg_ParseTuple(args, "O", &fd_seq)) + return NULL; + if (py_parse_fs_desc(fd_seq, &desc, &fds)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = nds_register_fs(&desc); + Py_END_ALLOW_THREADS + free(fds); + return PyLong_FromLong(ret); +} + +static PyObject *py_nds_unregister_fs(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *ignored; + + if (!PyArg_ParseTuple(args, "O", &ignored)) + return NULL; + return PyLong_FromLong(nds_unregister_fs(NULL)); +} + +static PyObject *py_nds_exit(PyObject *Py_UNUSED(self), + PyObject *Py_UNUSED(ignored)) +{ + int ret; + + Py_BEGIN_ALLOW_THREADS + ret = nds_exit(); + Py_END_ALLOW_THREADS + return PyLong_FromLong(ret); +} + +static int parse_nds_mem_ranges(PyObject *obj, struct nds_mem_range **ranges_out, + uint32_t *nr_out) +{ + struct nds_mem_range *ranges; + PyObject *seq; + Py_ssize_t count; + Py_ssize_t i; + + seq = PySequence_Fast(obj, "ranges must be a sequence of (addr, size)"); + if (!seq) + return -1; + count = PySequence_Fast_GET_SIZE(seq); + if (count <= 0 || count > NDS_MEM_MAX_RANGES) { + Py_DECREF(seq); + PyErr_SetString(PyExc_ValueError, "range count out of range"); + return -1; + } + ranges = calloc((size_t)count, sizeof(*ranges)); + if (!ranges) { + Py_DECREF(seq); + PyErr_NoMemory(); + return -1; + } + for (i = 0; i < count; i++) { + unsigned long long addr; + unsigned long long size; + + if (!PyArg_ParseTuple(PySequence_Fast_GET_ITEM(seq, i), "KK", + &addr, &size)) { + free(ranges); + Py_DECREF(seq); + return -1; + } + ranges[i].addr = addr; + ranges[i].size = size; + } + Py_DECREF(seq); + *ranges_out = ranges; + *nr_out = (uint32_t)count; + return 0; +} + +static PyObject *py_nds_register_mem(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct nds_mem_range *ranges; + PyObject *ranges_py; + uint32_t nr; + int flags = 0; + int ret; + + if (!PyArg_ParseTuple(args, "O|i", &ranges_py, &flags)) + return NULL; + if (parse_nds_mem_ranges(ranges_py, &ranges, &nr)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = nds_register_mem(&(struct nds_mem_desc){ + .ranges = ranges, .nr = nr, .flags = flags, + }); + Py_END_ALLOW_THREADS + free(ranges); + return PyLong_FromLong(ret); +} + +static PyObject *py_nds_unregister_mem(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct nds_mem_range *ranges; + PyObject *ranges_py; + uint32_t nr; + int flags = 0; + int ret; + + if (!PyArg_ParseTuple(args, "O|i", &ranges_py, &flags)) + return NULL; + if (parse_nds_mem_ranges(ranges_py, &ranges, &nr)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = nds_unregister_mem(&(struct nds_mem_desc){ + .ranges = ranges, .nr = nr, .flags = flags, + }); + Py_END_ALLOW_THREADS + free(ranges); + return PyLong_FromLong(ret); +} + +static PyObject *py_nds_io_new_ctx(PyObject *Py_UNUSED(self), PyObject *args) +{ + struct nds_io_ctx_param param = { }; + struct nds_io_ctx *ctx = NULL; + unsigned int max_io_cnt; + unsigned int flags = 0; + int ret; + + if (!PyArg_ParseTuple(args, "I|I", &max_io_cnt, &flags)) + return NULL; + param.max_io_cnt = max_io_cnt; + param.flags = flags; + + Py_BEGIN_ALLOW_THREADS + ret = nds_io_new_ctx(¶m, &ctx); + Py_END_ALLOW_THREADS + if (ret) + return PyLong_FromLong(ret); + + return PyCapsule_New(ctx, nds_ctx_capsule_name, nds_ctx_capsule_destructor); +} + +static PyObject *py_nds_io_destroy_ctx(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *capsule; + struct nds_io_ctx *ctx; + int ret; + + if (!PyArg_ParseTuple(args, "O", &capsule)) + return NULL; + ctx = nds_ctx_from_obj(capsule); + if (!ctx) + return NULL; + + /* Disable auto-destroy; this call owns the teardown. */ + if (PyCapsule_SetDestructor(capsule, NULL) < 0) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = nds_io_destroy_ctx(ctx); + Py_END_ALLOW_THREADS + + /* Invalidate capsule so later use fails cleanly. */ + if (PyCapsule_SetName(capsule, "nds_io_ctx_destroyed") < 0) + return NULL; + return PyLong_FromLong(ret); +} + +static PyObject *py_nds_io_dup_fd_for_test(PyObject *Py_UNUSED(self), + PyObject *args) +{ + PyObject *capsule; + struct nds_io_ctx *ctx; + int fd; + + if (!PyArg_ParseTuple(args, "O", &capsule)) + return NULL; + ctx = nds_ctx_from_obj(capsule); + if (!ctx) + return NULL; + + fd = fcntl(ctx->p2p_fd, F_DUPFD_CLOEXEC, 0); + return PyLong_FromLong(fd < 0 ? -errno : fd); +} + +static PyObject *py_nds_io_submit(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *capsule; + PyObject *iocb_seq; + PyObject *seq; + struct nds_io_ctx *ctx; + struct nds_io_cb *cbs = NULL; + struct nds_io_vec **iov = NULL; + Py_ssize_t count; + Py_ssize_t i; + int ret; + + if (!PyArg_ParseTuple(args, "OO", &capsule, &iocb_seq)) + return NULL; + ctx = nds_ctx_from_obj(capsule); + if (!ctx) + return NULL; + + seq = PySequence_Fast(iocb_seq, "iocbs must be a sequence"); + if (!seq) + return NULL; + count = PySequence_Fast_GET_SIZE(seq); + if (count < 0 || count > INT_MAX) { + Py_DECREF(seq); + PyErr_SetString(PyExc_ValueError, "iocb count out of range"); + return NULL; + } + if (!count) { + Py_DECREF(seq); + return PyLong_FromLong(0); + } + + cbs = calloc((size_t)count, sizeof(*cbs)); + iov = calloc((size_t)count, sizeof(*iov)); + if (!cbs || !iov) { + free(cbs); + free(iov); + Py_DECREF(seq); + return PyErr_NoMemory(); + } + + for (i = 0; i < count; i++) { + PyObject *item = PySequence_Fast_GET_ITEM(seq, i); + PyObject *fd_py; + PyObject *iov_py; + unsigned int opcode; + unsigned int rw_flags = 0; + unsigned long long offset; + unsigned long long user_data = 0; + long long fd; + int host_pid = 0; + + /* + * iocb tuple: + * (opcode, fd, offset, iov + * [, rw_flags=0, host_pid=0, user_data=0]) + * iov: sequence of (buf_addr, buf_len) + */ + if (!PyArg_ParseTuple(item, "IOKO|IiK", + &opcode, &fd_py, &offset, &iov_py, + &rw_flags, &host_pid, + &user_data)) { + goto parse_error; + } + fd = PyLong_AsLongLong(fd_py); + if (PyErr_Occurred()) + goto parse_error; + if (fd < 0 || fd > INT32_MAX) { + PyErr_SetString(PyExc_ValueError, "fd out of range"); + goto parse_error; + } + if (parse_nds_iov(iov_py, &iov[i], &cbs[i].iov_cnt)) + goto parse_error; + cbs[i].opcode = opcode; + cbs[i].rw_flags = rw_flags; + cbs[i].obj.fd = (int32_t)fd; + cbs[i].offset = offset; + cbs[i].user_data = user_data; + cbs[i].iov = iov[i]; + cbs[i].host_pid = (int32_t)host_pid; + } + Py_DECREF(seq); + + Py_BEGIN_ALLOW_THREADS + ret = nds_io_submit(ctx, (int)count, cbs); + Py_END_ALLOW_THREADS + + for (i = 0; i < count; i++) + free(iov[i]); + free(iov); + free(cbs); + return PyLong_FromLong(ret); + +parse_error: + for (i = 0; i < count; i++) + free(iov[i]); + free(iov); + free(cbs); + Py_DECREF(seq); + return NULL; +} + +static PyObject *py_nds_io_getevents(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *capsule; + struct nds_io_ctx *ctx; + struct nds_io_event *events = NULL; + struct timespec ts_storage; + struct timespec *timeout = NULL; + int min_nr; + int nr; + double timeout_sec = -1.0; + PyObject *list; + PyThreadState *thread_state; + int ret; + int i; + + if (!PyArg_ParseTuple(args, "Oii|d", &capsule, &min_nr, &nr, &timeout_sec)) + return NULL; + ctx = nds_ctx_from_obj(capsule); + if (!ctx) + return NULL; + if (min_nr < 0 || nr < 0 || min_nr > nr) + return PyLong_FromLong(-EINVAL); + if (!nr) + return PyList_New(0); + if (!isfinite(timeout_sec) || + (timeout_sec >= 0.0 && + timeout_sec > (double)INT64_MAX / 1000000000.0)) + return PyLong_FromLong(-EINVAL); + + events = calloc((size_t)nr, sizeof(*events)); + if (!events) + return PyErr_NoMemory(); + + if (timeout_sec >= 0.0) { + ts_storage.tv_sec = (time_t)timeout_sec; + ts_storage.tv_nsec = + (long)((timeout_sec - (double)ts_storage.tv_sec) * + 1000000000.0); + timeout = &ts_storage; + } + + thread_state = PyEval_SaveThread(); + ret = nds_io_getevents(ctx, min_nr, nr, events, timeout); + PyEval_RestoreThread(thread_state); + + if (ret < 0) { + free(events); + return PyLong_FromLong(ret); + } + + list = PyList_New(ret); + if (!list) { + free(events); + return NULL; + } + for (i = 0; i < ret; i++) { + PyObject *tup; + + tup = Py_BuildValue("(KL)", + (unsigned long long)events[i].user_data, + (long long)events[i].res); + if (!tup) { + Py_DECREF(list); + free(events); + return NULL; + } + PyList_SET_ITEM(list, i, tup); + } + free(events); + return list; +} + +static PyMethodDef NdsMethods[] = { + { "init", py_nds_init, METH_VARARGS, + "init() -> int\n" + "Open /dev/p2p_device. Each fd learns its NPU from its first buffer.\n" }, + { "exit", py_nds_exit, METH_NOARGS, + "exit() -> int\n" }, + { "register_fs", py_nds_register_fs, METH_VARARGS, + "register_fs(fs_fds) -> int\n" + "fs_fds: file/block fds to validate; no state is stored\n" }, + { "unregister_fs", py_nds_unregister_fs, METH_VARARGS, + "unregister_fs(fs_fds) -> int\n" + "Compatibility no-op.\n" }, + { "register_mem", py_nds_register_mem, METH_VARARGS, + "register_mem(ranges[, flags=0]) -> int\n" + "ranges: sequence of (addr, size) registered under one group\n" }, + { "unregister_mem", py_nds_unregister_mem, METH_VARARGS, + "unregister_mem(ranges[, flags=0]) -> int\n" + "ranges must match the registered group, in the same order.\n" }, + { "io_new_ctx", py_nds_io_new_ctx, METH_VARARGS, + "io_new_ctx(max_io_cnt[, flags=0]) -> ctx capsule | negative errno\n" }, + { "io_destroy_ctx", py_nds_io_destroy_ctx, METH_VARARGS, + "io_destroy_ctx(ctx) -> int\n" }, + { "_io_dup_fd_for_test", py_nds_io_dup_fd_for_test, METH_VARARGS, + "Internal test hook: duplicate the context fd.\n" }, + { "io_submit", py_nds_io_submit, METH_VARARGS, + "io_submit(ctx, iocbs) -> int\n\n" + "iocb: (opcode, fd, offset, iov" + "[, rw_flags=0, host_pid=0, user_data=0])\n" + "iov: sequence of (buf_addr, buf_len)\n" }, + { "io_getevents", py_nds_io_getevents, METH_VARARGS, + "io_getevents(ctx, min_nr, nr[, timeout_sec=-1]) -> list|int\n\n" + "On success returns [(user_data, res), ...].\n" + "On failure returns negative errno.\n" + "timeout_sec must be finite; < 0 waits forever and 0 is non-blocking.\n" }, + { NULL, NULL, 0, NULL } +}; + +static struct PyModuleDef nds_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "nds", + .m_doc = "NDS GDS-aligned P2P API", + .m_size = -1, + .m_methods = NdsMethods, +}; + +PyMODINIT_FUNC PyInit_nds(void) +{ + PyObject *module; + + module = PyModule_Create(&nds_module); + if (!module) + return NULL; + + if (PyModule_AddIntConstant(module, "NDS_API_VERSION", NDS_API_VERSION) || + PyModule_AddIntConstant(module, "NDS_IO_MAX_IO_CNT", + NDS_IO_MAX_IO_CNT) || + PyModule_AddIntConstant(module, "NDS_IO_MAX_IOV", NDS_IO_MAX_IOV) || + PyModule_AddIntConstant(module, "NDS_MEM_MAX_RANGES", + NDS_MEM_MAX_RANGES) || + PyModule_AddIntConstant(module, "NDS_IO_OP_PREAD", NDS_IO_OP_PREAD) || + PyModule_AddIntConstant(module, "NDS_IO_OP_PWRITE", NDS_IO_OP_PWRITE) || + PyModule_AddIntConstant(module, "NDS_IO_F_REGISTERED_MEM", + NDS_IO_F_REGISTERED_MEM)) { + Py_DECREF(module); + return NULL; + } + return module; +} diff --git a/tools/lib/nds/file_p2p/setup.py b/tools/lib/nds/file_p2p/setup.py new file mode 100644 index 000000000000..ebe2565943f5 --- /dev/null +++ b/tools/lib/nds/file_p2p/setup.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: GPL-2.0 +from setuptools import setup, Extension + +file_p2p_module = Extension( + "file_p2p", + sources=["file_p2p_api.c", "py_file_p2p_api.c", "p2p_common.c"], + depends=[ + "file_p2p_api.h", + "../../../include/uapi/linux/nds_p2p.h", + "p2p_common.h", + ], + extra_compile_args=["-Wall", "-Wextra", "-I../../../include/uapi"], +) + +nds_module = Extension( + "nds", + sources=["nds_api.c", "py_nds_api.c", "p2p_common.c"], + depends=[ + "nds_api.h", + "nds_api_internal.h", + "../../../include/uapi/linux/nds_p2p.h", + "p2p_common.h", + ], + extra_compile_args=["-Wall", "-Wextra", "-pthread", "-I../../../include/uapi"], + extra_link_args=["-pthread"], +) + +setup( + name="file_p2p", + version="1.0", + ext_modules=[file_p2p_module, nds_module], +) -- 2.52.0
反馈: 您发送到kernel@openeuler.org的补丁/补丁集,已成功转换为PR! PR链接地址: https://atomgit.com/openeuler/kernel/merge_requests/31092 邮件列表地址:https://mailweb.openeuler.org/archives/list/kernel@openeuler.org/message/KN4... FeedBack: The patch(es) which you have sent to kernel@openeuler.org mailing list has been converted to a pull request successfully! Pull request link: https://atomgit.com/openeuler/kernel/merge_requests/31092 Mailing list address: https://mailweb.openeuler.org/archives/list/kernel@openeuler.org/message/KN4...
participants (2)
-
patchwork bot -
Zizhi Wo