From: Hou Tao <houtao1@huawei.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 -------------------------------- Introduce drivers/nds and the nds_p2p UAPI so userspace can submit asynchronous PCI P2PDMA I/O against registered device memory. The driver registers provider pages, maps them to PCI bus addresses through P2PDMA, and issues direct-I/O kiocbs to the backing block device. The driver is module-only and requires a P2PDMA-capable queue. Signed-off-by: Hou Tao <houtao1@huawei.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- arch/arm64/configs/openeuler_defconfig | 6 + arch/x86/configs/openeuler_defconfig | 8 +- drivers/Kconfig | 2 + drivers/Makefile | 1 + drivers/nds/Kconfig | 16 + drivers/nds/Makefile | 6 + drivers/nds/debugfs.c | 188 +++ drivers/nds/debugfs.h | 22 + drivers/nds/dev.c | 1926 ++++++++++++++++++++++++ drivers/nds/mem.c | 65 + drivers/nds/mem.h | 17 + drivers/nds/mem_abi.h | 23 + drivers/nds/provider.c | 280 ++++ drivers/nds/provider.h | 40 + include/uapi/linux/nds_p2p.h | 132 ++ 15 files changed, 2731 insertions(+), 1 deletion(-) create mode 100644 drivers/nds/Kconfig create mode 100644 drivers/nds/Makefile create mode 100644 drivers/nds/debugfs.c create mode 100644 drivers/nds/debugfs.h create mode 100644 drivers/nds/dev.c create mode 100644 drivers/nds/mem.c create mode 100644 drivers/nds/mem.h create mode 100644 drivers/nds/mem_abi.h create mode 100644 drivers/nds/provider.c create mode 100644 drivers/nds/provider.h create mode 100644 include/uapi/linux/nds_p2p.h diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 1561eb379cc4..215013633a69 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -2517,10 +2517,16 @@ CONFIG_NVME_TARGET_FC=m CONFIG_NVME_TARGET_FCLOOP=m CONFIG_NVME_TARGET_TCP=m # CONFIG_NVME_TARGET_AUTH is not set # end of NVME Support +# +# NDS Support +# +CONFIG_NDS=m +# end of NDS Support + # # Misc devices # CONFIG_SENSORS_LIS3LV02D=m # CONFIG_AD525X_DPOT is not set diff --git a/arch/x86/configs/openeuler_defconfig b/arch/x86/configs/openeuler_defconfig index b38b88fb95aa..100753572e97 100644 --- a/arch/x86/configs/openeuler_defconfig +++ b/arch/x86/configs/openeuler_defconfig @@ -2086,11 +2086,11 @@ CONFIG_PCI_ATS=y CONFIG_PCI_DOE=y CONFIG_PCI_LOCKLESS_CONFIG=y CONFIG_PCI_IOV=y CONFIG_PCI_PRI=y CONFIG_PCI_PASID=y -# CONFIG_PCI_P2PDMA is not set +CONFIG_PCI_P2PDMA=y CONFIG_PCI_LABEL=y CONFIG_PCI_HYPERV=m CONFIG_VGA_ARB=y CONFIG_VGA_ARB_MAX_GPUS=64 CONFIG_HOTPLUG_PCI=y @@ -2455,10 +2455,16 @@ CONFIG_NVME_TARGET_FC=m CONFIG_NVME_TARGET_FCLOOP=m CONFIG_NVME_TARGET_TCP=m # CONFIG_NVME_TARGET_AUTH is not set # end of NVME Support +# +# NDS Support +# +CONFIG_NDS=m +# end of NDS Support + # # Misc devices # CONFIG_SENSORS_LIS3LV02D=m # CONFIG_AD525X_DPOT is not set diff --git a/drivers/Kconfig b/drivers/Kconfig index 35521e3200fa..de7dc87990be 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -36,10 +36,12 @@ source "drivers/pnp/Kconfig" source "drivers/block/Kconfig" source "drivers/nvme/Kconfig" +source "drivers/nds/Kconfig" + source "drivers/misc/Kconfig" source "drivers/scsi/Kconfig" source "drivers/ata/Kconfig" diff --git a/drivers/Makefile b/drivers/Makefile index 6f1408d617de..0754ee02bfc9 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -84,10 +84,11 @@ obj-$(CONFIG_DMA_SHARED_BUFFER) += dma-buf/ obj-$(CONFIG_NUBUS) += nubus/ obj-y += cxl/ obj-y += macintosh/ obj-y += scsi/ obj-y += nvme/ +obj-$(CONFIG_NDS) += nds/ obj-$(CONFIG_ATA) += ata/ obj-$(CONFIG_TARGET_CORE) += target/ obj-$(CONFIG_MTD) += mtd/ obj-$(CONFIG_SPI) += spi/ obj-$(CONFIG_SPMI) += spmi/ diff --git a/drivers/nds/Kconfig b/drivers/nds/Kconfig new file mode 100644 index 000000000000..3f0be3a1d8ea --- /dev/null +++ b/drivers/nds/Kconfig @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config NDS + tristate "NDS NVMe peer-to-peer I/O" + depends on m + depends on PCI_P2PDMA + depends on BLOCK && BLK_DEV_NVME + default m + help + NDS provides a character device and UAPI for PCI P2PDMA I/O + against registered device memory (e.g. NPU HBM). + + This driver can only be built as a module. The module will be + called p2p_dev. + + If unsure, say N. diff --git a/drivers/nds/Makefile b/drivers/nds/Makefile new file mode 100644 index 000000000000..94df04e04a05 --- /dev/null +++ b/drivers/nds/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# NDS / XDS NVMe peer-to-peer I/O + +obj-$(CONFIG_NDS) += p2p_dev.o +p2p_dev-y := dev.o provider.o debugfs.o mem.o diff --git a/drivers/nds/debugfs.c b/drivers/nds/debugfs.c new file mode 100644 index 000000000000..fabd9bbd6929 --- /dev/null +++ b/drivers/nds/debugfs.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/debugfs.h> +#include <linux/module.h> +#include <linux/percpu.h> +#include <linux/preempt.h> +#include <linux/seq_file.h> +#include <linux/string.h> +#include <asm/local.h> + +#include "debugfs.h" + +/* + * Per-CPU I/O counters, same pattern as block layer disk_stats / part_stat: + * Each field is indexed by P2P_IO_STATS_READ or P2P_IO_STATS_WRITE. Update + * this CPU under preempt_disable() (io_inflight via local_t); readers sum + * across CPUs. + */ +struct p2p_io_stats { + u64 io_issued[P2P_IO_STATS_NR]; + u64 io_bytes[P2P_IO_STATS_NR]; + local_t io_inflight[P2P_IO_STATS_NR]; + u64 io_failed[P2P_IO_STATS_NR]; + u64 io_issue_failed[P2P_IO_STATS_NR]; +}; + +struct p2p_io_stats_snapshot { + u64 io_issued[P2P_IO_STATS_NR]; + u64 io_bytes[P2P_IO_STATS_NR]; + s64 io_inflight[P2P_IO_STATS_NR]; + u64 io_failed[P2P_IO_STATS_NR]; + u64 io_issue_failed[P2P_IO_STATS_NR]; +}; + +static DEFINE_PER_CPU(struct p2p_io_stats, p2p_io_stats); +static struct dentry *p2p_debugfs_root; +static const char * const p2p_io_op_names[P2P_IO_STATS_NR] = { + [P2P_IO_STATS_READ] = "read", + [P2P_IO_STATS_WRITE] = "write", +}; + +static inline u32 p2p_io_stats_index(u32 op) +{ + return !!op; +} + +void p2p_stats_io_issued(u32 op, u64 bytes) +{ + struct p2p_io_stats *stats; + u32 index = p2p_io_stats_index(op); + + preempt_disable(); + stats = this_cpu_ptr(&p2p_io_stats); + stats->io_issued[index]++; + stats->io_bytes[index] += bytes; + local_inc(&stats->io_inflight[index]); + preempt_enable(); +} + +void p2p_stats_io_issue_failed(u32 op) +{ + struct p2p_io_stats *stats; + u32 index = p2p_io_stats_index(op); + + preempt_disable(); + stats = this_cpu_ptr(&p2p_io_stats); + stats->io_issue_failed[index]++; + preempt_enable(); +} + +void p2p_stats_io_complete(u32 op, int err) +{ + struct p2p_io_stats *stats; + u32 index = p2p_io_stats_index(op); + + preempt_disable(); + stats = this_cpu_ptr(&p2p_io_stats); + if (err) + stats->io_failed[index]++; + local_dec(&stats->io_inflight[index]); + preempt_enable(); +} + +static void p2p_io_stats_snapshot(struct p2p_io_stats_snapshot *out) +{ + unsigned int cpu; + u32 index; + + memset(out, 0, sizeof(*out)); + for_each_possible_cpu(cpu) { + struct p2p_io_stats *s = per_cpu_ptr(&p2p_io_stats, cpu); + + for (index = 0; index < P2P_IO_STATS_NR; index++) { + out->io_issued[index] += s->io_issued[index]; + out->io_bytes[index] += s->io_bytes[index]; + out->io_inflight[index] += local_read(&s->io_inflight[index]); + out->io_failed[index] += s->io_failed[index]; + out->io_issue_failed[index] += s->io_issue_failed[index]; + } + } +} + +static void p2p_summary_show_op(struct seq_file *seq, u32 index, + const struct p2p_io_stats_snapshot *snapshot) +{ + const char *name = p2p_io_op_names[index]; + + seq_printf(seq, "%s_io_issued: %llu\n", name, snapshot->io_issued[index]); + seq_printf(seq, "%s_io_bytes: %llu\n", name, snapshot->io_bytes[index]); + seq_printf(seq, "%s_io_inflight: %lld\n", name, snapshot->io_inflight[index]); + seq_printf(seq, "%s_io_failed: %llu\n", name, snapshot->io_failed[index]); + seq_printf(seq, "%s_io_issue_failed: %llu\n", name, snapshot->io_issue_failed[index]); +} + +static int p2p_summary_show(struct seq_file *seq, void *unused) +{ + struct p2p_io_stats_snapshot snapshot; + u32 index; + + p2p_io_stats_snapshot(&snapshot); + + for (index = 0; index < P2P_IO_STATS_NR; index++) + p2p_summary_show_op(seq, index, &snapshot); + return 0; +} + +static int p2p_summary_open(struct inode *inode, struct file *file) +{ + return single_open(file, p2p_summary_show, NULL); +} + +static ssize_t p2p_summary_write(struct file *file, + const char __user *buf, size_t count, + loff_t *ppos) +{ + unsigned int cpu; + u32 index; + + if (!count) + return 0; + + /* Reset cumulative counters only; leave io_inflight alone. */ + for_each_possible_cpu(cpu) { + struct p2p_io_stats *s = per_cpu_ptr(&p2p_io_stats, cpu); + + for (index = 0; index < P2P_IO_STATS_NR; index++) { + s->io_issued[index] = 0; + s->io_bytes[index] = 0; + s->io_failed[index] = 0; + s->io_issue_failed[index] = 0; + } + } + return count; +} + +static const struct file_operations p2p_summary_fops = { + .owner = THIS_MODULE, + .open = p2p_summary_open, + .read = seq_read, + .write = p2p_summary_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static int p2p_memory_open(struct inode *inode, struct file *file) +{ + return single_open(file, p2p_debugfs_memory_show, NULL); +} + +static const struct file_operations p2p_memory_fops = { + .owner = THIS_MODULE, + .open = p2p_memory_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void p2p_debugfs_init(void) +{ + p2p_debugfs_root = debugfs_create_dir("p2p_device", NULL); + debugfs_create_file("summary", 0600, p2p_debugfs_root, NULL, &p2p_summary_fops); + debugfs_create_file("memory", 0400, p2p_debugfs_root, NULL, &p2p_memory_fops); +} + +void p2p_debugfs_exit(void) +{ + debugfs_remove_recursive(p2p_debugfs_root); + p2p_debugfs_root = NULL; +} diff --git a/drivers/nds/debugfs.h b/drivers/nds/debugfs.h new file mode 100644 index 000000000000..419b31d699fb --- /dev/null +++ b/drivers/nds/debugfs.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef P2P_DEBUGFS_H_ +#define P2P_DEBUGFS_H_ + +#include <linux/types.h> + +#define P2P_IO_STATS_READ 0U +#define P2P_IO_STATS_WRITE 1U +#define P2P_IO_STATS_NR 2U + +struct seq_file; + +void p2p_debugfs_init(void); +void p2p_debugfs_exit(void); +int p2p_debugfs_memory_show(struct seq_file *seq, void *unused); + +void p2p_stats_io_issued(u32 op, u64 bytes); +void p2p_stats_io_issue_failed(u32 op); +/* One issue and one completion for each logical I/O; @err is 0 or -errno. */ +void p2p_stats_io_complete(u32 op, int err); + +#endif diff --git a/drivers/nds/dev.c b/drivers/nds/dev.c new file mode 100644 index 000000000000..e7c6043d3459 --- /dev/null +++ b/drivers/nds/dev.c @@ -0,0 +1,1926 @@ +// SPDX-License-Identifier: GPL-2.0 +#define pr_fmt(fmt) "p2p: " fmt + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/kdev_t.h> +#include <linux/fs.h> +#include <linux/cdev.h> +#include <linux/device.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#include <linux/ioctl.h> +#include <linux/vmalloc.h> +#include <linux/limits.h> +#include <linux/overflow.h> +#include <linux/mutex.h> +#include <linux/percpu-refcount.h> +#include <linux/pid.h> +#include <linux/random.h> +#include <linux/rcupdate.h> +#include <linux/seq_file.h> +#include <linux/sort.h> +#include <linux/string.h> +#include <linux/workqueue.h> +#include <linux/xarray.h> +#include <linux/wait.h> +#include <linux/ktime.h> +#ifdef CALC_CRC32 +#include <linux/crc32.h> +#endif + +#include <linux/blk-integrity.h> +#include <linux/blkdev.h> +#include <linux/bvec.h> +#include <linux/file.h> +#include <linux/fileattr.h> +#include <linux/magic.h> +#include <linux/pci.h> +#include <linux/pfn.h> +#include <linux/uio.h> + +#include <linux/nds_p2p.h> + +#include "mem.h" +#include "debugfs.h" +#include "provider.h" + +#define P2P_MAX_IOV 65536U +#define P2P_MAX_IOV_SIZE (2U << 30) +#define P2P_MIN_PAGE_SIZE (4U << 10) +#define P2P_MEM_COOKIE_SHIFT 48 +#define P2P_MEM_ID_MASK GENMASK_ULL(P2P_MEM_COOKIE_SHIFT - 1, 0) +/* Power-of-two CQ; caps in-flight and unharvested I/O per batch. */ +#define P2P_CQ_SIZE P2P_MAX_IO_NR +#define P2P_CQ_MASK (P2P_CQ_SIZE - 1) + +struct p2p_pa_iov { + u64 addr; + u64 len; +}; + +struct p2p_iov_map { + u64 aligned_addr; + u64 aligned_size; + u64 *pa_list; + u32 page_size; + u32 pa_num; +}; + +struct p2p_pinned_pa { + struct devmm_svm_process_id process_id; + struct p2p_iov_map *maps; + unsigned int pinned_map_nr; + u64 *pa_list; +}; + +struct p2p_batch; + +/* + * One translation segment of a registration. Adjacent ranges with the same + * page size and a page-aligned boundary share one segment. + */ +struct p2p_mem_seg { + u64 end; /* exclusive end of the registered bytes */ + u64 aligned_addr; /* devmm-aligned VA mapped by pa_list[0] */ + const u64 *pa_list; + u32 page_size; +}; + +struct p2p_registered_mem { + u64 handle; + /* Span of the registered ranges, reported by debugfs. */ + u64 start; + u64 end; + unsigned int nr_ranges; + /* Sorted segments; seg_starts holds their search keys. */ + unsigned int nr_segs; + u64 *seg_starts; + struct p2p_mem_seg *segs; + struct p2p_pinned_pa *pinned_pa; + /* Provider of every range; matches the owner fd. */ + struct p2p_provider *provider; + struct p2p_batch *owner; + struct pid *owner_tgid; + + struct percpu_ref io_refs; + struct completion io_zero; + struct rcu_head rcu; + struct list_head owner_node; +}; + +struct p2p_pinned_io_mem { + bool pinned; + bool reg_mem; + union { + struct p2p_pinned_pa *pinned_pa; + struct p2p_registered_mem *registered_mem; + }; +}; + +struct p2p_io_context; + +struct p2p_batch { + /* Cached from the first buffer; later requests reuse it. */ + struct p2p_provider *provider; + + /* Accepted CQ slots not yet retired; limited to P2P_CQ_SIZE. */ + atomic_t io_cnt; + + /* + * AIO-style completion ring (fs/aio.c shape): the producer publishes + * slots and advances cq_tail under completion_lock only; the consumer + * reads slots and advances cq_head under the ring_lock mutex only. + * Neither side takes the other's lock on the hot path. + */ + struct p2p_io_context **cq; + unsigned int cq_head; + unsigned int cq_tail; + /* Completed I/Os published to the CQ but not yet harvested. */ + atomic_t ready_events; + /* Accepted I/Os not yet harvested, including ready_events. */ + atomic_t outstanding_events; + spinlock_t completion_lock; + /* Serializes completion collection and final close. */ + struct mutex ring_lock; + + wait_queue_head_t wait; + struct list_head owned_mems; +}; + +struct p2p_io_context { + u32 op; + struct file *file; + struct kiocb kiocb; + /* The pages of pa_iov; one bvec for each physically contiguous run. */ + struct bio_vec *bvec; + unsigned int bvec_nr; + struct p2p_pa_iov *pa_iov; + unsigned int pa_iov_nr; + struct p2p_pinned_io_mem pinned_mem; + u64 data_size; + u64 user_data; + struct completion io_done; + /* One reference for the submitter and one for the VFS completion. */ + atomic_t io_ref; + int io_err; + struct p2p_batch *batch; +#if defined(CALC_CRC32) || defined(DUMP_CONTENT) + struct work_struct finalize_work; +#endif +}; + + +static dev_t dev; +static struct class *dev_class; +static struct cdev p2p_cdev; + +static DEFINE_XARRAY(registered_mems); +static DEFINE_MUTEX(registered_mem_lock); +static atomic64_t registered_mem_id = ATOMIC64_INIT(0); + +static int p2p_open(struct inode *inode, struct file *file); +static int p2p_release(struct inode *inode, struct file *file); +static long p2p_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +static void p2p_release_io(struct p2p_batch *batch); +static void p2p_revoke_all_registered_mem(struct p2p_batch *batch, + struct list_head *revoked); +static void p2p_destroy_registered_mem_list(struct list_head *revoked); +static void p2p_io_ctx_put(struct p2p_io_context *io_ctx); +static void p2p_publish_io_done(struct p2p_io_context *io_ctx); +#if defined(CALC_CRC32) || defined(DUMP_CONTENT) +static void p2p_finalize_io_work(struct work_struct *work); +#endif + +static const struct file_operations fops = { + .owner = THIS_MODULE, + .open = p2p_open, + .release = p2p_release, + .unlocked_ioctl = p2p_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +int p2p_debugfs_memory_show(struct seq_file *seq, void *unused) +{ + struct p2p_registered_mem *mem; + unsigned long index; + + rcu_read_lock(); + xa_for_each(®istered_mems, index, mem) { + /* RCU protects mem; the reference protects its arrays. */ + if (!percpu_ref_tryget_live(&mem->io_refs)) + continue; + seq_printf(seq, "handle: 0x%llx addr: 0x%llx size: 0x%llx", + mem->handle, mem->start, mem->end - mem->start); + if (mem->nr_ranges > 1) + seq_printf(seq, " ranges: %u segs: %u", mem->nr_ranges, mem->nr_segs); + seq_putc(seq, '\n'); + for (unsigned int i = 0; i < mem->nr_segs; i++) { + const struct p2p_mem_seg *seg = &mem->segs[i]; + + seq_printf(seq, " segment %u: va [0x%llx, 0x%llx) aligned 0x%llx page_size 0x%x\n", + i, mem->seg_starts[i], seg->end, + seg->aligned_addr, seg->page_size); + } + percpu_ref_put(&mem->io_refs); + } + rcu_read_unlock(); + + return 0; +} + +static int p2p_open(struct inode *inode, struct file *file) +{ + struct p2p_batch *batch; + + batch = kzalloc(sizeof(*batch), GFP_KERNEL); + if (!batch) + return -ENOMEM; + + batch->cq = kcalloc(P2P_CQ_SIZE, sizeof(*batch->cq), GFP_KERNEL); + if (!batch->cq) { + kfree(batch); + return -ENOMEM; + } + + spin_lock_init(&batch->completion_lock); + mutex_init(&batch->ring_lock); + init_waitqueue_head(&batch->wait); + INIT_LIST_HEAD(&batch->owned_mems); + atomic_set(&batch->io_cnt, 0); + /* Cross the unsigned counter wrap after eight completions. */ + batch->cq_head = -8U; + batch->cq_tail = -8U; + atomic_set(&batch->ready_events, 0); + atomic_set(&batch->outstanding_events, 0); + file->private_data = batch; + + return 0; +} + +static int p2p_release(struct inode *inode, struct file *file) +{ + struct p2p_batch *batch = file->private_data; + LIST_HEAD(revoked_mems); + + p2p_revoke_all_registered_mem(batch, &revoked_mems); + + if (atomic_read(&batch->io_cnt) > 0) + p2p_release_io(batch); + + p2p_destroy_registered_mem_list(&revoked_mems); + + kfree(batch->cq); + kfree(batch); + + file->private_data = NULL; + + return 0; +} + +/* + * devmm finds a host process by (host_pid, vm_id), and devid shares a union + * with vm_id. The id must stay 0 (the zeroed allocation): an NPU id there + * would name a VM, and every pin on an NPU other than 0 would fail. + */ +static void init_process_id(int host_pid, struct devmm_svm_process_id *process_id) +{ + rcu_read_lock(); + process_id->host_pid = pid_nr(find_vpid(host_pid)); + rcu_read_unlock(); +} + +static void init_current_process_id(struct devmm_svm_process_id *process_id) +{ + process_id->host_pid = task_tgid_nr(current); +} + +/* + * Select one provider for a request or registration. The first PA needs a + * lookup; later PAs normally need only a range check. @provider is shared by + * all maps, so the lookup cost does not grow with the IOV count. + */ +static int p2p_check_pa_list(struct p2p_provider **provider, const u64 *pa_list, + u32 pa_num, u32 page_size) +{ + u32 i; + + for (i = 0; i < pa_num; i++) { + struct p2p_provider *found; + + if (likely(*provider && + p2p_provider_contains(*provider, pa_list[i], page_size))) + continue; + found = p2p_provider_find(pa_list[i], page_size); + if (!found) { + pr_warn_ratelimited("PA 0x%llx is outside every HBM window\n", + pa_list[i]); + return -EFAULT; + } + if (*provider && found != *provider) { + pr_warn_ratelimited("PA 0x%llx belongs to %s, not %s\n", pa_list[i], + pci_name(found->pdev), pci_name((*provider)->pdev)); + return -EXDEV; + } + *provider = found; + } + return 0; +} + +static int p2p_bind_provider(struct p2p_batch *batch, struct p2p_provider *provider) +{ + struct p2p_provider *bound; + + /* Pointer comparison only; provider identity is already initialized. */ + bound = READ_ONCE(batch->provider); + if (likely(bound)) + return bound == provider ? 0 : -EXDEV; + bound = cmpxchg(&batch->provider, NULL, provider); + return !bound || bound == provider ? 0 : -EXDEV; +} + +static void p2p_put_pinned_pa(struct p2p_pinned_pa *pinned_pa) +{ + unsigned int i; + + for (i = 0; i < pinned_pa->pinned_map_nr; i++) { + struct p2p_iov_map *map = &pinned_pa->maps[i]; + + p2p_mem_put_pa_list(&pinned_pa->process_id, map->aligned_addr, + map->aligned_size, map->pa_list, + map->pa_num); + } + + kvfree(pinned_pa->pa_list); + kvfree(pinned_pa->maps); + kfree(pinned_pa); +} + +static void p2p_registered_mem_release(struct percpu_ref *ref) +{ + struct p2p_registered_mem *mem; + + mem = container_of(ref, struct p2p_registered_mem, io_refs); + pr_debug("registered memory handle 0x%llx released\n", mem->handle); + complete(&mem->io_zero); +} + +static void p2p_free_registered_mem_rcu(struct rcu_head *rcu) +{ + struct p2p_registered_mem *mem; + + mem = container_of(rcu, struct p2p_registered_mem, rcu); + percpu_ref_exit(&mem->io_refs); + kfree(mem); +} + +static void p2p_free_mem_segs(struct p2p_registered_mem *mem) +{ + kvfree(mem->seg_starts); + kvfree(mem->segs); +} + +static void p2p_destroy_registered_mem(struct p2p_registered_mem *mem) +{ + wait_for_completion(&mem->io_zero); + p2p_free_mem_segs(mem); + p2p_put_pinned_pa(mem->pinned_pa); + put_pid(mem->owner_tgid); + call_rcu(&mem->rcu, p2p_free_registered_mem_rcu); +} + +/* Pin each sorted range with its own devmm call; maps share one PA array. */ +static int p2p_pin_registered_pa(struct p2p_batch *batch, + const struct p2p_mem_range *ranges, unsigned int nr, + struct p2p_pinned_pa **pinned_pa_out, + struct p2p_provider **provider_out) +{ + struct p2p_provider *provider = READ_ONCE(batch->provider); + struct p2p_pinned_pa *pinned_pa; + u64 total_pa_num = 0; + unsigned int i; + int err; + + pinned_pa = kzalloc(sizeof(*pinned_pa), GFP_KERNEL); + if (!pinned_pa) + return -ENOMEM; + init_current_process_id(&pinned_pa->process_id); + + pinned_pa->maps = kvcalloc(nr, sizeof(*pinned_pa->maps), GFP_KERNEL); + if (!pinned_pa->maps) { + err = -ENOMEM; + goto put_pinned_pa; + } + + for (i = 0; i < nr; i++) { + struct p2p_iov_map *map = &pinned_pa->maps[i]; + u64 addr = ranges[i].addr; + u64 aligned_end; + u64 pa_num; + int page_size; + + page_size = p2p_mem_get_page_size(&pinned_pa->process_id, addr, ranges[i].size); + if (page_size < 0) { + err = page_size; + goto put_pinned_pa; + } + if (page_size < P2P_MIN_PAGE_SIZE || !is_power_of_2(page_size)) { + pr_err("invalid page size %d for registered range 0x%llx+0x%llx\n", + page_size, addr, ranges[i].size); + err = -EINVAL; + goto put_pinned_pa; + } + + if (check_add_overflow(addr + ranges[i].size, (u64)page_size - 1, &aligned_end)) { + err = -EOVERFLOW; + goto put_pinned_pa; + } + aligned_end &= ~((u64)page_size - 1); + + map->aligned_addr = addr & ~((u64)page_size - 1); + map->aligned_size = aligned_end - map->aligned_addr; + map->page_size = page_size; + pa_num = map->aligned_size / map->page_size; + if (!pa_num || pa_num > U32_MAX - total_pa_num) { + err = -E2BIG; + goto put_pinned_pa; + } + map->pa_num = pa_num; + total_pa_num += pa_num; + } + + pinned_pa->pa_list = kvmalloc_array(total_pa_num, sizeof(*pinned_pa->pa_list), GFP_KERNEL); + if (!pinned_pa->pa_list) { + err = -ENOMEM; + goto put_pinned_pa; + } + + total_pa_num = 0; + for (i = 0; i < nr; i++) { + struct p2p_iov_map *map = &pinned_pa->maps[i]; + + map->pa_list = pinned_pa->pa_list + total_pa_num; + err = p2p_mem_get_pa_list(&pinned_pa->process_id, map->aligned_addr, + map->aligned_size, map->pa_list, map->pa_num); + if (err) { + pr_err("registered PA list addr 0x%llx size 0x%llx num %u range %u err %d\n", + map->aligned_addr, map->aligned_size, map->pa_num, i, err); + goto put_pinned_pa; + } + pinned_pa->pinned_map_nr++; + err = p2p_check_pa_list(&provider, map->pa_list, map->pa_num, map->page_size); + if (err) + goto put_pinned_pa; + total_pa_num += map->pa_num; + } + + err = p2p_provider_prepare(provider); + if (err) + goto put_pinned_pa; + err = p2p_bind_provider(batch, provider); + if (err) + goto put_pinned_pa; + *pinned_pa_out = pinned_pa; + *provider_out = provider; + return 0; + +put_pinned_pa: + p2p_put_pinned_pa(pinned_pa); + return err; +} + +/* + * Build the lookup segments from the pinned maps. A range starting exactly at + * the page-aligned end of the previous segment with the same page size extends + * that segment: the maps are laid out in VA order in one PA array, so the + * previous segment's pa_list indexes straight into the next map. + */ +static int p2p_build_mem_segs(struct p2p_registered_mem *mem, const struct p2p_mem_range *ranges, + unsigned int nr) +{ + const struct p2p_iov_map *maps = mem->pinned_pa->maps; + unsigned int n = 0; + unsigned int i; + + mem->seg_starts = kvmalloc_array(nr, sizeof(*mem->seg_starts), GFP_KERNEL); + mem->segs = kvmalloc_array(nr, sizeof(*mem->segs), GFP_KERNEL); + if (!mem->seg_starts || !mem->segs) { + p2p_free_mem_segs(mem); + return -ENOMEM; + } + + for (i = 0; i < nr; i++) { + u64 end = ranges[i].addr + ranges[i].size; + + if (n && mem->segs[n - 1].end == ranges[i].addr && + mem->segs[n - 1].page_size == maps[i].page_size && + IS_ALIGNED(ranges[i].addr, maps[i].page_size)) { + mem->segs[n - 1].end = end; + continue; + } + mem->seg_starts[n] = ranges[i].addr; + mem->segs[n] = (struct p2p_mem_seg) { + .end = end, + .aligned_addr = maps[i].aligned_addr, + .pa_list = maps[i].pa_list, + .page_size = maps[i].page_size, + }; + n++; + } + + mem->nr_segs = n; + mem->nr_ranges = nr; + mem->start = ranges[0].addr; + mem->end = ranges[nr - 1].addr + ranges[nr - 1].size; + return 0; +} + +static u64 p2p_new_registered_mem_handle(void) +{ + u64 id = atomic64_inc_return(®istered_mem_id) & P2P_MEM_ID_MASK; + u64 cookie = get_random_u32() & U16_MAX; + + return cookie << P2P_MEM_COOKIE_SHIFT | id; +} + +static int p2p_publish_registered_mem(struct p2p_batch *batch, + struct p2p_registered_mem *mem) +{ + int err; + + mutex_lock(®istered_mem_lock); + mem->owner = batch; + err = xa_insert(®istered_mems, mem->handle, mem, GFP_KERNEL); + if (!err) { + list_add_tail(&mem->owner_node, &batch->owned_mems); + percpu_ref_reinit(&mem->io_refs); + } else { + pr_err("publish registered memory handle 0x%llx err %d\n", mem->handle, err); + } + mutex_unlock(®istered_mem_lock); + + return err; +} + +static int p2p_unpublish_registered_mem(struct p2p_batch *batch, u64 handle, + struct p2p_registered_mem **mem_out) +{ + struct p2p_registered_mem *mem; + int err = 0; + + mutex_lock(®istered_mem_lock); + rcu_read_lock(); + mem = xa_load(®istered_mems, handle); + if (!mem) { + err = -ENOENT; + goto unlock_rcu; + } + if (mem->owner != batch) { + pr_err("registered memory handle 0x%llx belongs to another p2p fd\n", handle); + err = -EPERM; + goto unlock_rcu; + } + if (mem->owner_tgid != task_tgid(current)) { + pr_err("registered memory handle 0x%llx belongs to another process\n", handle); + err = -EPERM; + goto unlock_rcu; + } + rcu_read_unlock(); + + xa_erase(®istered_mems, handle); + list_del_init(&mem->owner_node); + *mem_out = mem; + mutex_unlock(®istered_mem_lock); + return 0; + +unlock_rcu: + rcu_read_unlock(); + mutex_unlock(®istered_mem_lock); + return err; +} + +static void p2p_revoke_all_registered_mem(struct p2p_batch *batch, + struct list_head *revoked) +{ + struct p2p_registered_mem *mem, *next; + + if (list_empty(&batch->owned_mems)) + return; + + list_for_each_entry_safe(mem, next, &batch->owned_mems, owner_node) { + xa_erase(®istered_mems, mem->handle); + list_move_tail(&mem->owner_node, revoked); + pr_info("unregister memory handle 0x%llx through p2p fd release\n", + mem->handle); + } + + list_for_each_entry(mem, revoked, owner_node) + percpu_ref_kill(&mem->io_refs); +} + +static void p2p_destroy_registered_mem_list(struct list_head *revoked) +{ + struct p2p_registered_mem *mem; + struct p2p_registered_mem *next; + + list_for_each_entry_safe(mem, next, revoked, owner_node) { + list_del_init(&mem->owner_node); + p2p_destroy_registered_mem(mem); + } +} + +static int p2p_mem_range_cmp(const void *a, const void *b) +{ + const struct p2p_mem_range *ra = a; + const struct p2p_mem_range *rb = b; + + if (ra->addr < rb->addr) + return -1; + return ra->addr > rb->addr; +} + +/* Sort ranges by VA and reject empty, overflowing, or overlapping ones. */ +static int p2p_sort_mem_ranges(struct p2p_mem_range *ranges, unsigned int nr) +{ + unsigned int i; + u64 end; + + for (i = 0; i < nr; i++) { + if (!ranges[i].size) + return -EINVAL; + if (check_add_overflow(ranges[i].addr, ranges[i].size, &end)) + return -EOVERFLOW; + } + + sort(ranges, nr, sizeof(*ranges), p2p_mem_range_cmp, NULL); + for (i = 1; i < nr; i++) { + if (ranges[i].addr < ranges[i - 1].addr + ranges[i - 1].size) + return -EINVAL; + } + return 0; +} + +static int p2p_register_mem_ranges(struct p2p_batch *batch, struct p2p_mem_range *ranges, + unsigned int nr, u64 __user *handle_out) +{ + struct p2p_registered_mem *mem; + int err; + + err = p2p_sort_mem_ranges(ranges, nr); + if (err) + return err; + + mem = kzalloc(sizeof(*mem), GFP_KERNEL); + if (!mem) + return -ENOMEM; + mem->owner_tgid = get_task_pid(current, PIDTYPE_TGID); + init_completion(&mem->io_zero); + INIT_LIST_HEAD(&mem->owner_node); + + err = percpu_ref_init(&mem->io_refs, p2p_registered_mem_release, + PERCPU_REF_INIT_DEAD, GFP_KERNEL); + if (err) + goto put_owner_tgid; + + err = p2p_pin_registered_pa(batch, ranges, nr, &mem->pinned_pa, &mem->provider); + if (err) + goto exit_io_refs; + + err = p2p_build_mem_segs(mem, ranges, nr); + if (err) + goto put_pinned_pa; + + mem->handle = p2p_new_registered_mem_handle(); + if (put_user(mem->handle, handle_out)) { + err = -EFAULT; + goto free_segs; + } + + err = p2p_publish_registered_mem(batch, mem); + if (err) + goto free_segs; + + pr_info("registered memory handle 0x%llx addr 0x%llx size 0x%llx ranges %u segs %u\n", + mem->handle, mem->start, mem->end - mem->start, mem->nr_ranges, mem->nr_segs); + return 0; + +free_segs: + p2p_free_mem_segs(mem); +put_pinned_pa: + p2p_put_pinned_pa(mem->pinned_pa); +exit_io_refs: + percpu_ref_exit(&mem->io_refs); +put_owner_tgid: + put_pid(mem->owner_tgid); + kfree(mem); + return err; +} + +static int p2p_register_mem(struct p2p_batch *batch, void __user *arg) +{ + struct p2p_mem_register_param __user *user_param = arg; + struct p2p_mem_register_param param; + struct p2p_mem_range *ranges; + int err; + + if (copy_from_user(¶m, arg, sizeof(param))) + return -EFAULT; + if (param.flags || param.reserved[0] || param.reserved[1] || + param.reserved[2] || !param.ranges || + !param.nr || param.nr > P2P_MEM_MAX_RANGES) + return -EINVAL; + + ranges = kvmalloc_array(param.nr, sizeof(*ranges), GFP_KERNEL); + if (!ranges) + return -ENOMEM; + if (copy_from_user(ranges, u64_to_user_ptr(param.ranges), + array_size(param.nr, sizeof(*ranges)))) { + err = -EFAULT; + goto free_ranges; + } + err = p2p_register_mem_ranges(batch, ranges, param.nr, &user_param->mem_handle); +free_ranges: + kvfree(ranges); + return err; +} + +static int p2p_unregister_mem(struct p2p_batch *batch, void __user *arg) +{ + struct p2p_mem_unregister_param param; + struct p2p_registered_mem *mem; + int err; + + if (copy_from_user(¶m, arg, sizeof(param))) + return -EFAULT; + if (param.reserved[0] || param.reserved[1] || param.reserved[2] || + !param.mem_handle) + return -EINVAL; + + err = p2p_unpublish_registered_mem(batch, param.mem_handle, &mem); + if (err) { + pr_err("unregister memory handle 0x%llx err %d\n", param.mem_handle, err); + return err; + } + + percpu_ref_kill(&mem->io_refs); + p2p_destroy_registered_mem(mem); + + pr_info("unregistered memory handle 0x%llx\n", param.mem_handle); + return 0; +} + +#ifdef DUMP_CONTENT +static void dump_pa_content(u64 pa, u64 size) +{ + unsigned int data_len = min_t(u64, size, 16U << 10); + char prefix[64]; + void *addr; + + addr = ioremap(pa, data_len); + if (!addr) + return; + + scnprintf(prefix, sizeof(prefix), "p2p: PA 0x%llx content: ", pa); + print_hex_dump(KERN_INFO, prefix, DUMP_PREFIX_ADDRESS, 16, 1, addr, data_len, false); + + iounmap(addr); +} +#else +static inline void dump_pa_content(u64 pa, u64 size) +{ +} +#endif + +static int get_pa_iov(struct p2p_batch *batch, int host_pid, + const struct p2p_iov *iov, unsigned int iov_nr, + struct p2p_pa_iov **pa_iov, unsigned int *pa_iov_nr, + struct p2p_pinned_io_mem *pinned_mem) +{ + struct p2p_provider *provider = READ_ONCE(batch->provider); + struct p2p_pinned_pa *pinned_pa; + struct p2p_pa_iov *new_pa_iov; + struct p2p_iov_map *maps; + unsigned int new_pa_iov_nr; + unsigned int new_pa_iov_idx; + unsigned int i; + int err; + + pinned_pa = kzalloc(sizeof(*pinned_pa), GFP_KERNEL); + if (!pinned_pa) + return -ENOMEM; + init_process_id(host_pid, &pinned_pa->process_id); + pinned_pa->maps = kvcalloc(iov_nr, sizeof(*pinned_pa->maps), GFP_KERNEL); + if (!pinned_pa->maps) { + err = -ENOMEM; + goto put_pinned_pa; + } + maps = pinned_pa->maps; + + new_pa_iov_nr = 0; + for (i = 0; i < iov_nr; i++) { + u32 range_size; + u32 pa_num; + u32 offset; + int page_size; + + page_size = p2p_mem_get_page_size(&pinned_pa->process_id, iov[i].addr, iov[i].size); + if (page_size < P2P_MIN_PAGE_SIZE || !is_power_of_2(page_size)) { + pr_err_ratelimited("invalid page size %d for addr 0x%llx, iov %u\n", + page_size, iov[i].addr, i); + err = -EINVAL; + goto put_pinned_pa; + } + + offset = iov[i].addr & (page_size - 1); + range_size = offset + iov[i].size; + pa_num = DIV_ROUND_UP(range_size, (u32)page_size); + if (!pa_num || pa_num > UINT_MAX - new_pa_iov_nr) { + err = -E2BIG; + goto put_pinned_pa; + } + + maps[i].aligned_addr = iov[i].addr - offset; + maps[i].page_size = page_size; + maps[i].pa_num = pa_num; + maps[i].aligned_size = (u64)pa_num * page_size; + new_pa_iov_nr += maps[i].pa_num; + } + + pinned_pa->pa_list = kvmalloc_array(new_pa_iov_nr, sizeof(*pinned_pa->pa_list), GFP_KERNEL); + if (!pinned_pa->pa_list) { + err = -ENOMEM; + goto put_pinned_pa; + } + + new_pa_iov = kvmalloc_array(new_pa_iov_nr, sizeof(*new_pa_iov), GFP_KERNEL); + if (!new_pa_iov) { + err = -ENOMEM; + goto put_pinned_pa; + } + + new_pa_iov_idx = 0; + for (i = 0; i < iov_nr; i++) { + u32 remaining = iov[i].size; + u32 offset = iov[i].addr - maps[i].aligned_addr; + unsigned int j; + + maps[i].pa_list = pinned_pa->pa_list + new_pa_iov_idx; + err = p2p_mem_get_pa_list(&pinned_pa->process_id, + maps[i].aligned_addr, + maps[i].aligned_size, + maps[i].pa_list, maps[i].pa_num); + if (err) { + pr_err_ratelimited("PA list addr 0x%llx size 0x%llx num %u iov %u err %d\n", + maps[i].aligned_addr, maps[i].aligned_size, + maps[i].pa_num, i, err); + goto free_pa_iov; + } + pinned_pa->pinned_map_nr++; + err = p2p_check_pa_list(&provider, maps[i].pa_list, maps[i].pa_num, + maps[i].page_size); + if (err) + goto free_pa_iov; + + for (j = 0; j < maps[i].pa_num; j++) { + u32 len = min(remaining, maps[i].page_size - offset); + + new_pa_iov[new_pa_iov_idx].addr = maps[i].pa_list[j] + offset; + new_pa_iov[new_pa_iov_idx].len = len; + new_pa_iov_idx++; + remaining -= len; + offset = 0; + } + + if (remaining) { + pr_err_ratelimited("PA list leaves 0x%x bytes for iov %u\n", + remaining, i); + err = -EINVAL; + goto free_pa_iov; + } + } + + err = p2p_provider_prepare(provider); + if (err) + goto free_pa_iov; + err = p2p_bind_provider(batch, provider); + if (err) + goto free_pa_iov; + *pa_iov = new_pa_iov; + *pa_iov_nr = new_pa_iov_nr; + *pinned_mem = (struct p2p_pinned_io_mem) { + .pinned = true, + .pinned_pa = pinned_pa, + }; + return 0; + +free_pa_iov: + kvfree(new_pa_iov); +put_pinned_pa: + p2p_put_pinned_pa(pinned_pa); + return err; +} + +static int p2p_get_registered_mem(u64 handle, struct p2p_registered_mem **mem_out) +{ + struct p2p_registered_mem *mem; + int err; + + rcu_read_lock(); + mem = xa_load(®istered_mems, handle); + if (!mem) { + err = -ENOENT; + goto unlock; + } + if (!percpu_ref_tryget_live(&mem->io_refs)) { + err = -ESTALE; + goto unlock; + } + rcu_read_unlock(); + if (mem->owner_tgid != task_tgid(current)) { + percpu_ref_put(&mem->io_refs); + return -EPERM; + } + + *mem_out = mem; + return 0; + +unlock: + rcu_read_unlock(); + return err; +} + +/* + * Find the segment containing @addr. Try the caller's cursor and its successor + * before binary search: the IOVs of one request usually walk the group in VA + * order. The cursor is per request so concurrent submitters share nothing. + */ +static const struct p2p_mem_seg *p2p_mem_find_seg(const struct p2p_registered_mem *mem, u64 addr, + unsigned int *cursor) +{ + unsigned int lo = *cursor; + unsigned int hi; + + if (lo < mem->nr_segs && addr >= mem->seg_starts[lo]) { + if (addr < mem->segs[lo].end) + goto found; + lo++; + if (lo < mem->nr_segs && addr >= mem->seg_starts[lo] && addr < mem->segs[lo].end) + goto found; + } + + /* Find the last segment starting at or below addr. */ + lo = 0; + hi = mem->nr_segs; + while (lo < hi) { + unsigned int mid = lo + (hi - lo) / 2; + + if (mem->seg_starts[mid] <= addr) + lo = mid + 1; + else + hi = mid; + } + if (!lo || addr >= mem->segs[lo - 1].end) + return NULL; + lo--; +found: + *cursor = lo; + return &mem->segs[lo]; +} + +/* Translate or count one IOV. It must fit in one translation segment. */ +static int p2p_mem_map_iov(const struct p2p_registered_mem *mem, const struct p2p_iov *iov, + unsigned int *cursor, struct p2p_pa_iov *pa_iov, u64 *pa_iov_nr) +{ + const struct p2p_mem_seg *seg; + u64 seg_offset; + u64 page_idx; + u32 page_offset; + u32 remaining = iov->size; + + seg = p2p_mem_find_seg(mem, iov->addr, cursor); + if (!seg || remaining > seg->end - iov->addr) + return -ERANGE; + + seg_offset = iov->addr - seg->aligned_addr; + page_idx = seg_offset / seg->page_size; + page_offset = seg_offset % seg->page_size; + if (!pa_iov) { + *pa_iov_nr += DIV_ROUND_UP((u64)page_offset + remaining, seg->page_size); + return 0; + } + while (remaining) { + u32 len = min(remaining, seg->page_size - page_offset); + + pa_iov[*pa_iov_nr].addr = seg->pa_list[page_idx] + page_offset; + pa_iov[*pa_iov_nr].len = len; + (*pa_iov_nr)++; + remaining -= len; + page_idx++; + page_offset = 0; + } + return 0; +} + +static int get_registered_pa_iov(struct p2p_batch *batch, u64 handle, + const struct p2p_iov *iov, unsigned int iov_nr, + struct p2p_pa_iov **pa_iov_out, unsigned int *pa_iov_nr_out, + struct p2p_pinned_io_mem *pinned_mem) +{ + struct p2p_registered_mem *mem; + struct p2p_pa_iov *pa_iov; + unsigned int cursor = 0; + u64 pa_iov_nr = 0; + unsigned int i; + int err; + + err = p2p_get_registered_mem(handle, &mem); + if (err) + return err; + /* The registration already supplies its provider. */ + err = p2p_bind_provider(batch, mem->provider); + if (err) + goto put_mem; + + for (i = 0; i < iov_nr; i++) { + err = p2p_mem_map_iov(mem, &iov[i], &cursor, NULL, &pa_iov_nr); + if (err) + goto put_mem; + if (pa_iov_nr > UINT_MAX) { + err = -E2BIG; + goto put_mem; + } + } + + pa_iov = kvmalloc_array(pa_iov_nr, sizeof(*pa_iov), GFP_KERNEL); + if (!pa_iov) { + err = -ENOMEM; + goto put_mem; + } + + cursor = 0; + pa_iov_nr = 0; + for (i = 0; i < iov_nr; i++) { + err = p2p_mem_map_iov(mem, &iov[i], &cursor, pa_iov, &pa_iov_nr); + if (err) + goto free_pa_iov; + } + + *pa_iov_out = pa_iov; + *pa_iov_nr_out = pa_iov_nr; + *pinned_mem = (struct p2p_pinned_io_mem) { + .pinned = true, + .reg_mem = true, + .registered_mem = mem, + }; + return 0; + +free_pa_iov: + kvfree(pa_iov); +put_mem: + percpu_ref_put(&mem->io_refs); + return err; +} + +static void p2p_unpin_io_mem(struct p2p_pinned_io_mem *pinned_mem) +{ + if (!pinned_mem->pinned) + return; + + if (pinned_mem->reg_mem) + percpu_ref_put(&pinned_mem->registered_mem->io_refs); + else + p2p_put_pinned_pa(pinned_mem->pinned_pa); + pinned_mem->pinned = false; +} + +static void free_io_ctx(struct p2p_io_context *io_ctx) +{ + p2p_unpin_io_mem(&io_ctx->pinned_mem); + kvfree(io_ctx->bvec); + kvfree(io_ctx->pa_iov); + fput(io_ctx->file); + kfree(io_ctx); +} + +static struct p2p_io_context *new_io_ctx(u32 op, struct file *file, struct p2p_pa_iov *pa_iov, + unsigned int pa_iov_nr, struct bio_vec *bvec, + unsigned int bvec_nr, + struct p2p_pinned_io_mem *pinned_mem, u64 data_size, + u64 user_data) +{ + struct p2p_io_context *io_ctx; + + io_ctx = kzalloc(sizeof(*io_ctx), GFP_KERNEL); + if (!io_ctx) + return ERR_PTR(-ENOMEM); + + io_ctx->op = op; + io_ctx->file = file; + io_ctx->pa_iov = pa_iov; + io_ctx->pa_iov_nr = pa_iov_nr; + io_ctx->bvec = bvec; + io_ctx->bvec_nr = bvec_nr; + io_ctx->pinned_mem = *pinned_mem; + io_ctx->data_size = data_size; + io_ctx->user_data = user_data; + + atomic_set(&io_ctx->io_ref, 1); + init_completion(&io_ctx->io_done); +#if defined(CALC_CRC32) || defined(DUMP_CONTENT) + INIT_WORK(&io_ctx->finalize_work, p2p_finalize_io_work); +#endif + + pinned_mem->pinned = false; + return io_ctx; +} + +static void p2p_wake_batch_waiters(struct p2p_batch *batch) +{ + /* + * Pair condition stores (ready_events / outstanding_events / cq_tail + * under completion_lock, then unlock) with waitqueue_active: without + * this full barrier a waiter can observe !active, then miss the + * published condition and sleep. wait_event* still re-checks after + * prepare_to_wait. + */ + smp_mb(); + if (waitqueue_active(&batch->wait)) + wake_up(&batch->wait); +} + +static void p2p_publish_io_done(struct p2p_io_context *io_ctx) +{ + struct p2p_batch *batch = io_ctx->batch; + unsigned long flags; + + /* + * Producer hot path: completion_lock only (AIO's ctx->completion_lock + * shape). Always publish; final close waits + * for natural publish then harvests under ring_lock. + */ + spin_lock_irqsave(&batch->completion_lock, flags); + WARN_ON_ONCE(batch->cq_tail - READ_ONCE(batch->cq_head) >= P2P_CQ_SIZE); + batch->cq[batch->cq_tail & P2P_CQ_MASK] = io_ctx; + /* Slot stores must be visible before cq_tail advances. */ + smp_wmb(); + WRITE_ONCE(batch->cq_tail, batch->cq_tail + 1); + atomic_inc(&batch->ready_events); + spin_unlock_irqrestore(&batch->completion_lock, flags); + + p2p_wake_batch_waiters(batch); + /* + * Last touch of io_ctx: collection and close wait for io_done before + * freeing. + */ + complete(&io_ctx->io_done); +} + +static const char *p2p_io_op_name(u32 op) +{ + return op == P2P_IO_WRITE ? "write" : "read"; +} + +#ifdef CALC_CRC32 +static void dump_io_ctx_crc32(const struct p2p_io_context *io_ctx) +{ + u32 crc = ~0U; + unsigned int i; + void *buffer; + + /* + * Logical-I/O finalization observed every NVMe DMA completion before + * this helper. Order the following CPU reads after those completions. + */ + dma_rmb(); + + buffer = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buffer) { + pr_err("crc32 failed file=%pD length=0x%llx err=%d\n", + io_ctx->file, io_ctx->data_size, -ENOMEM); + return; + } + + for (i = 0; i < io_ctx->pa_iov_nr; i++) { + const struct p2p_pa_iov *iov = &io_ctx->pa_iov[i]; + void __iomem *addr; + u64 offset = 0; + + addr = ioremap(iov->addr, iov->len); + if (!addr) { + pr_err("crc32 failed file=%pD length=0x%llx err=%d\n", + io_ctx->file, io_ctx->data_size, -ENOMEM); + kfree(buffer); + return; + } + + while (offset < iov->len) { + size_t chunk = min_t(u64, iov->len - offset, PAGE_SIZE); + + memcpy_fromio(buffer, addr + offset, chunk); + crc = crc32_le(crc, buffer, chunk); + offset += chunk; + } + iounmap(addr); + } + + kfree(buffer); + crc ^= ~0U; + pr_info("crc32 file=%pD length=0x%llx crc32=0x%08x\n", + io_ctx->file, io_ctx->data_size, crc); +} +#else +static inline void dump_io_ctx_crc32(const struct p2p_io_context *io_ctx) +{ +} +#endif + +#if defined(CALC_CRC32) || defined(DUMP_CONTENT) +static void p2p_finalize_io_work(struct work_struct *work) +{ + struct p2p_io_context *io_ctx = + container_of(work, struct p2p_io_context, finalize_work); + + dump_io_ctx_crc32(io_ctx); + dump_pa_content(io_ctx->pa_iov[0].addr, + min_t(u64, io_ctx->data_size, io_ctx->pa_iov[0].len)); + /* + * Registered unpin here (process context). One-shot stays pinned until + * retire; pinned_mem itself is an embedded field and remains valid. + */ + if (io_ctx->pinned_mem.reg_mem) + p2p_unpin_io_mem(&io_ctx->pinned_mem); + p2p_publish_io_done(io_ctx); +} +#endif + +static void p2p_io_ctx_put(struct p2p_io_context *io_ctx) +{ + if (!atomic_dec_and_test(&io_ctx->io_ref)) + return; + +#if defined(CALC_CRC32) || defined(DUMP_CONTENT) + if (io_ctx->op == P2P_IO_READ && !io_ctx->io_err) { + schedule_work(&io_ctx->finalize_work); + return; + } +#endif + + /* + * Registered: drop the percpu_ref on the completion path (safe). + * One-shot: leave pinned until p2p_retire_cq_head so + * devmm_put_mem_pa_list never runs in interrupt context. + */ + if (io_ctx->pinned_mem.reg_mem) + p2p_unpin_io_mem(&io_ctx->pinned_mem); + + p2p_publish_io_done(io_ctx); +} + +/* + * Record the VFS result of one logical I/O and drop the completion reference. + * The event carries no byte count, so a short transfer is an error. Runs in + * the context of the VFS completion: hard interrupt, a workqueue, or the + * submitter when the VFS completed the I/O without queuing it. + */ +static void p2p_io_done(struct p2p_io_context *io_ctx, long res) +{ + int err = 0; + + if (res < 0) + err = res; + else if ((u64)res != io_ctx->data_size) + err = -EIO; + + if (err) + pr_err_ratelimited("%s file=%pD length=0x%llx result %ld err %d\n", + p2p_io_op_name(io_ctx->op), io_ctx->file, + io_ctx->data_size, res, err); + p2p_stats_io_complete(io_ctx->op, err); + io_ctx->io_err = err; + p2p_io_ctx_put(io_ctx); +} + +static void p2p_kiocb_complete(struct kiocb *kiocb, long res) +{ + struct p2p_io_context *io_ctx = container_of(kiocb, struct p2p_io_context, kiocb); + + /* Queued writes retain freeze protection until completion. */ + if (io_ctx->op == P2P_IO_WRITE) + kiocb_end_write(kiocb); + p2p_io_done(io_ctx, res); +} + +/* + * Issue one logical I/O as one direct-I/O kiocb over the HBM pages. The + * filesystem maps the file offset and the block layer splits and DMA-maps the + * pages as PCI P2PDMA memory. The caller holds its own io_ctx reference: + * p2p_kiocb_complete() can run before the VFS call returns. + */ +static void p2p_issue_io(struct p2p_io_context *io_ctx, loff_t offset) +{ + struct kiocb *kiocb = &io_ctx->kiocb; + struct iov_iter iter; + ssize_t ret; + + init_sync_kiocb(kiocb, io_ctx->file); + kiocb->ki_pos = offset; + /* The fd need not be O_DIRECT: the direct path is the only P2P path. */ + kiocb->ki_flags |= IOCB_DIRECT; + kiocb->ki_complete = p2p_kiocb_complete; + + atomic_inc(&io_ctx->io_ref); + p2p_stats_io_issued(io_ctx->op, io_ctx->data_size); + + if (io_ctx->op == P2P_IO_WRITE) { + kiocb->ki_flags |= IOCB_WRITE; + iov_iter_bvec(&iter, ITER_SOURCE, io_ctx->bvec, io_ctx->bvec_nr, + io_ctx->data_size); + ret = vfs_iocb_iter_write(io_ctx->file, kiocb, &iter); + } else { + iov_iter_bvec(&iter, ITER_DEST, io_ctx->bvec, io_ctx->bvec_nr, + io_ctx->data_size); + ret = vfs_iocb_iter_read(io_ctx->file, kiocb, &iter); + } + + /* The VFS completed this I/O; ki_complete will not run. */ + if (ret != -EIOCBQUEUED) + p2p_io_done(io_ctx, ret); +} + +static int validate_io_param(const struct p2p_io_param *param) +{ + if (param->op != P2P_IO_READ && param->op != P2P_IO_WRITE) + return -EINVAL; + if (param->flags & ~P2P_IO_F_MASK) + return -EOPNOTSUPP; + if (!param->iov_nr || param->iov_nr > P2P_MAX_IOV) + return -EINVAL; + if (param->reserved || param->reserved2[0] || param->reserved2[1] || param->reserved2[2]) + return -EINVAL; + if (param->flags & P2P_IO_F_REGISTERED_MEM) + return !param->mem_handle || param->host_pid ? -EINVAL : 0; + + return param->host_pid <= 0 ? -EINVAL : 0; +} + +/* XFS can keep the data of a realtime file on another device than s_bdev. */ +static bool p2p_is_xfs_realtime(struct file *file) +{ + struct fileattr fa = { }; + + if (vfs_fileattr_get(file->f_path.dentry, &fa)) + return true; + return fa.fsx_xflags & FS_XFLAG_REALTIME; +} + +/* + * Check the backing filesystem and queue for an fd. Filesystem checks may + * use a directory fd; I/O checks require a file or block device with direct + * I/O support. + * + * The data must reach an NVMe queue that takes P2PDMA pages, and no layer on + * the way may touch the pages with the CPU. The checks hold no state, so they + * run for every request. + */ +static int p2p_check_target(struct file *file, struct block_device **bdev_out, + bool for_io) +{ + struct inode *inode = file_inode(file); + struct block_device *bdev; + + if (S_ISBLK(inode->i_mode)) { + bdev = I_BDEV(file->f_mapping->host); + } else if (S_ISREG(inode->i_mode) || (!for_io && S_ISDIR(inode->i_mode))) { + /* + * Only filesystems with an iomap direct-I/O path that keeps all + * file data on s_bdev. + */ + switch (inode->i_sb->s_magic) { + case EXT4_SUPER_MAGIC: + break; + case XFS_SUPER_MAGIC: + if (S_ISREG(inode->i_mode) && p2p_is_xfs_realtime(file)) + return -EOPNOTSUPP; + break; + default: + return -EOPNOTSUPP; + } + bdev = inode->i_sb->s_bdev; + if (!bdev) + return -EOPNOTSUPP; + } else { + return -EINVAL; + } + + if (for_io && !(file->f_mode & FMODE_CAN_ODIRECT)) + return -EOPNOTSUPP; + /* + * A bvec iterator bypasses the block layer's own P2PDMA queue check. + * The flag is set on a PCIe NVMe namespace queue and on dm/md queues + * only when their supported targets and lower devices pass P2PDMA. + */ + if (!blk_queue_pci_p2pdma(bdev_get_queue(bdev))) + return -EOPNOTSUPP; + /* Protection information is generated and verified with the CPU. */ + if (blk_get_integrity(bdev->bd_disk)) + return -EOPNOTSUPP; + + *bdev_out = bdev; + return 0; +} + +/* Validate payload sizes, address bounds and target alignment in one pass. */ +static int p2p_validate_io_range(struct file *file, struct block_device *bdev, + const struct p2p_io_param *param, const struct p2p_iov *iov, + u64 *data_size) +{ + unsigned int lbs_mask = bdev_logical_block_size(bdev) - 1; + u64 offset = param->offset; + u64 size = 0; + unsigned int i; + + /* An append kiocb ignores the offset. */ + if (file->f_flags & O_APPEND || offset > S64_MAX || (offset & lbs_mask)) + return -EINVAL; + for (i = 0; i < param->iov_nr; i++) { + u64 end; + + if (!iov[i].size || iov[i].size > P2P_MAX_IOV_SIZE || iov[i].reserved || + ((iov[i].addr | iov[i].size) & lbs_mask) || + check_add_overflow((u64)iov[i].addr, (u64)iov[i].size, &end) || + check_add_overflow(size, (u64)iov[i].size, &size)) + return -EINVAL; + } + if (size > P2P_MAX_IO_BYTES || size > S64_MAX - offset) + return -EINVAL; + *data_size = size; + return 0; +} + +/* Check the actual iterator after contiguous PA segments have been merged. */ +static int p2p_check_bvec_align(const struct bio_vec *bvec, unsigned int nr, + u32 lbs) +{ + unsigned int i; + + for (i = 0; i < nr; i++) { + u64 addr = page_to_phys(bvec[i].bv_page) + bvec[i].bv_offset; + + if (addr % lbs || bvec[i].bv_len % lbs) + return -EINVAL; + } + return 0; +} + +/* + * Turn the PA segments of one request into a bvec array over the P2PDMA pages + * of the HBM window. Every PA is inside the window (p2p_check_pa_list()), so + * pfn_to_page() names a valid page. Adjacent segments that are physically + * contiguous share one multi-page bvec. + */ +static struct bio_vec *p2p_build_bvec(const struct p2p_pa_iov *pa_iov, unsigned int pa_iov_nr, + unsigned int *bvec_nr_out) +{ + struct bio_vec *bvec; + unsigned int n = 0; + unsigned int i; + u64 next_pa = 0; + + bvec = kvmalloc_array(pa_iov_nr, sizeof(*bvec), GFP_KERNEL); + if (!bvec) + return NULL; + + for (i = 0; i < pa_iov_nr; i++) { + u64 pa = pa_iov[i].addr; + u32 len = pa_iov[i].len; + + /* P2P_MAX_IO_BYTES caps the sum below UINT_MAX. */ + if (n && pa == next_pa) + bvec[n - 1].bv_len += len; + else + bvec_set_page(&bvec[n++], pfn_to_page(PHYS_PFN(pa)), len, + offset_in_page(pa)); + next_pa = pa + len; + } + + *bvec_nr_out = n; + return bvec; +} + +static int p2p_submit_one(struct p2p_batch *batch, const struct p2p_io_param *param) +{ + struct p2p_pa_iov *pa_iov = NULL; + struct bio_vec *bvec = NULL; + struct p2p_iov *iov; + struct file *file; + struct block_device *bdev; + struct p2p_io_context *io_ctx; + struct p2p_pinned_io_mem pinned_mem = { }; + u64 data_size; + unsigned int pa_iov_nr; + unsigned int bvec_nr; + int err; + + err = validate_io_param(param); + if (err) + return err; + + iov = kvmalloc_array(param->iov_nr, sizeof(*iov), GFP_KERNEL); + if (!iov) + return -ENOMEM; + if (copy_from_user(iov, u64_to_user_ptr(param->iov), + array_size(param->iov_nr, sizeof(*iov)))) { + err = -EFAULT; + goto free_iov; + } + + file = fget(param->file_fd); + if (!file) { + err = -EBADF; + goto free_iov; + } + if (!(file->f_mode & (param->op == P2P_IO_WRITE ? FMODE_WRITE : FMODE_READ))) { + err = -EBADF; + goto put_file; + } + err = p2p_check_target(file, &bdev, true); + if (err) + goto put_file; + err = p2p_validate_io_range(file, bdev, param, iov, &data_size); + if (err) + goto put_file; + + if (param->flags & P2P_IO_F_REGISTERED_MEM) + err = get_registered_pa_iov(batch, param->mem_handle, iov, param->iov_nr, + &pa_iov, &pa_iov_nr, &pinned_mem); + else + err = get_pa_iov(batch, param->host_pid, iov, param->iov_nr, + &pa_iov, &pa_iov_nr, &pinned_mem); + if (err) + goto put_file; + + bvec = p2p_build_bvec(pa_iov, pa_iov_nr, &bvec_nr); + if (!bvec) { + err = -ENOMEM; + goto unpin; + } + err = p2p_check_bvec_align(bvec, bvec_nr, + bdev_logical_block_size(bdev)); + if (err) + goto unpin; + + io_ctx = new_io_ctx(param->op, file, pa_iov, pa_iov_nr, bvec, bvec_nr, &pinned_mem, + data_size, param->user_data); + if (IS_ERR(io_ctx)) { + err = PTR_ERR(io_ctx); + goto unpin; + } + /* io_ctx owns the file, the pin and both arrays from here on. */ + kvfree(iov); + + io_ctx->batch = batch; + if (atomic_inc_return(&batch->io_cnt) > P2P_CQ_SIZE) { + if (atomic_dec_and_test(&batch->io_cnt)) + p2p_wake_batch_waiters(batch); + free_io_ctx(io_ctx); + return -EAGAIN; + } + atomic_inc(&batch->outstanding_events); + + /* The item is accepted: every result from here on is an event. */ + p2p_issue_io(io_ctx, param->offset); + p2p_io_ctx_put(io_ctx); + return 0; + +unpin: + p2p_unpin_io_mem(&pinned_mem); + kvfree(bvec); + kvfree(pa_iov); +put_file: + fput(file); +free_iov: + kvfree(iov); + return err; +} + +static int p2p_submit_io(struct p2p_batch *batch, void __user *arg) +{ + struct p2p_io_batch_param param; + struct p2p_io_param *items; + unsigned int accepted = 0; + unsigned int i; + int err; + + if (copy_from_user(¶m, arg, sizeof(param))) + return -EFAULT; + /* Every accepted item needs one slot in this fd's completion queue. */ + if (!param.nr || param.nr > P2P_MAX_IO_NR || param.reserved) + return -EINVAL; + + items = kvmalloc_array(param.nr, sizeof(*items), GFP_KERNEL); + if (!items) + return -ENOMEM; + if (copy_from_user(items, u64_to_user_ptr(param.items), + array_size(param.nr, sizeof(*items)))) { + err = -EFAULT; + goto out; + } + + for (i = 0; i < param.nr; i++) { + err = p2p_submit_one(batch, &items[i]); + if (err) + goto out; + accepted++; + } + err = accepted; +out: + kvfree(items); + return accepted ? accepted : err; +} + +/* + * Pop the CQ head slot and free its io_ctx. Caller holds ring_lock and must + * have finished delivering (or discarding) the slot's single logical event. + * Counts for remaining events were already adjusted by the caller. + */ +static void p2p_retire_cq_head(struct p2p_batch *batch) +{ + struct p2p_io_context *io_ctx; + + /* Pairs with the producer's smp_wmb() before cq_tail. */ + smp_rmb(); + io_ctx = batch->cq[batch->cq_head & P2P_CQ_MASK]; + WRITE_ONCE(batch->cq_head, batch->cq_head + 1); + + /* Publisher's complete() is its final field access before unpin. */ + wait_for_completion(&io_ctx->io_done); + /* Publish cq_head before returning its CQ slot to submitters. */ + smp_mb__before_atomic(); + atomic_dec(&batch->io_cnt); + free_io_ctx(io_ctx); +} + +/* + * Final close waits for each accepted I/O, then releases its CQ entry. + * No other fd operation can run here: the final file reference is gone. + */ +static void p2p_release_io(struct p2p_batch *batch) +{ + mutex_lock(&batch->ring_lock); + while (atomic_read(&batch->io_cnt)) { + /* + * Sleep with ring_lock held so getevents cannot interleave. + * Publish only takes completion_lock, so it can still wake us. + */ + wait_event(batch->wait, + batch->cq_head != READ_ONCE(batch->cq_tail) || + !atomic_read(&batch->io_cnt)); + + while (batch->cq_head != READ_ONCE(batch->cq_tail)) { + atomic_dec(&batch->ready_events); + atomic_dec(&batch->outstanding_events); + p2p_wake_batch_waiters(batch); + p2p_retire_cq_head(batch); + } + } + mutex_unlock(&batch->ring_lock); +} + +static void p2p_fill_io_event(struct p2p_io_context *io_ctx, struct p2p_io_event *out) +{ + out->user_data = io_ctx->user_data; + out->res = io_ctx->io_err; + out->reserved[0] = 0; + out->reserved[1] = 0; + out->reserved[2] = 0; + out->reserved[3] = 0; +} + +static bool p2p_cq_ready(struct p2p_batch *batch, unsigned int min_nr) +{ + unsigned int ready = atomic_read(&batch->ready_events); + unsigned int outstanding = atomic_read(&batch->outstanding_events); + + return ready >= min_nr || ready == outstanding; +} + +/* + * AIO-style getevents: wait on ready logical I/Os, harvest under the + * ring_lock mutex only, copy one event at a time (no kvmalloc intermediate). + * Each CQ slot is one logical I/O / one event. On copy_to_user fault the slot + * stays at head for the next call. + */ +static int p2p_get_io_events(struct p2p_batch *batch, void __user *arg) +{ + struct p2p_getevents_param param; + struct p2p_io_event __user *user_events; + struct p2p_io_context *io_ctx; + struct p2p_io_event event; + unsigned int total = 0; + int ret = 0; + + if (copy_from_user(¶m, arg, sizeof(param))) + return -EFAULT; + if (param.min_nr < 0 || param.max_nr <= 0 || + param.min_nr > param.max_nr || !param.events) + return -EINVAL; + + user_events = u64_to_user_ptr(param.events); + + if (param.timeout_ns) { + ktime_t until = param.timeout_ns < 0 ? KTIME_MAX : + ns_to_ktime(param.timeout_ns); + + ret = wait_event_interruptible_hrtimeout( + batch->wait, p2p_cq_ready(batch, param.min_nr), until); + if (ret == -ERESTARTSYS) + ret = -EINTR; + else if (ret == -ETIME) + ret = 0; + /* Fall through and harvest whatever is already on the CQ. */ + } + + mutex_lock(&batch->ring_lock); + while (total < (unsigned int)param.max_nr) { + if (batch->cq_head == READ_ONCE(batch->cq_tail)) + break; + /* Pairs with the producer's smp_wmb() before cq_tail. */ + smp_rmb(); + io_ctx = batch->cq[batch->cq_head & P2P_CQ_MASK]; + + p2p_fill_io_event(io_ctx, &event); + if (copy_to_user(&user_events[total], &event, sizeof(event))) { + ret = -EFAULT; + break; + } + total++; + p2p_retire_cq_head(batch); + } + + /* + * Single consumer under ring_lock: one atomic_sub for all successfully + * copied events instead of a per-event atomic_dec. + */ + if (total) { + atomic_sub(total, &batch->ready_events); + atomic_sub(total, &batch->outstanding_events); + p2p_wake_batch_waiters(batch); + } + mutex_unlock(&batch->ring_lock); + + if (total) + return (int)total; + return ret; +} + +static int p2p_check_fd(void __user *arg) +{ + struct p2p_check_fd_param param; + struct block_device *bdev; + struct file *file; + int err; + + if (copy_from_user(¶m, arg, sizeof(param))) + return -EFAULT; + if (param.flags || param.reserved) + return -EINVAL; + file = fget(param.fd); + if (!file) + return -EBADF; + err = p2p_check_target(file, &bdev, false); + fput(file); + return err; +} + +static int p2p_get_info(void __user *arg) +{ + const struct p2p_device_info info = { .version = P2P_UAPI_VERSION }; + + return copy_to_user(arg, &info, sizeof(info)) ? -EFAULT : 0; +} + +static long p2p_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct p2p_batch *batch = file->private_data; + + switch (cmd) { + case IOCTL_GET_INFO: + return p2p_get_info((void __user *)arg); + case IOCTL_CHECK_FD: + return p2p_check_fd((void __user *)arg); + case IOCTL_SUBMIT_IO: + return p2p_submit_io(batch, (void __user *)arg); + case IOCTL_REGISTER_MEM: + return p2p_register_mem(batch, (void __user *)arg); + case IOCTL_UNREGISTER_MEM: + return p2p_unregister_mem(batch, (void __user *)arg); + case IOCTL_GET_IO_EVENTS: + return p2p_get_io_events(batch, (void __user *)arg); + default: + return -ENOTTY; + } +} + +/* Root only by default; a udev rule can grant access to the XDS node. */ +static char *p2p_devnode(const struct device *device, umode_t *mode) +{ + if (mode) + *mode = 0600; + return NULL; +} + +static void p2p_destroy_devices(void) +{ + device_destroy(dev_class, dev); +} + +static int p2p_create_devices(void) +{ + struct device *device; + + device = device_create(dev_class, NULL, dev, NULL, "p2p_device"); + if (IS_ERR(device)) { + pr_err("create device err %ld\n", PTR_ERR(device)); + return PTR_ERR(device); + } + return 0; +} + +static int __init p2p_drv_init(void) +{ + int err; + + err = p2p_mem_init(); + if (err) + return err; + + err = p2p_providers_init(); + if (err) + goto exit_mem; + + p2p_debugfs_init(); + + err = alloc_chrdev_region(&dev, 0, 1, "p2p_device"); + if (err < 0) { + pr_err("allocate major number err %d\n", err); + goto exit_debugfs; + } + + cdev_init(&p2p_cdev, &fops); + err = cdev_add(&p2p_cdev, dev, 1); + if (err < 0) { + pr_err("add the device to the system err %d\n", err); + goto free_dev; + } + + dev_class = class_create("p2p_class"); + if (IS_ERR(dev_class)) { + err = PTR_ERR(dev_class); + pr_err("create class err %d\n", err); + goto del_cdev; + } + dev_class->devnode = p2p_devnode; + + err = p2p_create_devices(); + if (err) + goto del_cls; + + pr_info("driver inserted done, major %d\n", MAJOR(dev)); + return 0; + +del_cls: + class_destroy(dev_class); +del_cdev: + cdev_del(&p2p_cdev); +free_dev: + unregister_chrdev_region(dev, 1); +exit_debugfs: + p2p_debugfs_exit(); + p2p_providers_exit(); +exit_mem: + p2p_mem_exit(); + return err; +} + +static void __exit p2p_drv_exit(void) +{ + p2p_debugfs_exit(); + p2p_destroy_devices(); + class_destroy(dev_class); + cdev_del(&p2p_cdev); + unregister_chrdev_region(dev, 1); + WARN_ON_ONCE(!xa_empty(®istered_mems)); + xa_destroy(®istered_mems); + /* Drain the registered-memory RCU frees before the code goes away. */ + rcu_barrier(); + /* No fd is open any more: release the HBM pools and the PCI devices. */ + p2p_providers_exit(); + p2p_mem_exit(); + pr_info("driver removed done\n"); +} + +module_init(p2p_drv_init); +module_exit(p2p_drv_exit); +MODULE_DESCRIPTION("XDS NVMe peer-to-peer I/O driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/nds/mem.c b/drivers/nds/mem.c new file mode 100644 index 000000000000..f34f15d02ca4 --- /dev/null +++ b/drivers/nds/mem.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +#define pr_fmt(fmt) "p2p: " fmt + +#include <linux/errno.h> +#include <linux/module.h> + +#include "mem.h" + +static typeof(devmm_get_mem_pa_list) *get_mem_pa_list; +static typeof(devmm_put_mem_pa_list) *put_mem_pa_list; +static typeof(devmm_get_mem_page_size) *get_mem_page_size; + +int p2p_mem_init(void) +{ + get_mem_pa_list = symbol_get(devmm_get_mem_pa_list); + if (!get_mem_pa_list) { + pr_err("required symbol devmm_get_mem_pa_list is unavailable\n"); + return -ENODEV; + } + + put_mem_pa_list = symbol_get(devmm_put_mem_pa_list); + if (!put_mem_pa_list) { + pr_err("required symbol devmm_put_mem_pa_list is unavailable\n"); + goto put_get_mem_pa_list; + } + + get_mem_page_size = symbol_get(devmm_get_mem_page_size); + if (!get_mem_page_size) { + pr_err("required symbol devmm_get_mem_page_size is unavailable\n"); + goto put_put_mem_pa_list; + } + + return 0; + +put_put_mem_pa_list: + symbol_put(devmm_put_mem_pa_list); +put_get_mem_pa_list: + symbol_put(devmm_get_mem_pa_list); + return -ENODEV; +} + +int p2p_mem_get_pa_list(struct devmm_svm_process_id *process_id, u64 addr, + u64 size, u64 *pa_list, u32 pa_num) +{ + return get_mem_pa_list(process_id, addr, size, pa_list, pa_num); +} + +void p2p_mem_put_pa_list(struct devmm_svm_process_id *process_id, u64 addr, + u64 size, u64 *pa_list, u32 pa_num) +{ + put_mem_pa_list(process_id, addr, size, pa_list, pa_num); +} + +int p2p_mem_get_page_size(struct devmm_svm_process_id *process_id, u64 addr, + u64 size) +{ + return get_mem_page_size(process_id, addr, size); +} + +void p2p_mem_exit(void) +{ + symbol_put(devmm_get_mem_page_size); + symbol_put(devmm_put_mem_pa_list); + symbol_put(devmm_get_mem_pa_list); +} diff --git a/drivers/nds/mem.h b/drivers/nds/mem.h new file mode 100644 index 000000000000..617bf44a1346 --- /dev/null +++ b/drivers/nds/mem.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef P2P_MEM_H_ +#define P2P_MEM_H_ + +#include "mem_abi.h" + +int p2p_mem_get_pa_list(struct devmm_svm_process_id *process_id, u64 addr, + u64 size, u64 *pa_list, u32 pa_num); +void p2p_mem_put_pa_list(struct devmm_svm_process_id *process_id, u64 addr, + u64 size, u64 *pa_list, u32 pa_num); +int p2p_mem_get_page_size(struct devmm_svm_process_id *process_id, u64 addr, + u64 size); + +int p2p_mem_init(void); +void p2p_mem_exit(void); + +#endif diff --git a/drivers/nds/mem_abi.h b/drivers/nds/mem_abi.h new file mode 100644 index 000000000000..10c5a515c64c --- /dev/null +++ b/drivers/nds/mem_abi.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef P2P_MEM_ABI_H_ +#define P2P_MEM_ABI_H_ + +#include <linux/types.h> + +struct devmm_svm_process_id { + int32_t host_pid; + union { + uint16_t devid; + uint16_t vm_id; + }; + uint16_t vfid; +}; + +int devmm_get_mem_pa_list(struct devmm_svm_process_id *process_id, u64 addr, + u64 size, u64 *pa_list, u32 pa_num); +void devmm_put_mem_pa_list(struct devmm_svm_process_id *process_id, u64 addr, + u64 size, u64 *pa_list, u32 pa_num); +int devmm_get_mem_page_size(struct devmm_svm_process_id *process_id, u64 addr, + u64 size); + +#endif diff --git a/drivers/nds/provider.c b/drivers/nds/provider.c new file mode 100644 index 000000000000..b110ba3dbd9a --- /dev/null +++ b/drivers/nds/provider.c @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: GPL-2.0 +#define pr_fmt(fmt) "p2p: " fmt + +#include <linux/kernel.h> +#include <linux/memremap.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/pci.h> +#include <linux/pci-p2pdma.h> +#include <linux/pfn.h> +#include <linux/ratelimit.h> +#include <linux/sort.h> + +#include "provider.h" + +/* + * The PCI BAR that holds HBM. BAR 4 on the PCI device; the test stub backs + * "HBM" with an NVMe controller memory buffer, which QEMU puts in BAR 2. + */ +static int hbm_bar = 4; +module_param(hbm_bar, int, 0444); +MODULE_PARM_DESC(hbm_bar, "PCI BAR of the PCI device that holds HBM (default 4)"); + +static unsigned long pagemap_align; +module_param(pagemap_align, ulong, 0444); +MODULE_PARM_DESC(pagemap_align, "PCI HBM pagemap alignment: 0=auto, 2M or 1G"); + +static struct p2p_provider providers[P2P_MAX_DEVS]; +static unsigned int nr_providers; +static DEFINE_MUTEX(provider_lock); + +static int p2p_provider_cmp(const void *a, const void *b) +{ + const struct p2p_provider *pa = a; + const struct p2p_provider *pb = b; + + if (pa->start < pb->start) + return -1; + return pa->start > pb->start; +} + +struct p2p_provider *p2p_provider_find(u64 pa, u64 len) +{ + unsigned int lo = 0; + unsigned int hi = nr_providers; + + while (lo < hi) { + unsigned int mid = lo + (hi - lo) / 2; + struct p2p_provider *provider = &providers[mid]; + + if (pa < provider->start) + hi = mid; + else if (pa >= provider->end) + lo = mid + 1; + else + return p2p_provider_contains(provider, pa, len) ? provider : NULL; + } + return NULL; +} + +static const struct pci_device_id p2p_npu_device_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xd802) }, + { } +}; + +/* QEMU NVMe CMB used by stub.ko; it has no memory BAR 4. */ +static const struct pci_device_id p2p_stub_device_ids[] = { + { PCI_DEVICE(0x1b36, 0x0010) }, + { } +}; + +static bool p2p_provider_can_dedup(const struct p2p_provider *provider) +{ + return hbm_bar == 4 && pci_match_id(p2p_npu_device_ids, provider->pdev); +} + +static int p2p_provider_register_pages(struct p2p_provider *provider) +{ +#ifdef CONFIG_ARCH_WANT_OPTIMIZE_P2PDMA_VMEMMAP + size_t size, align; + + if (!p2p_provider_can_dedup(provider)) + return pci_p2pdma_add_resource(provider->pdev, hbm_bar, 0, 0); + + size = pci_resource_len(provider->pdev, hbm_bar); + align = pci_p2pdma_max_pagemap_align(provider->pdev, hbm_bar, size, 0); + if (pagemap_align) { + if (align < pagemap_align) { + pci_err(provider->pdev, + "requested pagemap alignment %#lx exceeds maximum %#zx\n", + pagemap_align, align); + return -EOPNOTSUPP; + } + align = pagemap_align; + } + if (align == PAGE_SIZE) + return pci_p2pdma_add_resource(provider->pdev, hbm_bar, 0, 0); + + return pci_p2pdma_add_resource_aligned(provider->pdev, hbm_bar, 0, align, 0); +#else + if (pagemap_align) { + pci_err(provider->pdev, + "pagemap alignment requires P2PDMA vmemmap support\n"); + return -EOPNOTSUPP; + } + return pci_p2pdma_add_resource(provider->pdev, hbm_bar, 0, 0); +#endif +} + +static void p2p_provider_remove_pages(struct p2p_provider *provider) +{ + if (!provider->page_resources) + return; + /* + * Run the devm_memremap_pages() unmap action, then free the P2PDMA + * pool and its metadata. Unmapping only the pages would leave a stale + * pool entry. Reused resources are outside this group. + */ + devres_release_group(&provider->pdev->dev, provider->page_resources); + provider->page_resources = NULL; +} + +/* Give the window struct pages, or reuse an exact existing P2PDMA mapping. */ +static int p2p_provider_add_pages(struct p2p_provider *provider) +{ + struct dev_pagemap *pgmap; + int err = 0; + + pgmap = get_dev_pagemap(PHYS_PFN(provider->start), NULL); + if (!pgmap) { + /* + * A new range in an existing pool cannot be removed separately: + * the PCI API has no operation to remove a gen_pool range. + * Only create a pool whose full lifetime we can manage. + */ + if (rcu_access_pointer(provider->pdev->p2pdma)) { + pci_err(provider->pdev, "cannot add HBM to an existing P2PDMA pool\n"); + return -EBUSY; + } + provider->page_resources = devres_open_group(&provider->pdev->dev, NULL, + GFP_KERNEL); + if (!provider->page_resources) + return -ENOMEM; + err = p2p_provider_register_pages(provider); + devres_close_group(&provider->pdev->dev, provider->page_resources); + if (err) + p2p_provider_remove_pages(provider); + return err; + } + + if (pgmap->type != MEMORY_DEVICE_PCI_P2PDMA || + pgmap->range.start != provider->start || + pgmap->range.end != provider->end - 1) { + pci_err(provider->pdev, "BAR %d is covered by a foreign pagemap\n", hbm_bar); + err = -EBUSY; + } else if (p2p_provider_can_dedup(provider) && pagemap_align && + pgmap->vmemmap_shift != ilog2(pagemap_align) - PAGE_SHIFT) { + pci_err(provider->pdev, + "HBM pagemap shift %lu does not match requested alignment %#lx\n", + pgmap->vmemmap_shift, pagemap_align); + err = -EBUSY; + } else { + pci_info(provider->pdev, "reusing HBM pagemap with vmemmap shift %lu\n", + pgmap->vmemmap_shift); + } + put_dev_pagemap(pgmap); + return err; +} + +int p2p_provider_prepare(struct p2p_provider *provider) +{ + size_t size = provider->end - provider->start; + int err = 0; + + /* Acquire the page mapping and pool published by the release store. */ + if (smp_load_acquire(&provider->ready)) + return 0; + + mutex_lock(&provider_lock); + if (provider->ready) + goto unlock; + + /* Keep PCI driver probe/unbind from changing devres during setup. */ + device_lock(&provider->pdev->dev); + err = p2p_provider_add_pages(provider); + if (err) + goto unlock_device; + + /* + * Take the whole pool, so nobody else gets HBM from it (sysfs + * p2pmem/allocate, in-kernel P2PDMA clients). The allocation also holds + * a pagemap reference: an early unbind of the PCI device blocks until + * module exit instead of freeing struct pages under in-flight I/O. + */ + provider->pool_addr = pci_alloc_p2pmem(provider->pdev, size); + if (!provider->pool_addr) { + pci_err(provider->pdev, "HBM p2pmem pool is in use\n"); + err = -EBUSY; + p2p_provider_remove_pages(provider); + goto unlock_device; + } + + /* Publish the page mapping and reserved pool to concurrent callers. */ + smp_store_release(&provider->ready, true); + pci_info(provider->pdev, "HBM window [0x%llx, 0x%llx) is ready\n", + provider->start, provider->end); +unlock_device: + device_unlock(&provider->pdev->dev); +unlock: + mutex_unlock(&provider_lock); + return err; +} + +int p2p_providers_init(void) +{ + struct pci_dev *pdev = NULL; + unsigned int found = 0; + + if (hbm_bar < 0 || hbm_bar >= PCI_STD_NUM_BARS) { + pr_err("invalid hbm_bar %d\n", hbm_bar); + return -EINVAL; + } + if (pagemap_align && pagemap_align != PMD_SIZE && pagemap_align != PUD_SIZE) { + pr_err("invalid pagemap_align %#lx; use 0, %#lx or %#lx\n", pagemap_align, + (unsigned long)PMD_SIZE, (unsigned long)PUD_SIZE); + return -EINVAL; + } +#ifndef CONFIG_ARCH_WANT_OPTIMIZE_P2PDMA_VMEMMAP + if (pagemap_align) { + pr_err("pagemap_align requires P2PDMA vmemmap support\n"); + return -EOPNOTSUPP; + } +#endif + + for_each_pci_dev(pdev) { + struct p2p_provider *provider; + + if (!pci_match_id(p2p_npu_device_ids, pdev) && + !pci_match_id(p2p_stub_device_ids, pdev)) + continue; + if (!(pci_resource_flags(pdev, hbm_bar) & IORESOURCE_MEM) || + !pci_resource_len(pdev, hbm_bar)) { + pci_warn(pdev, "no memory BAR %d, skipped\n", hbm_bar); + continue; + } + if (found == P2P_MAX_DEVS) { + pci_warn(pdev, "provider limit %u reached, skipped\n", P2P_MAX_DEVS); + continue; + } + + provider = &providers[found++]; + /* Hold a reference after for_each_pci_dev() drops its own. */ + provider->pdev = pci_dev_get(pdev); + provider->start = pci_resource_start(pdev, hbm_bar); + provider->end = provider->start + pci_resource_len(pdev, hbm_bar); + } + + sort(providers, found, sizeof(*providers), p2p_provider_cmp, NULL); + nr_providers = found; + pr_info("found %u PCI P2PDMA provider(s), HBM in BAR %d\n", found, hbm_bar); + return 0; +} + +void p2p_providers_exit(void) +{ + unsigned int i; + + for (i = 0; i < nr_providers; i++) { + struct p2p_provider *provider = &providers[i]; + + if (provider->ready) + pci_free_p2pmem(provider->pdev, provider->pool_addr, + provider->end - provider->start); + /* Return the pool before unmapping its pages. */ + p2p_provider_remove_pages(provider); + pci_dev_put(provider->pdev); + *provider = (struct p2p_provider) { }; + } + nr_providers = 0; +} diff --git a/drivers/nds/provider.h b/drivers/nds/provider.h new file mode 100644 index 000000000000..40a62748364c --- /dev/null +++ b/drivers/nds/provider.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef P2P_PROVIDER_H_ +#define P2P_PROVIDER_H_ + +#include <linux/types.h> + +struct pci_dev; + +/* One provider: the PCI device whose HBM BAR is a PCI P2PDMA provider. */ +struct p2p_provider { + struct pci_dev *pdev; + /* HBM window [start, end) in CPU physical address space. */ + u64 start; + u64 end; + /* Set once the window has struct pages and its pool is reserved. */ + bool ready; + void *pool_addr; + /* Owns new P2PDMA resources; NULL for reused mappings. */ + void *page_resources; +}; + +#define P2P_MAX_DEVS 32U + +int p2p_providers_init(void); +void p2p_providers_exit(void); + +/* Find the provider whose HBM window contains all of [@pa, @pa + @len). */ +struct p2p_provider *p2p_provider_find(u64 pa, u64 len); + +/* Make the HBM window a P2PDMA provider. Idempotent; may sleep. */ +int p2p_provider_prepare(struct p2p_provider *provider); + +static inline bool p2p_provider_contains(const struct p2p_provider *provider, + u64 pa, u64 len) +{ + return pa >= provider->start && pa < provider->end && + len <= provider->end - pa; +} + +#endif diff --git a/include/uapi/linux/nds_p2p.h b/include/uapi/linux/nds_p2p.h new file mode 100644 index 000000000000..f69bd610a41f --- /dev/null +++ b/include/uapi/linux/nds_p2p.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef NDS_P2P_H_ +#define NDS_P2P_H_ + +#include <linux/types.h> + +/* + * UAPI v2: PCI P2PDMA + asynchronous VFS I/O. + * + * All clients open /dev/p2p_device. An fd learns its provider from the first + * registered or one-shot buffer. Later buffers on that fd must use the same + * provider. No request or device-node name carries a PCI device id. + */ +/* GET_INFO uses bits 15:8 for major and bits 7:0 for minor. */ +#define P2P_UAPI_VERSION_MAJOR 2U +#define P2P_UAPI_VERSION_MINOR 0U +#define P2P_UAPI_VERSION ((P2P_UAPI_VERSION_MAJOR << 8) | P2P_UAPI_VERSION_MINOR) + +struct p2p_iov { + __u64 addr; /* HBM addr */ + __u32 size; + __u32 reserved; /* must be zero */ +}; + +#define P2P_IO_F_REGISTERED_MEM (1U << 0) +#define P2P_IO_F_MASK P2P_IO_F_REGISTERED_MEM + +#define P2P_IO_READ 0U +#define P2P_IO_WRITE 1U + +/* One logical I/O moves at most this many bytes (the kernel MAX_RW_COUNT). */ +#define P2P_MAX_IO_BYTES 0x7ffff000U + +/* + * Fixed-size request; iov is a userspace pointer. One batch item is one + * logical I/O at byte offset @offset in @file_fd, covering all iov segments + * and completing as one event carrying user_data. A transfer shorter than the + * iov total completes with -EIO. + */ +struct p2p_io_param { + unsigned int op; /* P2P_IO_READ / P2P_IO_WRITE */ + unsigned int flags; /* P2P_IO_F_* */ + int file_fd; /* regular file or block device; no O_APPEND */ + int host_pid; /* one-shot buffer owner; 0 if registered */ + __u64 mem_handle; /* from IOCTL_REGISTER_MEM, or 0 */ + __u64 user_data; /* echo'd in the completion event */ + __u64 offset; /* byte offset in file_fd */ + __u64 iov; /* userspace pointer to struct p2p_iov[] */ + unsigned int iov_nr; + unsigned int reserved; /* must be zero */ + __u64 reserved2[3]; /* must be zero */ +}; + +#define P2P_MAX_IO_NR 1024U + +/* + * Submit independent p2p_io_param records in array order. The ioctl stops at + * the first item that cannot be accepted and returns the accepted prefix, or + * a negative errno when the first item fails. Each accepted item produces one + * completion event. The ioctl may sleep in the filesystem. + */ +struct p2p_io_batch_param { + __u64 items; /* userspace pointer to p2p_io_param[] */ + __u32 nr; + __u32 reserved; +}; + +struct p2p_io_event { + __u64 user_data; + __s64 res; /* 0 on success or -errno */ + __u64 reserved[4]; +}; + +struct p2p_getevents_param { + __s32 min_nr; + __s32 max_nr; + __s64 timeout_ns; /* <0 infinite, 0 nonblock, >0 relative */ + __u64 events; /* userspace pointer to struct p2p_io_event[] */ +}; + +struct p2p_mem_unregister_param { + __u64 mem_handle; + __u64 reserved[3]; +}; + +struct p2p_mem_range { + __u64 addr; + __u64 size; +}; + +#define P2P_MEM_MAX_RANGES 8192U + +/* + * Register several non-overlapping VA ranges under one handle. Registered I/O + * with that handle may address any registered range. Each IOV must fit in one + * translation segment; crossing a segment boundary fails with -ERANGE. Adjacent + * ranges merge only at a page-aligned boundary with the same page size. Every + * range must be in one provider's window and match the fd's cached provider. + * A different provider fails with -EXDEV; an address outside every provider + * fails with -EFAULT. + * IOCTL_UNREGISTER_MEM releases the registration. + */ +struct p2p_mem_register_param { + __u64 ranges; /* userspace pointer to struct p2p_mem_range[] */ + __u32 nr; /* 1..P2P_MEM_MAX_RANGES */ + __u32 flags; /* must be zero */ + __u64 reserved[3]; /* must be zero */ + __u64 mem_handle; /* output */ +}; + +/* Validate a file, directory or block-device fd's filesystem and P2PDMA queue. */ +struct p2p_check_fd_param { + __s32 fd; + __u32 flags; /* must be zero */ + __u64 reserved; /* must be zero */ +}; + +/* Output only. Feature flags and reserved fields are zero for version 2.0. */ +struct p2p_device_info { + __u32 version; /* major in bits 15:8, minor in bits 7:0; upper bits zero */ + __u32 flags; + __u64 reserved[3]; +}; + +#define IOCTL_GET_INFO _IOR('k', 1, struct p2p_device_info) +#define IOCTL_CHECK_FD _IOW('k', 2, struct p2p_check_fd_param) +#define IOCTL_REGISTER_MEM _IOWR('k', 3, struct p2p_mem_register_param) +#define IOCTL_UNREGISTER_MEM _IOW('k', 4, struct p2p_mem_unregister_param) +#define IOCTL_SUBMIT_IO _IOWR('k', 5, struct p2p_io_batch_param) +#define IOCTL_GET_IO_EVENTS _IOWR('k', 6, struct p2p_getevents_param) + +#endif -- 2.52.0