From: hotforest <hotforest@gmail.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 -------------------------------- Add pci_p2pdma_add_resource_aligned() to register P2PDMA memory with PMD- or PUD-sized compound pages when the resource geometry and architecture support vmemmap deduplication. Add pci_p2pdma_max_pagemap_align() to select the largest supported alignment. Keep pci_p2pdma_add_resource() as a PAGE_SIZE wrapper so existing providers retain their base-page behavior. Use the same alignment for the gen_pool allocation granularity and page free path. Reject mappings through p2pmem/allocate for compound-page providers because these pages are restricted to kernel DMA use. Reject a second registration that uses a different alignment on the same device. Signed-off-by: hotforest <hotforest@gmail.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- drivers/pci/p2pdma.c | 107 ++++++++++++++++++++++++++++++++----- include/linux/pci-p2pdma.h | 21 +++++++- 2 files changed, 112 insertions(+), 16 deletions(-) diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 5522377f671d..b057a82bd547 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -9,10 +9,11 @@ */ #define pr_fmt(fmt) "pci-p2pdma: " fmt #include <linux/ctype.h> #include <linux/dma-map-ops.h> +#include <linux/mm.h> #include <linux/pci-p2pdma.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/genalloc.h> #include <linux/memremap.h> @@ -22,10 +23,11 @@ #include <linux/xarray.h> struct pci_p2pdma { struct gen_pool *pool; bool p2pmem_published; + size_t align; struct xarray map_types; }; struct pci_p2pdma_pagemap { struct dev_pagemap pgmap; @@ -119,10 +121,14 @@ static int p2pmem_alloc_mmap(struct file *filp, struct kobject *kobj, p2pdma = rcu_dereference(pdev->p2pdma); if (!p2pdma) { ret = -ENODEV; goto out; } + if (p2pdma->align != PAGE_SIZE) { + ret = -EOPNOTSUPP; + goto out; + } kaddr = (void *)gen_pool_alloc_owner(p2pdma->pool, len, (void **)&ref); if (!kaddr) { ret = -ENOMEM; goto out; @@ -194,16 +200,16 @@ static const struct attribute_group p2pmem_group = { static void p2pdma_page_free(struct page *page) { struct pci_p2pdma_pagemap *pgmap = to_p2p_pgmap(page_pgmap(page)); /* safe to dereference while a reference is held to the percpu ref */ - struct pci_p2pdma *p2pdma = - rcu_dereference_protected(pgmap->provider->p2pdma, 1); + struct pci_p2pdma *p2pdma; struct percpu_ref *ref; + p2pdma = rcu_dereference_protected(pgmap->provider->p2pdma, 1); gen_pool_free_owner(p2pdma->pool, (uintptr_t)page_to_virt(page), - PAGE_SIZE, (void **)&ref); + p2pdma->align, (void **)&ref); percpu_ref_put(ref); } static const struct dev_pagemap_ops p2pdma_pgmap_ops = { .page_free = p2pdma_page_free, @@ -225,22 +231,23 @@ static void pci_p2pdma_release(void *data) gen_pool_destroy(p2pdma->pool); sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group); xa_destroy(&p2pdma->map_types); } -static int pci_p2pdma_setup(struct pci_dev *pdev) +static int pci_p2pdma_setup(struct pci_dev *pdev, size_t align) { int error = -ENOMEM; struct pci_p2pdma *p2p; p2p = devm_kzalloc(&pdev->dev, sizeof(*p2p), GFP_KERNEL); if (!p2p) return -ENOMEM; xa_init(&p2p->map_types); - p2p->pool = gen_pool_create(PAGE_SHIFT, dev_to_node(&pdev->dev)); + p2p->align = align; + p2p->pool = gen_pool_create(ilog2(align), dev_to_node(&pdev->dev)); if (!p2p->pool) goto out; error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev); if (error) @@ -271,22 +278,71 @@ static void pci_p2pdma_unmap_mappings(void *data) */ sysfs_remove_file_from_group(&pdev->dev.kobj, &p2pmem_alloc_attr.attr, p2pmem_group.name); } +static bool pci_p2pdma_vmemmap_can_optimize(size_t align) +{ + struct dev_pagemap pgmap = { }; + + pgmap.vmemmap_shift = ilog2(align) - PAGE_SHIFT; + return vmemmap_can_optimize(NULL, &pgmap); +} + /** - * pci_p2pdma_add_resource - add memory for use as p2p memory + * pci_p2pdma_max_pagemap_align - find the largest deduplicated page geometry + * @pdev: the provider device + * @bar: PCI BAR to add + * @size: effective size of the resource, not zero + * @offset: offset into the PCI BAR + * + * Return PAGE_SIZE when the architecture cannot deduplicate vmemmap pages or + * when the resource has no supported PUD or PMD alignment. + */ +size_t pci_p2pdma_max_pagemap_align(struct pci_dev *pdev, int bar, + size_t size, u64 offset) +{ + resource_size_t start = pci_resource_start(pdev, bar); + + if (IS_ALIGNED(start | size | offset, PUD_SIZE) && + pci_p2pdma_vmemmap_can_optimize(PUD_SIZE)) + return PUD_SIZE; + if (IS_ALIGNED(start | size | offset, PMD_SIZE) && + pci_p2pdma_vmemmap_can_optimize(PMD_SIZE)) + return PMD_SIZE; + return PAGE_SIZE; +} +EXPORT_SYMBOL_GPL(pci_p2pdma_max_pagemap_align); + +static int pci_p2pdma_check_pagemap_align(struct pci_dev *pdev, int bar, + size_t size, size_t align, u64 offset) +{ + if (align == PAGE_SIZE) + return 0; + if (align != PUD_SIZE && align != PMD_SIZE) + return -EINVAL; + if (!IS_ALIGNED(pci_resource_start(pdev, bar) | size | offset, align)) + return -EINVAL; + if (!pci_p2pdma_vmemmap_can_optimize(align)) + return -EOPNOTSUPP; + return 0; +} + +/** + * pci_p2pdma_add_resource_aligned - add memory with compound struct pages * @pdev: the device to add the memory to * @bar: PCI BAR to add * @size: size of the memory to add, may be zero to use the whole BAR + * @align: allocation and compound-page alignment * @offset: offset into the PCI BAR * - * The memory will be given ZONE_DEVICE struct pages so that it may - * be used with any DMA request. + * The memory will be given ZONE_DEVICE struct pages so that it may be used + * with kernel DMA requests. Compound-page resources cannot be mapped through + * the p2pmem allocate sysfs file. */ -int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, - u64 offset) +int pci_p2pdma_add_resource_aligned(struct pci_dev *pdev, int bar, size_t size, + size_t align, u64 offset) { struct pci_p2pdma_pagemap *p2p_pgmap; struct dev_pagemap *pgmap; struct pci_p2pdma *p2pdma; void *addr; @@ -299,15 +355,22 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, return -EINVAL; if (!size) size = pci_resource_len(pdev, bar) - offset; - if (size + offset > pci_resource_len(pdev, bar)) + if (size > pci_resource_len(pdev, bar) - offset) + return -EINVAL; + + error = pci_p2pdma_check_pagemap_align(pdev, bar, size, align, offset); + if (error) + return error; + + if (pdev->p2pdma && pdev->p2pdma->align != align) return -EINVAL; if (!pdev->p2pdma) { - error = pci_p2pdma_setup(pdev); + error = pci_p2pdma_setup(pdev, align); if (error) return error; } p2p_pgmap = devm_kzalloc(&pdev->dev, sizeof(*p2p_pgmap), GFP_KERNEL); @@ -318,10 +381,12 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, pgmap->range.start = pci_resource_start(pdev, bar) + offset; pgmap->range.end = pgmap->range.start + size - 1; pgmap->nr_range = 1; pgmap->type = MEMORY_DEVICE_PCI_P2PDMA; pgmap->ops = &p2pdma_pgmap_ops; + if (align > PAGE_SIZE) + pgmap->vmemmap_shift = ilog2(align) - PAGE_SHIFT; p2p_pgmap->provider = pdev; p2p_pgmap->bus_offset = pci_bus_address(pdev, bar) - pci_resource_start(pdev, bar); @@ -342,21 +407,35 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, range_len(&pgmap->range), dev_to_node(&pdev->dev), &pgmap->ref); if (error) goto pages_free; - pci_info(pdev, "added peer-to-peer DMA memory %#llx-%#llx\n", - pgmap->range.start, pgmap->range.end); + pci_info(pdev, "added peer-to-peer DMA memory %#llx-%#llx alignment %#lx\n", + pgmap->range.start, pgmap->range.end, (unsigned long)align); return 0; pages_free: devm_memunmap_pages(&pdev->dev, pgmap); pgmap_free: devm_kfree(&pdev->dev, p2p_pgmap); return error; } +EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource_aligned); + +/** + * pci_p2pdma_add_resource - add memory with base struct pages + * @pdev: the device to add the memory to + * @bar: PCI BAR to add + * @size: size of the memory to add, may be zero to use the whole BAR + * @offset: offset into the PCI BAR + */ +int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, + u64 offset) +{ + return pci_p2pdma_add_resource_aligned(pdev, bar, size, PAGE_SIZE, offset); +} EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource); /* * Note this function returns the parent PCI device with a * reference taken. It is the caller's responsibility to drop diff --git a/include/linux/pci-p2pdma.h b/include/linux/pci-p2pdma.h index 2c07aa6b7665..183cd650320d 100644 --- a/include/linux/pci-p2pdma.h +++ b/include/linux/pci-p2pdma.h @@ -16,11 +16,15 @@ struct block_device; struct scatterlist; #ifdef CONFIG_PCI_P2PDMA int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, - u64 offset); + u64 offset); +int pci_p2pdma_add_resource_aligned(struct pci_dev *pdev, int bar, + size_t size, size_t align, u64 offset); +size_t pci_p2pdma_max_pagemap_align(struct pci_dev *pdev, int bar, + size_t size, u64 offset); int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients, int num_clients, bool verbose); bool pci_has_p2pmem(struct pci_dev *pdev); struct pci_dev *pci_p2pmem_find_many(struct device **clients, int num_clients); void *pci_alloc_p2pmem(struct pci_dev *pdev, size_t size); @@ -34,14 +38,27 @@ int pci_p2pdma_enable_store(const char *page, struct pci_dev **p2p_dev, bool *use_p2pdma); ssize_t pci_p2pdma_enable_show(char *page, struct pci_dev *p2p_dev, bool use_p2pdma); #else /* CONFIG_PCI_P2PDMA */ static inline int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, - size_t size, u64 offset) + size_t size, u64 offset) { return -EOPNOTSUPP; } + +static inline int pci_p2pdma_add_resource_aligned(struct pci_dev *pdev, int bar, + size_t size, size_t align, + u64 offset) +{ + return -EOPNOTSUPP; +} + +static inline size_t pci_p2pdma_max_pagemap_align(struct pci_dev *pdev, int bar, + size_t size, u64 offset) +{ + return PAGE_SIZE; +} static inline int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients, int num_clients, bool verbose) { return -1; } -- 2.52.0