[PATCH OLK-6.6 0/7] P2PDMA PATCHSET PART 2
Add some p2pdma api and dedup of vmemmap. hotforest (7): PCI/P2PDMA: whitelist HiSilicon root ports PCI/P2PDMA: disable CPU access through bvec iterators mm: allow compound zone device pages mm: add P2PDMA vmemmap optimization capability PCI/P2PDMA: add aligned resource registration x86: enable P2PDMA vmemmap optimization arm64: enable P2PDMA vmemmap optimization arch/arm64/Kconfig | 1 + arch/x86/Kconfig | 1 + drivers/pci/p2pdma.c | 115 +++++++++++++++++++++++++++++++------ include/linux/memremap.h | 6 +- include/linux/mm.h | 7 ++- include/linux/mmzone.h | 13 ++++- include/linux/pci-p2pdma.h | 21 ++++++- lib/iov_iter.c | 36 ++++++++++-- mm/Kconfig | 7 ++- mm/memremap.c | 2 +- 10 files changed, 176 insertions(+), 33 deletions(-) -- 2.52.0
From: hotforest <hotforest@gmail.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 -------------------------------- Allow peer-to-peer DMA through HiSilicon root ports (19e5:a120), including ports on different host bridges. A whitelist entry without REQ_SAME_HOST_BRIDGE gives unconditional cross-root-complex behavior; no boot parameter or runtime switch is needed. Signed-off-by: hotforest <hotforest@gmail.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- drivers/pci/p2pdma.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 5139c205287c..b807f3fa221d 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -447,10 +447,12 @@ static const struct pci_p2pdma_whitelist_entry { {PCI_VENDOR_ID_ZHAOXIN, 0x1003, REQ_SAME_HOST_BRIDGE}, {PCI_VENDOR_ID_ZHAOXIN, 0x1005, REQ_SAME_HOST_BRIDGE}, {PCI_VENDOR_ID_ZHAOXIN, 0x1006, REQ_SAME_HOST_BRIDGE}, {PCI_VENDOR_ID_ZHAOXIN, 0x1007, REQ_SAME_HOST_BRIDGE}, {PCI_VENDOR_ID_ZHAOXIN, 0x1008, 0}, + /* HiSilicon Kunpeng 920 root ports */ + {PCI_VENDOR_ID_HUAWEI, 0xa120, 0}, {} }; /* * If the first device on host's root bus is either devfn 00.0 or a PCIe -- 2.52.0
From: hotforest <hotforest@gmail.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 -------------------------------- Stop all bvec copy and zero operations before they map a PCI P2PDMA page for CPU access. Return a short count so callers fail the operation instead of reading, writing, or zeroing device memory with the CPU. Make fault_in_iov_iter_readable() and fault_in_iov_iter_writeable() report a P2PDMA bvec as not faultable. This prevents buffered write loops from retrying a zero-length copy forever. The check applies to every PCI P2PDMA provider. No provider opt-in, static key, or runtime switch is used. Signed-off-by: hotforest <hotforest@gmail.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- lib/iov_iter.c | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/lib/iov_iter.c b/lib/iov_iter.c index a4cc8d68b409..d461b56c41f9 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -11,10 +11,11 @@ #include <linux/splice.h> #include <linux/compat.h> #include <net/checksum.h> #include <linux/scatterlist.h> #include <linux/instrumented.h> +#include <linux/memremap.h> /* covers ubuf and kbuf alike */ #define iterate_buf(i, n, base, len, off, __p, STEP) { \ size_t __maybe_unused off = 0; \ len = n; \ @@ -50,12 +51,16 @@ size_t off = 0; \ unsigned skip = i->iov_offset; \ while (n) { \ unsigned offset = p->bv_offset + skip; \ unsigned left; \ - void *kaddr = kmap_local_page(p->bv_page + \ - offset / PAGE_SIZE); \ + struct page *bv_pg = p->bv_page + \ + offset / PAGE_SIZE; \ + void *kaddr; \ + if (is_pci_p2pdma_page(bv_pg)) \ + break; \ + kaddr = kmap_local_page(bv_pg); \ base = kaddr + offset % PAGE_SIZE; \ len = min(min(n, (size_t)(p->bv_len - skip)), \ (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \ left = (STEP); \ kunmap_local(kaddr); \ @@ -194,10 +199,27 @@ static int copyin(void *to, const void __user *from, size_t n) instrument_copy_from_user_after(to, from, n, res); } return res; } +/* + * The copy and zero helpers make no progress on a p2pdma page. Report such + * a bvec as not faultable so that callers do not retry the copy forever. + */ +static bool iov_iter_bvec_has_p2pdma(const struct iov_iter *i) +{ + const struct bio_vec *bv; + unsigned int offset; + + if (!iov_iter_is_bvec(i) || !i->count || !i->nr_segs) + return false; + + bv = i->bvec; + offset = bv->bv_offset + i->iov_offset; + return is_pci_p2pdma_page(bv->bv_page + offset / PAGE_SIZE); +} + /* * fault_in_iov_iter_readable - fault in iov iterator for reading * @i: iterator * @size: maximum length * @@ -205,11 +227,12 @@ static int copyin(void *to, const void __user *from, size_t n) * @size. For each iovec, fault in each page that constitutes the iovec. * * Returns the number of bytes not faulted in (like copy_to_user() and * copy_from_user()). * - * Always returns 0 for non-userspace iterators. + * Returns 0 for non-userspace iterators, except for a bvec iterator whose + * next page is p2pdma memory: that page cannot be faulted in. */ size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size) { if (iter_is_ubuf(i)) { size_t n = min(size, iov_iter_count(i)); @@ -231,10 +254,12 @@ size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size) count -= len - ret; if (ret) break; } return count + size; + } else if (iov_iter_bvec_has_p2pdma(i)) { + return size; } return 0; } EXPORT_SYMBOL(fault_in_iov_iter_readable); @@ -248,11 +273,12 @@ EXPORT_SYMBOL(fault_in_iov_iter_readable); * some or all of the pages in @i aren't in memory. * * Returns the number of bytes not faulted in, like copy_to_user() and * copy_from_user(). * - * Always returns 0 for non-user-space iterators. + * Returns 0 for non-user-space iterators, except for a bvec iterator whose + * next page is p2pdma memory: that page cannot be faulted in. */ size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size) { if (iter_is_ubuf(i)) { size_t n = min(size, iov_iter_count(i)); @@ -274,10 +300,12 @@ size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size) count -= len - ret; if (ret) break; } return count + size; + } else if (iov_iter_bvec_has_p2pdma(i)) { + return size; } return 0; } EXPORT_SYMBOL(fault_in_iov_iter_writeable); -- 2.52.0
From: hotforest <hotforest@gmail.com> mainline inclusion from mainline-v6.15-rc1 commit 82ba975e4c43d98afebced82d940ddb7aec42a9d category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- Zone device pages are used to represent various type of device memory managed by device drivers. Currently compound zone device pages are not supported. This is because MEMORY_DEVICE_FS_DAX pages are the only user of higher order zone device pages and have their own page reference counting. A future change will unify FS DAX reference counting with normal page reference counting rules and remove the special FS DAX reference counting. Supporting that requires compound zone device pages. Supporting compound zone device pages requires compound_head() to distinguish between head and tail pages whilst still preserving the special struct page fields that are specific to zone device pages. A tail page is distinguished by having bit zero being set in page->compound_head, with the remaining bits pointing to the head page. For zone device pages page->compound_head is shared with page->pgmap. The page->pgmap field must be common to all pages within a folio, even if the folio spans memory sections. Therefore pgmap is the same for both head and tail pages and can be moved into the folio and we can use the standard scheme to find compound_head from a tail page. Link: https://lkml.kernel.org/r/67055d772e6102accf85161d0b57b0b3944292bf.174071340... Signed-off-by: Alistair Popple <apopple@nvidia.com> Signed-off-by: Balbir Singh <balbirs@nvidia.com> Reviewed-by: Jason Gunthorpe <jgg@nvidia.com> Reviewed-by: Dan Williams <dan.j.williams@intel.com> Acked-by: David Hildenbrand <david@redhat.com> Tested-by: Alison Schofield <alison.schofield@intel.com> Cc: Alexander Gordeev <agordeev@linux.ibm.com> Cc: Asahi Lina <lina@asahilina.net> Cc: Bjorn Helgaas <bhelgaas@google.com> Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Christian Borntraeger <borntraeger@linux.ibm.com> Cc: Christoph Hellwig <hch@lst.de> Cc: Chunyan Zhang <zhang.lyra@gmail.com> Cc: "Darrick J. Wong" <djwong@kernel.org> Cc: Dave Chinner <david@fromorbit.com> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: Dave Jiang <dave.jiang@intel.com> Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com> Cc: Heiko Carstens <hca@linux.ibm.com> Cc: Huacai Chen <chenhuacai@kernel.org> Cc: Ira Weiny <ira.weiny@intel.com> Cc: Jan Kara <jack@suse.cz> Cc: Jason Gunthorpe <jgg@ziepe.ca> Cc: John Hubbard <jhubbard@nvidia.com> Cc: linmiaohe <linmiaohe@huawei.com> Cc: Logan Gunthorpe <logang@deltatee.com> Cc: Matthew Wilcow (Oracle) <willy@infradead.org> Cc: Michael "Camp Drill Sergeant" Ellerman <mpe@ellerman.id.au> Cc: Nicholas Piggin <npiggin@gmail.com> Cc: Peter Xu <peterx@redhat.com> Cc: Sven Schnelle <svens@linux.ibm.com> Cc: Ted Ts'o <tytso@mit.edu> Cc: Vasily Gorbik <gor@linux.ibm.com> Cc: Vishal Verma <vishal.l.verma@intel.com> Cc: Vivek Goyal <vgoyal@redhat.com> Cc: WANG Xuerui <kernel@xen0n.name> Cc: Will Deacon <will@kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Conflicts: include/linux/mmzone.h [OLK-6.6 keeps pgmap in struct page and has no folio->pgmap. Use compound_head()->pgmap in page_pgmap(). Limit this backport to the pgmap accessors needed by compound P2PDMA pages; omit unrelated folio layout and device-private migration changes.] Signed-off-by: hotforest <hotforest@gmail.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- drivers/pci/p2pdma.c | 6 +++--- include/linux/memremap.h | 6 +++--- include/linux/mmzone.h | 13 ++++++++++++- mm/memremap.c | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index b807f3fa221d..5522377f671d 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -192,11 +192,11 @@ static const struct attribute_group p2pmem_group = { .name = "p2pmem", }; static void p2pdma_page_free(struct page *page) { - struct pci_p2pdma_pagemap *pgmap = to_p2p_pgmap(page->pgmap); + struct pci_p2pdma_pagemap *pgmap = to_p2p_pgmap(page_pgmap(page)); /* safe to dereference while a reference is held to the percpu ref */ struct pci_p2pdma *p2pdma = rcu_dereference_protected(pgmap->provider->p2pdma, 1); struct percpu_ref *ref; @@ -1024,12 +1024,12 @@ static enum pci_p2pdma_map_type pci_p2pdma_map_type(struct dev_pagemap *pgmap, */ enum pci_p2pdma_map_type pci_p2pdma_map_segment(struct pci_p2pdma_map_state *state, struct device *dev, struct scatterlist *sg) { - if (state->pgmap != sg_page(sg)->pgmap) { - state->pgmap = sg_page(sg)->pgmap; + if (state->pgmap != page_pgmap(sg_page(sg))) { + state->pgmap = page_pgmap(sg_page(sg)); state->map = pci_p2pdma_map_type(state->pgmap, dev); state->bus_off = to_p2p_pgmap(state->pgmap)->bus_offset; } if (state->map == PCI_P2PDMA_MAP_BUS_ADDR) { diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 816532f7ae92..237b04d9ab1b 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -167,11 +167,11 @@ static inline unsigned long pgmap_vmemmap_nr(struct dev_pagemap *pgmap) static inline bool is_device_private_page(const struct page *page) { return IS_ENABLED(CONFIG_DEVICE_PRIVATE) && is_zone_device_page(page) && - page->pgmap->type == MEMORY_DEVICE_PRIVATE; + page_pgmap(page)->type == MEMORY_DEVICE_PRIVATE; } static inline bool folio_is_device_private(const struct folio *folio) { return is_device_private_page(&folio->page); @@ -179,17 +179,17 @@ static inline bool folio_is_device_private(const struct folio *folio) static inline bool is_pci_p2pdma_page(const struct page *page) { return IS_ENABLED(CONFIG_PCI_P2PDMA) && is_zone_device_page(page) && - page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA; + page_pgmap(page)->type == MEMORY_DEVICE_PCI_P2PDMA; } static inline bool is_device_coherent_page(const struct page *page) { return is_zone_device_page(page) && - page->pgmap->type == MEMORY_DEVICE_COHERENT; + page_pgmap(page)->type == MEMORY_DEVICE_COHERENT; } static inline bool folio_is_device_coherent(const struct folio *folio) { return is_device_coherent_page(&folio->page); diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index a6ed47ed840b..abe337e2d7b5 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1162,10 +1162,16 @@ static inline enum zone_type folio_zonenum(const struct folio *folio) static inline bool is_zone_device_page(const struct page *page) { return page_zonenum(page) == ZONE_DEVICE; } +static inline struct dev_pagemap *page_pgmap(const struct page *page) +{ + VM_WARN_ON_ONCE_PAGE(!is_zone_device_page(page), page); + return compound_head(page)->pgmap; +} + /* * Consecutive zone device pages should not be merged into the same sgl * or bvec segment with other types of pages or if they belong to different * pgmaps. Otherwise getting the pgmap of a given segment is not possible * without scanning the entire segment. This helper returns true either if @@ -1177,11 +1183,11 @@ static inline bool zone_device_pages_have_same_pgmap(const struct page *a, { if (is_zone_device_page(a) != is_zone_device_page(b)) return false; if (!is_zone_device_page(a)) return true; - return a->pgmap == b->pgmap; + return page_pgmap(a) == page_pgmap(b); } extern void memmap_init_zone_device(struct zone *, unsigned long, unsigned long, struct dev_pagemap *); #else @@ -1192,10 +1198,15 @@ static inline bool is_zone_device_page(const struct page *page) static inline bool zone_device_pages_have_same_pgmap(const struct page *a, const struct page *b) { return true; } + +static inline struct dev_pagemap *page_pgmap(const struct page *page) +{ + return NULL; +} #endif static inline bool folio_is_zone_device(const struct folio *folio) { return is_zone_device_page(&folio->page); diff --git a/mm/memremap.c b/mm/memremap.c index 8c5ebe5b0fc7..04306f3568dd 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -515,11 +515,11 @@ void zone_device_page_init(struct page *page) { /* * Drivers shouldn't be allocating pages after calling * memunmap_pages(). */ - WARN_ON_ONCE(!percpu_ref_tryget_live(&page->pgmap->ref)); + WARN_ON_ONCE(!percpu_ref_tryget_live(&page_pgmap(page)->ref)); set_page_count(page, 1); lock_page(page); } EXPORT_SYMBOL_GPL(zone_device_page_init); -- 2.52.0
From: hotforest <hotforest@gmail.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 -------------------------------- Add ARCH_WANT_OPTIMIZE_P2PDMA_VMEMMAP so architectures can enable vmemmap deduplication for P2PDMA independently of device DAX. Let either the DAX or P2PDMA architecture capability enable the generic vmemmap optimization check. The implementation and its minimum metadata page requirement are shared. No architecture selects the new capability yet, so this change alone does not change runtime behavior. Signed-off-by: hotforest <hotforest@gmail.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- include/linux/mm.h | 7 ++++--- mm/Kconfig | 7 +++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 8784c13db3d0..28eda1d3e5c8 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4152,11 +4152,12 @@ void vmemmap_populate_print_last(void); void vmemmap_free(unsigned long start, unsigned long end, struct vmem_altmap *altmap); #endif #define VMEMMAP_RESERVE_NR 2 -#ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP +#if defined(CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP) || \ + defined(CONFIG_ARCH_WANT_OPTIMIZE_P2PDMA_VMEMMAP) static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { unsigned long nr_pages; unsigned long nr_vmemmap_pages; @@ -4165,12 +4166,12 @@ static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, return false; nr_pages = pgmap_vmemmap_nr(pgmap); nr_vmemmap_pages = ((nr_pages * sizeof(struct page)) >> PAGE_SHIFT); /* - * For vmemmap optimization with DAX we need minimum 2 vmemmap - * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst + * Vmemmap optimization needs a minimum of 2 vmemmap pages. See the + * layout diagram in Documentation/mm/vmemmap_dedup.rst. */ return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR); } /* * If we don't have an architecture override, use the generic rule diff --git a/mm/Kconfig b/mm/Kconfig index 89b15236b874..7c69ffb52e24 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -533,16 +533,19 @@ config SPARSEMEM_VMEMMAP help SPARSEMEM_VMEMMAP uses a virtually mapped memmap to optimise pfn_to_page and page_to_pfn operations. This is the most efficient option when sufficient kernel resources are available. # -# Select this config option from the architecture Kconfig, if it is preferred -# to enable the feature of HugeTLB/dev_dax vmemmap optimization. +# Select these config options from the architecture Kconfig, if it is preferred +# to enable the feature of HugeTLB/dev_dax/P2PDMA vmemmap optimization. # config ARCH_WANT_OPTIMIZE_DAX_VMEMMAP bool +config ARCH_WANT_OPTIMIZE_P2PDMA_VMEMMAP + bool + config ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP bool config HAVE_MEMBLOCK_PHYS_MAP bool -- 2.52.0
From: hotforest <hotforest@gmail.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 -------------------------------- Add pci_p2pdma_add_resource_aligned() to register P2PDMA memory with PMD- or PUD-sized compound pages when the resource geometry and architecture support vmemmap deduplication. Add pci_p2pdma_max_pagemap_align() to select the largest supported alignment. Keep pci_p2pdma_add_resource() as a PAGE_SIZE wrapper so existing providers retain their base-page behavior. Use the same alignment for the gen_pool allocation granularity and page free path. Reject mappings through p2pmem/allocate for compound-page providers because these pages are restricted to kernel DMA use. Reject a second registration that uses a different alignment on the same device. Signed-off-by: hotforest <hotforest@gmail.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- drivers/pci/p2pdma.c | 107 ++++++++++++++++++++++++++++++++----- include/linux/pci-p2pdma.h | 21 +++++++- 2 files changed, 112 insertions(+), 16 deletions(-) diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c index 5522377f671d..b057a82bd547 100644 --- a/drivers/pci/p2pdma.c +++ b/drivers/pci/p2pdma.c @@ -9,10 +9,11 @@ */ #define pr_fmt(fmt) "pci-p2pdma: " fmt #include <linux/ctype.h> #include <linux/dma-map-ops.h> +#include <linux/mm.h> #include <linux/pci-p2pdma.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/genalloc.h> #include <linux/memremap.h> @@ -22,10 +23,11 @@ #include <linux/xarray.h> struct pci_p2pdma { struct gen_pool *pool; bool p2pmem_published; + size_t align; struct xarray map_types; }; struct pci_p2pdma_pagemap { struct dev_pagemap pgmap; @@ -119,10 +121,14 @@ static int p2pmem_alloc_mmap(struct file *filp, struct kobject *kobj, p2pdma = rcu_dereference(pdev->p2pdma); if (!p2pdma) { ret = -ENODEV; goto out; } + if (p2pdma->align != PAGE_SIZE) { + ret = -EOPNOTSUPP; + goto out; + } kaddr = (void *)gen_pool_alloc_owner(p2pdma->pool, len, (void **)&ref); if (!kaddr) { ret = -ENOMEM; goto out; @@ -194,16 +200,16 @@ static const struct attribute_group p2pmem_group = { static void p2pdma_page_free(struct page *page) { struct pci_p2pdma_pagemap *pgmap = to_p2p_pgmap(page_pgmap(page)); /* safe to dereference while a reference is held to the percpu ref */ - struct pci_p2pdma *p2pdma = - rcu_dereference_protected(pgmap->provider->p2pdma, 1); + struct pci_p2pdma *p2pdma; struct percpu_ref *ref; + p2pdma = rcu_dereference_protected(pgmap->provider->p2pdma, 1); gen_pool_free_owner(p2pdma->pool, (uintptr_t)page_to_virt(page), - PAGE_SIZE, (void **)&ref); + p2pdma->align, (void **)&ref); percpu_ref_put(ref); } static const struct dev_pagemap_ops p2pdma_pgmap_ops = { .page_free = p2pdma_page_free, @@ -225,22 +231,23 @@ static void pci_p2pdma_release(void *data) gen_pool_destroy(p2pdma->pool); sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group); xa_destroy(&p2pdma->map_types); } -static int pci_p2pdma_setup(struct pci_dev *pdev) +static int pci_p2pdma_setup(struct pci_dev *pdev, size_t align) { int error = -ENOMEM; struct pci_p2pdma *p2p; p2p = devm_kzalloc(&pdev->dev, sizeof(*p2p), GFP_KERNEL); if (!p2p) return -ENOMEM; xa_init(&p2p->map_types); - p2p->pool = gen_pool_create(PAGE_SHIFT, dev_to_node(&pdev->dev)); + p2p->align = align; + p2p->pool = gen_pool_create(ilog2(align), dev_to_node(&pdev->dev)); if (!p2p->pool) goto out; error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev); if (error) @@ -271,22 +278,71 @@ static void pci_p2pdma_unmap_mappings(void *data) */ sysfs_remove_file_from_group(&pdev->dev.kobj, &p2pmem_alloc_attr.attr, p2pmem_group.name); } +static bool pci_p2pdma_vmemmap_can_optimize(size_t align) +{ + struct dev_pagemap pgmap = { }; + + pgmap.vmemmap_shift = ilog2(align) - PAGE_SHIFT; + return vmemmap_can_optimize(NULL, &pgmap); +} + /** - * pci_p2pdma_add_resource - add memory for use as p2p memory + * pci_p2pdma_max_pagemap_align - find the largest deduplicated page geometry + * @pdev: the provider device + * @bar: PCI BAR to add + * @size: effective size of the resource, not zero + * @offset: offset into the PCI BAR + * + * Return PAGE_SIZE when the architecture cannot deduplicate vmemmap pages or + * when the resource has no supported PUD or PMD alignment. + */ +size_t pci_p2pdma_max_pagemap_align(struct pci_dev *pdev, int bar, + size_t size, u64 offset) +{ + resource_size_t start = pci_resource_start(pdev, bar); + + if (IS_ALIGNED(start | size | offset, PUD_SIZE) && + pci_p2pdma_vmemmap_can_optimize(PUD_SIZE)) + return PUD_SIZE; + if (IS_ALIGNED(start | size | offset, PMD_SIZE) && + pci_p2pdma_vmemmap_can_optimize(PMD_SIZE)) + return PMD_SIZE; + return PAGE_SIZE; +} +EXPORT_SYMBOL_GPL(pci_p2pdma_max_pagemap_align); + +static int pci_p2pdma_check_pagemap_align(struct pci_dev *pdev, int bar, + size_t size, size_t align, u64 offset) +{ + if (align == PAGE_SIZE) + return 0; + if (align != PUD_SIZE && align != PMD_SIZE) + return -EINVAL; + if (!IS_ALIGNED(pci_resource_start(pdev, bar) | size | offset, align)) + return -EINVAL; + if (!pci_p2pdma_vmemmap_can_optimize(align)) + return -EOPNOTSUPP; + return 0; +} + +/** + * pci_p2pdma_add_resource_aligned - add memory with compound struct pages * @pdev: the device to add the memory to * @bar: PCI BAR to add * @size: size of the memory to add, may be zero to use the whole BAR + * @align: allocation and compound-page alignment * @offset: offset into the PCI BAR * - * The memory will be given ZONE_DEVICE struct pages so that it may - * be used with any DMA request. + * The memory will be given ZONE_DEVICE struct pages so that it may be used + * with kernel DMA requests. Compound-page resources cannot be mapped through + * the p2pmem allocate sysfs file. */ -int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, - u64 offset) +int pci_p2pdma_add_resource_aligned(struct pci_dev *pdev, int bar, size_t size, + size_t align, u64 offset) { struct pci_p2pdma_pagemap *p2p_pgmap; struct dev_pagemap *pgmap; struct pci_p2pdma *p2pdma; void *addr; @@ -299,15 +355,22 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, return -EINVAL; if (!size) size = pci_resource_len(pdev, bar) - offset; - if (size + offset > pci_resource_len(pdev, bar)) + if (size > pci_resource_len(pdev, bar) - offset) + return -EINVAL; + + error = pci_p2pdma_check_pagemap_align(pdev, bar, size, align, offset); + if (error) + return error; + + if (pdev->p2pdma && pdev->p2pdma->align != align) return -EINVAL; if (!pdev->p2pdma) { - error = pci_p2pdma_setup(pdev); + error = pci_p2pdma_setup(pdev, align); if (error) return error; } p2p_pgmap = devm_kzalloc(&pdev->dev, sizeof(*p2p_pgmap), GFP_KERNEL); @@ -318,10 +381,12 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, pgmap->range.start = pci_resource_start(pdev, bar) + offset; pgmap->range.end = pgmap->range.start + size - 1; pgmap->nr_range = 1; pgmap->type = MEMORY_DEVICE_PCI_P2PDMA; pgmap->ops = &p2pdma_pgmap_ops; + if (align > PAGE_SIZE) + pgmap->vmemmap_shift = ilog2(align) - PAGE_SHIFT; p2p_pgmap->provider = pdev; p2p_pgmap->bus_offset = pci_bus_address(pdev, bar) - pci_resource_start(pdev, bar); @@ -342,21 +407,35 @@ int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, range_len(&pgmap->range), dev_to_node(&pdev->dev), &pgmap->ref); if (error) goto pages_free; - pci_info(pdev, "added peer-to-peer DMA memory %#llx-%#llx\n", - pgmap->range.start, pgmap->range.end); + pci_info(pdev, "added peer-to-peer DMA memory %#llx-%#llx alignment %#lx\n", + pgmap->range.start, pgmap->range.end, (unsigned long)align); return 0; pages_free: devm_memunmap_pages(&pdev->dev, pgmap); pgmap_free: devm_kfree(&pdev->dev, p2p_pgmap); return error; } +EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource_aligned); + +/** + * pci_p2pdma_add_resource - add memory with base struct pages + * @pdev: the device to add the memory to + * @bar: PCI BAR to add + * @size: size of the memory to add, may be zero to use the whole BAR + * @offset: offset into the PCI BAR + */ +int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, + u64 offset) +{ + return pci_p2pdma_add_resource_aligned(pdev, bar, size, PAGE_SIZE, offset); +} EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource); /* * Note this function returns the parent PCI device with a * reference taken. It is the caller's responsibility to drop diff --git a/include/linux/pci-p2pdma.h b/include/linux/pci-p2pdma.h index 2c07aa6b7665..183cd650320d 100644 --- a/include/linux/pci-p2pdma.h +++ b/include/linux/pci-p2pdma.h @@ -16,11 +16,15 @@ struct block_device; struct scatterlist; #ifdef CONFIG_PCI_P2PDMA int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size, - u64 offset); + u64 offset); +int pci_p2pdma_add_resource_aligned(struct pci_dev *pdev, int bar, + size_t size, size_t align, u64 offset); +size_t pci_p2pdma_max_pagemap_align(struct pci_dev *pdev, int bar, + size_t size, u64 offset); int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients, int num_clients, bool verbose); bool pci_has_p2pmem(struct pci_dev *pdev); struct pci_dev *pci_p2pmem_find_many(struct device **clients, int num_clients); void *pci_alloc_p2pmem(struct pci_dev *pdev, size_t size); @@ -34,14 +38,27 @@ int pci_p2pdma_enable_store(const char *page, struct pci_dev **p2p_dev, bool *use_p2pdma); ssize_t pci_p2pdma_enable_show(char *page, struct pci_dev *p2p_dev, bool use_p2pdma); #else /* CONFIG_PCI_P2PDMA */ static inline int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, - size_t size, u64 offset) + size_t size, u64 offset) { return -EOPNOTSUPP; } + +static inline int pci_p2pdma_add_resource_aligned(struct pci_dev *pdev, int bar, + size_t size, size_t align, + u64 offset) +{ + return -EOPNOTSUPP; +} + +static inline size_t pci_p2pdma_max_pagemap_align(struct pci_dev *pdev, int bar, + size_t size, u64 offset) +{ + return PAGE_SIZE; +} static inline int pci_p2pdma_distance_many(struct pci_dev *provider, struct device **clients, int num_clients, bool verbose) { return -1; } -- 2.52.0
From: hotforest <hotforest@gmail.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 -------------------------------- Select ARCH_WANT_OPTIMIZE_P2PDMA_VMEMMAP on x86-64. This declares that the architecture supports compound-page vmemmap deduplication for aligned P2PDMA resources independently of the DAX capability. Signed-off-by: hotforest <hotforest@gmail.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- arch/x86/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index c9e017f4488d..4c625d9ebe06 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -135,10 +135,11 @@ config X86 select ARCH_WANT_GENERAL_HUGETLB select ARCH_WANT_HUGE_PMD_SHARE select ARCH_HAS_OPTIMIZED_POLL select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANT_OPTIMIZE_DAX_VMEMMAP if X86_64 + select ARCH_WANT_OPTIMIZE_P2PDMA_VMEMMAP if X86_64 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP if X86_64 select ARCH_WANTS_THP_SWAP if X86_64 select ARCH_HAS_PARANOID_L1D_FLUSH select BUILDTIME_TABLE_SORT select CLKEVT_I8253 -- 2.52.0
From: hotforest <hotforest@gmail.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 -------------------------------- Select ARCH_WANT_OPTIMIZE_P2PDMA_VMEMMAP for ARM64_4K_PAGES. The generic 4 KiB-page configuration supports PMD and PUD compound-page vmemmap deduplication for aligned P2PDMA resources without enabling a DAX-specific architecture capability. Signed-off-by: hotforest <hotforest@gmail.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- arch/arm64/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index c4a87da2a709..5df3aaa9ff14 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -114,10 +114,11 @@ config ARM64 select ARCH_WANT_COMPAT_IPC_PARSE_VERSION if COMPAT select ARCH_WANT_DEFAULT_BPF_JIT select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT select ARCH_WANT_FRAME_POINTERS select ARCH_WANT_HUGE_PMD_SHARE if ARM64_4K_PAGES || (ARM64_16K_PAGES && !ARM64_VA_BITS_36) + select ARCH_WANT_OPTIMIZE_P2PDMA_VMEMMAP if ARM64_4K_PAGES select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP if !PREEMPT_RT select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANTS_NO_INSTR select ARCH_WANTS_THP_SWAP if ARM64_4K_PAGES select ARCH_HAS_UBSAN_SANITIZE_ALL -- 2.52.0
反馈： 您发送到kernel@openeuler.org的补丁/补丁集，已成功转换为PR！ PR链接地址： https://atomgit.com/openeuler/kernel/merge_requests/30811 邮件列表地址：https://mailweb.openeuler.org/archives/list/kernel@openeuler.org/message/QKJ... FeedBack: The patch(es) which you have sent to kernel@openeuler.org mailing list has been converted to a pull request successfully! Pull request link: https://atomgit.com/openeuler/kernel/merge_requests/30811 Mailing list address: https://mailweb.openeuler.org/archives/list/kernel@openeuler.org/message/QKJ...
participants (2)
-
patchwork bot -
Zizhi Wo