From: Hou Tao <houtao1@huawei.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10302 -------------------------------- Stop all bvec copy and zero operations before they map a PCI P2PDMA page for CPU access. Return a short count so callers fail the operation instead of reading, writing, or zeroing device memory with the CPU. Make fault_in_iov_iter_readable() and fault_in_iov_iter_writeable() report a P2PDMA bvec as not faultable. This prevents buffered write loops from retrying a zero-length copy forever. The check applies to every PCI P2PDMA provider. No provider opt-in, static key, or runtime switch is used. Signed-off-by: Hou Tao <houtao1@huawei.com> Signed-off-by: Zizhi Wo <wozizhi@huawei.com> --- lib/iov_iter.c | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/lib/iov_iter.c b/lib/iov_iter.c index a4cc8d68b409..d461b56c41f9 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -11,10 +11,11 @@ #include <linux/splice.h> #include <linux/compat.h> #include <net/checksum.h> #include <linux/scatterlist.h> #include <linux/instrumented.h> +#include <linux/memremap.h> /* covers ubuf and kbuf alike */ #define iterate_buf(i, n, base, len, off, __p, STEP) { \ size_t __maybe_unused off = 0; \ len = n; \ @@ -50,12 +51,16 @@ size_t off = 0; \ unsigned skip = i->iov_offset; \ while (n) { \ unsigned offset = p->bv_offset + skip; \ unsigned left; \ - void *kaddr = kmap_local_page(p->bv_page + \ - offset / PAGE_SIZE); \ + struct page *bv_pg = p->bv_page + \ + offset / PAGE_SIZE; \ + void *kaddr; \ + if (is_pci_p2pdma_page(bv_pg)) \ + break; \ + kaddr = kmap_local_page(bv_pg); \ base = kaddr + offset % PAGE_SIZE; \ len = min(min(n, (size_t)(p->bv_len - skip)), \ (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \ left = (STEP); \ kunmap_local(kaddr); \ @@ -194,10 +199,27 @@ static int copyin(void *to, const void __user *from, size_t n) instrument_copy_from_user_after(to, from, n, res); } return res; } +/* + * The copy and zero helpers make no progress on a p2pdma page. Report such + * a bvec as not faultable so that callers do not retry the copy forever. + */ +static bool iov_iter_bvec_has_p2pdma(const struct iov_iter *i) +{ + const struct bio_vec *bv; + unsigned int offset; + + if (!iov_iter_is_bvec(i) || !i->count || !i->nr_segs) + return false; + + bv = i->bvec; + offset = bv->bv_offset + i->iov_offset; + return is_pci_p2pdma_page(bv->bv_page + offset / PAGE_SIZE); +} + /* * fault_in_iov_iter_readable - fault in iov iterator for reading * @i: iterator * @size: maximum length * @@ -205,11 +227,12 @@ static int copyin(void *to, const void __user *from, size_t n) * @size. For each iovec, fault in each page that constitutes the iovec. * * Returns the number of bytes not faulted in (like copy_to_user() and * copy_from_user()). * - * Always returns 0 for non-userspace iterators. + * Returns 0 for non-userspace iterators, except for a bvec iterator whose + * next page is p2pdma memory: that page cannot be faulted in. */ size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size) { if (iter_is_ubuf(i)) { size_t n = min(size, iov_iter_count(i)); @@ -231,10 +254,12 @@ size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size) count -= len - ret; if (ret) break; } return count + size; + } else if (iov_iter_bvec_has_p2pdma(i)) { + return size; } return 0; } EXPORT_SYMBOL(fault_in_iov_iter_readable); @@ -248,11 +273,12 @@ EXPORT_SYMBOL(fault_in_iov_iter_readable); * some or all of the pages in @i aren't in memory. * * Returns the number of bytes not faulted in, like copy_to_user() and * copy_from_user(). * - * Always returns 0 for non-user-space iterators. + * Returns 0 for non-user-space iterators, except for a bvec iterator whose + * next page is p2pdma memory: that page cannot be faulted in. */ size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size) { if (iter_is_ubuf(i)) { size_t n = min(size, iov_iter_count(i)); @@ -274,10 +300,12 @@ size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size) count -= len - ret; if (ret) break; } return count + size; + } else if (iov_iter_bvec_has_p2pdma(i)) { + return size; } return 0; } EXPORT_SYMBOL(fault_in_iov_iter_writeable); -- 2.52.0