From: Tong Tiangen <tongtiangen@huawei.com> hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IDBGTZ CVE: NA ------------------------------- Large memory migration imposes significant overhead on the CPU. This patch introduces enhancements to the page migration by offloading the copy to UDMA hardware which based on the DMA engine framework, and a new migration mode has been introduced, If DMA migration fails, it will fall back to CPU migration. DMA migration supports both asynchronous and synchronous modes for subsequent debugging purposes. Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> Signed-off-by: Zeng Heng <zengheng4@huawei.com> --- arch/arm64/configs/openeuler_defconfig | 1 + include/linux/migrate_mode.h | 2 + include/linux/mm.h | 8 ++ mm/Kconfig | 7 ++ mm/Makefile | 1 + mm/migrate.c | 5 + mm/migrate_dma.c | 143 +++++++++++++++++++++++++ 7 files changed, 167 insertions(+) create mode 100644 mm/migrate_dma.c diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index b9a451bd7d65..44e1c0debb1a 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -1186,6 +1186,7 @@ CONFIG_COMPACTION=y CONFIG_COMPACT_UNEVICTABLE_DEFAULT=1 CONFIG_PAGE_REPORTING=y CONFIG_MIGRATION=y +CONFIG_MIGRATE_PAGES_DMA_OFFLOADING=y CONFIG_DEVICE_MIGRATION=y CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION=y CONFIG_ARCH_ENABLE_THP_MIGRATION=y diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h index 0f577f932bb4..4c992bc66575 100644 --- a/include/linux/migrate_mode.h +++ b/include/linux/migrate_mode.h @@ -19,6 +19,8 @@ enum migrate_mode { MIGRATE_SYNC_NO_COPY, }; +#define MIGRATE_ASYNC_DMA_OFFLOADING (10) + enum migrate_reason { MR_COMPACTION, MR_MEMORY_FAILURE, diff --git a/include/linux/mm.h b/include/linux/mm.h index c0040a2014c4..036822cb1b9d 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1330,6 +1330,14 @@ void put_pages_list(struct list_head *pages); void split_page(struct page *page, unsigned int order); void folio_copy(struct folio *dst, struct folio *src); int folio_mc_copy(struct folio *dst, struct folio *src); +#ifdef CONFIG_MIGRATE_PAGES_DMA_OFFLOADING +int folio_dma_copy(struct folio *dst, struct folio *src); +#else +static inline int folio_dma_copy(struct folio *dst, struct folio *src) +{ + return -ENODEV; +} +#endif unsigned long nr_free_buffer_pages(void); diff --git a/mm/Kconfig b/mm/Kconfig index 4eb0642b71e5..9408a3ca7f4c 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -696,6 +696,13 @@ config MIGRATION pages as migration can relocate pages to satisfy a huge page allocation instead of reclaiming. +config MIGRATE_PAGES_DMA_OFFLOADING + bool "Support to use DMA channels for page migration" + depends on DMA_ENGINE + default y + help + Allows kernel to use dma offloading for page migraton. + config DEVICE_MIGRATION def_bool MIGRATION && ZONE_DEVICE diff --git a/mm/Makefile b/mm/Makefile index e45cdeda47b7..dd81f8c33e03 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -92,6 +92,7 @@ obj-$(CONFIG_FAILSLAB) += failslab.o obj-$(CONFIG_FAIL_PAGE_ALLOC) += fail_page_alloc.o obj-$(CONFIG_MEMTEST) += memtest.o obj-$(CONFIG_MIGRATION) += migrate.o +obj-$(CONFIG_MIGRATE_PAGES_DMA_OFFLOADING) += migrate_dma.o obj-$(CONFIG_NUMA) += memory-tiers.o obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o diff --git a/mm/migrate.c b/mm/migrate.c index 4edd29d9a041..8b1058b28175 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -583,6 +583,11 @@ static int folio_migrate_mc_copy(struct folio *dst, struct folio *src, if (mode == MIGRATE_SYNC_NO_COPY) return 0; + if (mode == MIGRATE_ASYNC_DMA_OFFLOADING && folio_test_hugetlb(src)) + /* if dma offloading fail, fallback */ + if (!folio_dma_copy(dst, src)) + return 0; + return folio_mc_copy(dst, src); } diff --git a/mm/migrate_dma.c b/mm/migrate_dma.c new file mode 100644 index 000000000000..c6ff6f1258aa --- /dev/null +++ b/mm/migrate_dma.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Support to use DMA channels for page migration. + * + * Copyright (C) 2025 Huawei Limited + */ + +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> + +/* DMA channel track its transfers done */ +struct dma_channel_work { + struct dma_chan *chan; + enum dma_status status; + struct completion done; +}; + +static void folios_dma_copy_completion_callback(void *param, + const struct dmaengine_result *result) +{ + struct dma_channel_work *chan_work = param; + + if (result) { + enum dmaengine_tx_result dma_res = result->result; + + if (dma_res == DMA_TRANS_NOERROR) + chan_work->status = DMA_COMPLETE; + else + chan_work->status = DMA_ERROR; + } + + complete(&chan_work->done); +} + +static int process_folio_dma_transfer(struct dma_channel_work *chan_work, + struct folio *src, struct folio *dst) +{ + struct dma_chan *chan = chan_work->chan; + struct device *dev = dmaengine_get_dma_device(chan); + enum dma_ctrl_flags flags = DMA_CTRL_ACK; + struct dma_async_tx_descriptor *tx; + dma_addr_t src_handle, dst_handle; + size_t size = folio_size(src); + int ret; + + flags |= DMA_PREP_INTERRUPT; + + src_handle = dma_map_page(dev, &src->page, 0, size, DMA_TO_DEVICE); + if (dma_mapping_error(dev, src_handle)) { + pr_err("map dma src page error.\n"); + return -ENOMEM; + } + + dst_handle = dma_map_page(dev, &dst->page, 0, size, DMA_FROM_DEVICE); + if (dma_mapping_error(dev, dst_handle)) { + pr_err("map dma dst page error.\n"); + ret = -ENOMEM; + goto out_unmap; + } + + tx = dmaengine_prep_dma_memcpy(chan, dst_handle, src_handle, + size, flags); + if (unlikely(!tx)) { + pr_err("prep dma memcpy error.\n"); + ret = -EBUSY; + goto out_unmap_all; + } + + tx->callback_result = folios_dma_copy_completion_callback; + tx->callback_param = chan_work; + init_completion(&chan_work->done); + chan_work->status = DMA_ERROR; + + if (dma_submit_error(dmaengine_submit(tx))) { + pr_err("dma submit error.\n"); + ret = -EINVAL; + goto out_unmap_all; + } + + dma_async_issue_pending(chan); + if (!wait_for_completion_timeout(&chan_work->done, + msecs_to_jiffies(1000))) { + ret = -ETIMEDOUT; + goto out_unmap_all; + } + + ret = (chan_work->status == DMA_COMPLETE) ? 0 : -EPROTO; + +out_unmap_all: + dma_unmap_page(dev, dst_handle, size, DMA_FROM_DEVICE); +out_unmap: + dma_unmap_page(dev, src_handle, size, DMA_TO_DEVICE); + + return ret; +} + +static bool folio_dma_chan_filter(struct dma_chan *chan, void *param) +{ + return !strcmp(dev_name(chan->device->dev), "ub_dma_device"); +} + +int folio_dma_copy(struct folio *dst, struct folio *src) +{ + struct dma_channel_work *chan_work; + struct dma_slave_config dma_cfg; + struct dma_chan *chan; + dma_cap_mask_t mask; + int ret = -ENODEV; + + dma_cap_zero(mask); + dma_cap_set(DMA_MEMCPY, mask); + chan = dma_request_channel(mask, folio_dma_chan_filter, NULL); + if (!chan) { + pr_err("failed to allocate dma channel.\n"); + return ret; + } + + memset(&dma_cfg, 0, sizeof(dma_cfg)); + dma_cfg.direction = DMA_MEM_TO_MEM; + ret = dmaengine_slave_config(chan, &dma_cfg); + if (ret) { + pr_err("failed to config dma channel.\n"); + goto out_release; + } + + chan_work = kmalloc(sizeof(*chan_work), GFP_KERNEL); + if (unlikely(!chan_work)) { + pr_err("failed to allocate memory for chan work.\n"); + goto out_release; + } + + chan_work->chan = chan; + ret = process_folio_dma_transfer(chan_work, src, dst); + if (unlikely(ret)) + pr_err("failed to process folio dma transfer.\n"); + + kfree(chan_work); +out_release: + dma_release_channel(chan); + + return ret; +} +EXPORT_SYMBOL(folio_dma_copy); -- 2.25.1