On 2022/1/14 17:45, Kefeng Wang wrote:

On 2022/1/14 17:16, Zhuling wrote:

...

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4O31I?from=project-issue CVE: NA

---

Register pmem in arm64: Use memmap(memmap=nn[KMG]!ss[KMG]) reserve memory for pmem to register persistent memory in arm64. when the kernel restart or update, the data in PMEM will not be lost and can be loaded faster. this is a general features. Currently, only one pmem can be registered and use.

If you want use this features, you need do as follows: 1.Reserve memory: add memmap to reserve memory in grub.cfg    memmap=nn[KMG]!ss[KMG] exp:memmap=100K!0x1a0000000. 2.Insmod nd_e820.ko: modprobe nd_e820. 3.Check pmem device in /dev exp: /dev/pmem0

driver/nvdimm/e820.c: The function of this file is scan "iomem_resource" and take advantage of nvdimm resource discovery mechanism by registering a resource named "Persistent Memory (legacy)", this function doesn't depend on architecture.

We will push the feature to linux kernel community and discuss to modify the file name. because people have a mistaken notion that the e820.c is depend on x86.

Signed-off-by: Zhuling zhuling8@huawei.com

arch/arm64/Kconfig         | 21 ++++++++++++++++++++   arch/arm64/kernel/Makefile |  1 +   arch/arm64/kernel/pmem.c   | 35 +++++++++++++++++++++++++++++++++   arch/arm64/kernel/setup.c  | 10 ++++++++++   arch/arm64/mm/init.c       | 48 ++++++++++++++++++++++++++++++++++++++++++++++   drivers/nvdimm/Kconfig     |  7 +++++++   drivers/nvdimm/Makefile    |  1 +   7 files changed, 123 insertions(+)   create mode 100644 arch/arm64/kernel/pmem.c

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 405e5ce..deec2bb 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1322,6 +1322,27 @@ config RODATA_FULL_DEFAULT_ENABLED         This requires the linear region to be mapped down to pages,         which may adversely affect performance in some cases.   +config ARM64_PMEM_RESERVE +    bool "Reserve memory for persistent storage" +    default n +    help +      Use memmap=nn[KMG]!ss[KMG](memmap=100K!0x1a0000000) reserve +      memory for persistent storage.

+      Say y here to enable this feature.

如果ARM64_PMEM_LEGACY_DEVICE 不开启，单独的reserve 没有意义，所以不应该可选择；

我们是想做成独立的功能，不仅仅是只给pmem用

...

+config ARM64_PMEM_LEGACY_DEVICE +    bool "Create persistent storage" +    depends on BLK_DEV +    depends on LIBNVDIMM +    select ARM64_PMEM_RESERVE +    help +      Use reserved memory for persistent storage when the kernel +      restart or update. the data in PMEM will not be lost and +      can be loaded faster.

+      Say y if unsure.

config ARM64_SW_TTBR0_PAN       bool "Emulate Privileged Access Never using TTBR0_EL1 switching"       help diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 169d90f..f615325 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -68,6 +68,7 @@ obj-$(CONFIG_ARM64_PTR_AUTH)        += pointer_auth.o   obj-$(CONFIG_SHADOW_CALL_STACK)        += scs.o   obj-$(CONFIG_ARM64_MTE)            += mte.o   obj-$(CONFIG_MPAM)            += mpam/ +obj-$(CONFIG_ARM64_PMEM_LEGACY_DEVICE)   += pmem.o     obj-y                    += vdso/ probes/   obj-$(CONFIG_COMPAT_VDSO)        += vdso32/ diff --git a/arch/arm64/kernel/pmem.c b/arch/arm64/kernel/pmem.c new file mode 100644 index 0000000..36405a2 --- /dev/null +++ b/arch/arm64/kernel/pmem.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +/*

• • Copyright(c) 2021 Huawei Technologies Co., Ltd
• • Derived from x86 and arm64 implement PMEM.

这个跟x86下的一模一样，如果不去重构，这里的描述最好不加华为license，并且最好加上

based on arch/x86/kernel/pmem.c

好的，这部分我完善下

补丁顺序和内容调整下，

1. arm64支持pmem和ARM64_PMEM_LEGACY_DEVICE，

2.  cmdline的支持+reserve_pmem；

补丁的顺序我调整下

...

• */

+#include <linux/platform_device.h> +#include <linux/init.h> +#include <linux/ioport.h> +#include <linux/module.h>

+static int found(struct resource *res, void *data) +{ +    return 1; +}

+static int __init register_e820_pmem(void) +{ +    struct platform_device *pdev; +    int rc;

+    rc = walk_iomem_res_desc(IORES_DESC_PERSISTENT_MEMORY_LEGACY, +                IORESOURCE_MEM, 0, -1, NULL, found); +    if (rc <= 0) +        return 0;

+    /* +     * See drivers/nvdimm/e820.c for the implementation, this is +     * simply here to trigger the module to load on demand. +     */ +    pdev = platform_device_alloc("e820_pmem", -1);

+return platform_device_add(pdev); +} +device_initcall(register_e820_pmem); diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 92d75e3..3eee532 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -70,6 +70,10 @@ static int __init arm64_enable_cpu0_hotplug(char *str)   __setup("arm64_cpu0_hotplug", arm64_enable_cpu0_hotplug);   #endif   +#ifdef CONFIG_ARM64_PMEM_RESERVE +extern struct resource pmem_res; +#endif

phys_addr_t __fdt_pointer __initdata;     /* @@ -303,6 +307,12 @@ static void __init request_standard_resources(void)               request_resource(res, &pin_memory_resource);   #endif       }

+#ifdef CONFIG_ARM64_PMEM_RESERVE +    if (pmem_res.end && pmem_res.start) +        request_resource(&iomem_resource, &pmem_res); +#endif

}     static int __init reserve_memblock_reserved_regions(void) diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index b9eb312..21f0153 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -115,6 +115,16 @@ phys_addr_t arm64_dma_phys_limit __ro_after_init;     static unsigned long long pmem_size, pmem_start;   +#ifdef CONFIG_ARM64_PMEM_RESERVE +struct resource pmem_res = { +    .name = "Persistent Memory (legacy)", +    .start = 0, +    .end = 0, +    .flags = IORESOURCE_MEM, +    .desc = IORES_DESC_PERSISTENT_MEMORY_LEGACY +}; +#endif

#ifndef CONFIG_KEXEC_CORE   static void __init reserve_crashkernel(void)   { @@ -408,6 +418,26 @@ static int __init reserve_park_mem(void)   }   #endif   +static bool __init is_mem_valid(unsigned long long mem_size, unsigned long long mem_start) +{ +    if (!memblock_is_region_memory(mem_start, mem_size)) { +        pr_warn("cannot reserve mem: region is not memory!\n"); +        return false; +    }

+    if (memblock_is_region_reserved(mem_start, mem_size)) { +        pr_warn("cannot reserve mem: region overlaps reserved memory!\n"); +        return false; +    }

+    if (!IS_ALIGNED(mem_start, SZ_2M)) { +        pr_warn("cannot reserve mem: base address is not 2MB aligned!\n"); +        return false; +    }

+    return true; +}

这个代码明显只为了pmem_reserve，应该放到一块； 甚至说可以直接放到reserve_pmem里面，没有独立出来的意义；

包括pmem_size/start，为啥不放到一块；

另外  上次提过，2M对齐目的是啥；

因为再看这部分代码时，发现很多reserve mem都用到这些判断，所以我们想把这部分单独拿出来，作为一个通用功能；

2M对齐是参考其他reserver memory做的，作为通用功能时给其他需要函数使用，pmem没有特殊要求，这部分可以删除

或者如有必要保留这部分，将2M修改为page size通用些；

...

static int need_remove_real_memblock __initdata;     static int __init parse_memmap_one(char *p) @@ -472,6 +502,20 @@ static int __init parse_memmap_opt(char *str)   }   early_param("memmap", parse_memmap_opt);   +#ifdef CONFIG_ARM64_PMEM_RESERVE +static void __init reserve_pmem(void) +{ +    if (!is_mem_valid(pmem_size, pmem_start)) +        return;

+    memblock_remove(pmem_start, pmem_size); +    pr_info("pmem reserved: 0x%016llx - 0x%016llx (%lld MB)\n", +        pmem_start, pmem_start + pmem_size, pmem_size >> 20); +    pmem_res.start = pmem_start; +    pmem_res.end = pmem_start + pmem_size - 1; +} +#endif

加个stub，下面不需要在加ifdef

是从stub改到ifdef的。。。我再看下有没有规范可以参考；

...

void __init arm64_memblock_init(void)   {       const s64 linear_region_size = BIT(vabits_actual - 1); @@ -646,6 +690,10 @@ void __init bootmem_init(void)       reserve_quick_kexec();   #endif   +#ifdef CONFIG_ARM64_PMEM_RESERVE +    reserve_pmem(); +#endif

reserve_pin_memory_res();         memblock_dump_all(); diff --git a/drivers/nvdimm/Kconfig b/drivers/nvdimm/Kconfig index b7d1eb3..ce5d4fd 100644 --- a/drivers/nvdimm/Kconfig +++ b/drivers/nvdimm/Kconfig @@ -132,3 +132,10 @@ config NVDIMM_TEST_BUILD         infrastructure.     endif

+config PMEM_LEGACY +    tristate "Pmem_legacy" +    depends on X86 || ARM64 +    select X86_PMEM_LEGACY if X86 +    select ARM64_PMEM_LEGACY_DEVICE if ARM64

diff --git a/drivers/nvdimm/Makefile b/drivers/nvdimm/Makefile index 0407753..6f8dc92 100644 --- a/drivers/nvdimm/Makefile +++ b/drivers/nvdimm/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_LIBNVDIMM) += libnvdimm.o   obj-$(CONFIG_BLK_DEV_PMEM) += nd_pmem.o   obj-$(CONFIG_ND_BTT) += nd_btt.o   obj-$(CONFIG_ND_BLK) += nd_blk.o +obj-$(CONFIG_PMEM_LEGACY) += nd_e820.o   obj-$(CONFIG_OF_PMEM) += of_pmem.o   obj-$(CONFIG_VIRTIO_PMEM) += virtio_pmem.o nd_virtio.o

.

On 2022/1/14 17:26, Kefeng Wang wrote:

On 2022/1/14 17:16, Zhuling wrote:

...

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4O31I?from=project-issue CVE: NA

Add "!" in memmap parameter, use memmap(memmap=nn[KMG]!ss[KMG]) reserve memory for pmem which register persistent memory in arm64.

Signed-off-by: Zhuling zhuling8@huawei.com

Documentation/admin-guide/kernel-parameters.txt | 9 +++++++--   arch/arm64/mm/init.c                            | 8 ++++++++   2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 64be32b..1953570 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2829,10 +2829,15 @@               will be eaten.         memmap=nn[KMG]!ss[KMG] -            [KNL,X86] Mark specific memory as protected. +            [KNL,X86,ARM64] Mark specific memory as protected.               Region of memory to be used, from ss to ss+nn. -            The memory region may be marked as e820 type 12 (0xc) +            [X86]The memory region may be marked as e820 type 12 (0xc)               and is NVDIMM or ADR memory. +            [ARM64]Reserve memory for persistent storage when the kernel +            restart or update. the data in PMEM will not be lost and can +            be loaded faster +            Example: +                memmap=100K!0x1a0000000

说明只支持1个，不支持多个配置

我在pmem里说明了下，OK，我在这里也添加说明下

...

memmap=<size>%<offset>-<oldtype>+<newtype>               [KNL,ACPI] Convert memory within the specified region diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 6ebfabd..b9eb312 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -56,6 +56,8 @@   s64 memstart_addr __ro_after_init = -1;   EXPORT_SYMBOL(memstart_addr);   +phys_addr_t start_at, mem_size;

为什么要加这个；全局的会覆盖函数内的声明；

好的，我下一版本修改下

...

#ifdef CONFIG_PIN_MEMORY   struct resource pin_memory_resource = {       .name = "Pin memory", @@ -111,6 +113,8 @@ static void __init reserve_pin_memory_res(void)    */   phys_addr_t arm64_dma_phys_limit __ro_after_init;   +static unsigned long long pmem_size, pmem_start;

可以用u64

OK，应该都可以，之前参考是看这个用的比较多

...

#ifndef CONFIG_KEXEC_CORE   static void __init reserve_crashkernel(void)   { @@ -442,6 +446,10 @@ static int __init parse_memmap_one(char *p)           start_at = memparse(p + 1, &p);           memblock_reserve(start_at, mem_size);           memblock_mark_memmap(start_at, mem_size); +    } else if (*p == '!') { +        start_at = memparse(p+1, &p); +        pmem_start = start_at; +        pmem_size = mem_size;       } else           pr_info("Unrecognized memmap option, please check the parameter.\n");

.