January 2022 - Kernel - mailweb.openeuler.org

[PATCH openEuler-1.0-LTS] ip_gre: validate csum_start only on pull
by Yang Yingliang 18 Jan '22

18 Jan '22

From: Willem de Bruijn <willemb(a)google.com> mainline inclusion from mainline-v5.14 commit 8a0ed250f911da31a2aef52101bc707846a800ff category: bugfix bugzilla: NA CVE: CVE-2021-39633 ------------------------------------------------- The GRE tunnel device can pull existing outer headers in ipge_xmit. This is a rare path, apparently unique to this device. The below commit ensured that pulling does not move skb->data beyond csum_start. But it has a false positive if ip_summed is not CHECKSUM_PARTIAL and thus csum_start is irrelevant. Refine to exclude this. At the same time simplify and strengthen the test. Simplify, by moving the check next to the offending pull, making it more self documenting and removing an unnecessary branch from other code paths. Strengthen, by also ensuring that the transport header is correct and therefore the inner headers will be after skb_reset_inner_headers. The transport header is set to csum_start in skb_partial_csum_set. Link: https://lore.kernel.org/netdev/YS+h%2FtqCJJiQei+W@shredder/ Fixes: 1d011c4803c7 ("ip_gre: add validation for csum_start") Reported-by: Ido Schimmel <idosch(a)idosch.org> Suggested-by: Alexander Duyck <alexander.duyck(a)gmail.com> Signed-off-by: Willem de Bruijn <willemb(a)google.com> Reviewed-by: Alexander Duyck <alexanderduyck(a)fb.com> Signed-off-by: David S. Miller <davem(a)davemloft.net> Signed-off-by: Huang Guobin <huangguobin4(a)huawei.com> Reviewed-by: Wei Yongjun <weiyongjun1(a)huawei.com> Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com> --- net/ipv4/ip_gre.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index a8a37d1128201..0c431fd4b1200 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -449,8 +449,6 @@ static void __gre_xmit(struct sk_buff *skb, struct net_device *dev, static int gre_handle_offloads(struct sk_buff *skb, bool csum) { - if (csum && skb_checksum_start(skb) < skb->data) - return -EINVAL; return iptunnel_handle_offloads(skb, csum ? SKB_GSO_GRE_CSUM : SKB_GSO_GRE); } @@ -682,15 +680,20 @@ static netdev_tx_t ipgre_xmit(struct sk_buff *skb, } if (dev->header_ops) { + const int pull_len = tunnel->hlen + sizeof(struct iphdr); + if (skb_cow_head(skb, 0)) goto free_skb; tnl_params = (const struct iphdr *)skb->data; + if (pull_len > skb_transport_offset(skb)) + goto free_skb; + /* Pull skb since ip_tunnel_xmit() needs skb->data pointing * to gre header. */ - skb_pull(skb, tunnel->hlen + sizeof(struct iphdr)); + skb_pull(skb, pull_len); skb_reset_mac_header(skb); } else { if (skb_cow_head(skb, dev->needed_headroom)) -- 2.25.1

1 0

[PATCH openEuler-5.10 01/82] clocksource/arm_arch_timer: Add build-time guards for unhandled register accesses
by Zheng Zengkai 17 Jan '22

17 Jan '22

From: Marc Zyngier <maz(a)kernel.org> mainline inclusion from mainline-v5.16-rc1 commit 4775bc63f880 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4QCBG CVE: NA -------------------------- As we are about to change the registers that are used by the driver, start by adding build-time checks to ensure that we always handle all registers and access modes. Suggested-by: Mark Rutland <mark.rutland(a)arm.com> Signed-off-by: Marc Zyngier <maz(a)kernel.org> Link: https://lore.kernel.org/r/20211017124225.3018098-2-maz@kernel.org Signed-off-by: Daniel Lezcano <daniel.lezcano(a)linaro.org> Signed-off-by: Xiongfeng Wang <wangxiongfeng2(a)huawei.com> Reviewed-by: Hanjun Guo <guohanjun(a)huawei.com> Signed-off-by: Zheng Zengkai <zhengzengkai(a)huawei.com> --- arch/arm/include/asm/arch_timer.h | 12 ++++++++++++ arch/arm64/include/asm/arch_timer.h | 13 ++++++++++++- drivers/clocksource/arm_arch_timer.c | 8 ++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/arch/arm/include/asm/arch_timer.h b/arch/arm/include/asm/arch_timer.h index 99175812d903..a5d27cff28fa 100644 --- a/arch/arm/include/asm/arch_timer.h +++ b/arch/arm/include/asm/arch_timer.h @@ -34,6 +34,8 @@ void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u32 val) case ARCH_TIMER_REG_TVAL: asm volatile("mcr p15, 0, %0, c14, c2, 0" : : "r" (val)); break; + default: + BUILD_BUG(); } } else if (access == ARCH_TIMER_VIRT_ACCESS) { switch (reg) { @@ -43,7 +45,11 @@ void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u32 val) case ARCH_TIMER_REG_TVAL: asm volatile("mcr p15, 0, %0, c14, c3, 0" : : "r" (val)); break; + default: + BUILD_BUG(); } + } else { + BUILD_BUG(); } isb(); @@ -62,6 +68,8 @@ u32 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg) case ARCH_TIMER_REG_TVAL: asm volatile("mrc p15, 0, %0, c14, c2, 0" : "=r" (val)); break; + default: + BUILD_BUG(); } } else if (access == ARCH_TIMER_VIRT_ACCESS) { switch (reg) { @@ -71,7 +79,11 @@ u32 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg) case ARCH_TIMER_REG_TVAL: asm volatile("mrc p15, 0, %0, c14, c3, 0" : "=r" (val)); break; + default: + BUILD_BUG(); } + } else { + BUILD_BUG(); } return val; diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index ce37c14f9417..7dfb7c6261bb 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -112,6 +112,8 @@ void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u32 val) case ARCH_TIMER_REG_TVAL: write_sysreg(val, cntp_tval_el0); break; + default: + BUILD_BUG(); } } else if (access == ARCH_TIMER_VIRT_ACCESS) { switch (reg) { @@ -121,7 +123,11 @@ void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u32 val) case ARCH_TIMER_REG_TVAL: write_sysreg(val, cntv_tval_el0); break; + default: + BUILD_BUG(); } + } else { + BUILD_BUG(); } isb(); @@ -136,6 +142,8 @@ u32 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg) return read_sysreg(cntp_ctl_el0); case ARCH_TIMER_REG_TVAL: return arch_timer_reg_read_stable(cntp_tval_el0); + default: + BUILD_BUG(); } } else if (access == ARCH_TIMER_VIRT_ACCESS) { switch (reg) { @@ -143,10 +151,13 @@ u32 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg) return read_sysreg(cntv_ctl_el0); case ARCH_TIMER_REG_TVAL: return arch_timer_reg_read_stable(cntv_tval_el0); + default: + BUILD_BUG(); } } - BUG(); + BUILD_BUG(); + unreachable(); } static inline u32 arch_timer_get_cntfrq(void) diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 3ea1c22314f9..f3c65e5181ec 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -103,6 +103,8 @@ void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val, case ARCH_TIMER_REG_TVAL: writel_relaxed(val, timer->base + CNTP_TVAL); break; + default: + BUILD_BUG(); } } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); @@ -113,6 +115,8 @@ void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val, case ARCH_TIMER_REG_TVAL: writel_relaxed(val, timer->base + CNTV_TVAL); break; + default: + BUILD_BUG(); } } else { arch_timer_reg_write_cp15(access, reg, val); @@ -134,6 +138,8 @@ u32 arch_timer_reg_read(int access, enum arch_timer_reg reg, case ARCH_TIMER_REG_TVAL: val = readl_relaxed(timer->base + CNTP_TVAL); break; + default: + BUILD_BUG(); } } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) { struct arch_timer *timer = to_arch_timer(clk); @@ -144,6 +150,8 @@ u32 arch_timer_reg_read(int access, enum arch_timer_reg reg, case ARCH_TIMER_REG_TVAL: val = readl_relaxed(timer->base + CNTV_TVAL); break; + default: + BUILD_BUG(); } } else { val = arch_timer_reg_read_cp15(access, reg); -- 2.20.1

1 81

[PATCH OLK-5.10 v3 1/3] arm64: Add memmap parameter for pmem
by Zhuling 17 Jan '22

17 Jan '22

hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4O31I?from=project-issue CVE: NA Add "!" in memmap parameter, use memmap(memmap=nn[KMG]!ss[KMG]) reserve memory for pmem which register persistent memory in arm64. Signed-off-by: Zhuling <zhuling8(a)huawei.com> --- Documentation/admin-guide/kernel-parameters.txt | 9 +++++++-- arch/arm64/mm/init.c | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 64be32b..1953570 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2829,10 +2829,15 @@ will be eaten. memmap=nn[KMG]!ss[KMG] - [KNL,X86] Mark specific memory as protected. + [KNL,X86,ARM64] Mark specific memory as protected. Region of memory to be used, from ss to ss+nn. - The memory region may be marked as e820 type 12 (0xc) + [X86]The memory region may be marked as e820 type 12 (0xc) and is NVDIMM or ADR memory. + [ARM64]Reserve memory for persistent storage when the kernel + restart or update. the data in PMEM will not be lost and can + be loaded faster + Example: + memmap=100K!0x1a0000000 memmap=<size>%<offset>-<oldtype>+<newtype> [KNL,ACPI] Convert memory within the specified region diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 6ebfabd..b9eb312 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -56,6 +56,8 @@ s64 memstart_addr __ro_after_init = -1; EXPORT_SYMBOL(memstart_addr); +phys_addr_t start_at, mem_size; + #ifdef CONFIG_PIN_MEMORY struct resource pin_memory_resource = { .name = "Pin memory", @@ -111,6 +113,8 @@ static void __init reserve_pin_memory_res(void) */ phys_addr_t arm64_dma_phys_limit __ro_after_init; +static unsigned long long pmem_size, pmem_start; + #ifndef CONFIG_KEXEC_CORE static void __init reserve_crashkernel(void) { @@ -442,6 +446,10 @@ static int __init parse_memmap_one(char *p) start_at = memparse(p + 1, &p); memblock_reserve(start_at, mem_size); memblock_mark_memmap(start_at, mem_size); + } else if (*p == '!') { + start_at = memparse(p+1, &p); + pmem_start = start_at; + pmem_size = mem_size; } else pr_info("Unrecognized memmap option, please check the parameter.\n"); -- 2.9.5

3 7

[PATCH openEuler-1.0-LTS 1/2] hugetlbfs: extend the definition of hugepages parameter to support node allocation
by Yang Yingliang 17 Jan '22

17 Jan '22

From: Zhenguo Yao <yaozhenguo1(a)gmail.com> mainline inclusion from mainline-v5.16-rc1 commit b5389086ad7be0453c55e0069a89856d1fbdf605 category: feature bugzilla: 186043 CVE: NA -------------------------------- We can specify the number of hugepages to allocate at boot. But the hugepages is balanced in all nodes at present. In some scenarios, we only need hugepages in one node. For example: DPDK needs hugepages which are in the same node as NIC. If DPDK needs four hugepages of 1G size in node1 and system has 16 numa nodes we must reserve 64 hugepages on the kernel cmdline. But only four hugepages are used. The others should be free after boot. If the system memory is low(for example: 64G), it will be an impossible task. So extend the hugepages parameter to support specifying hugepages on a specific node. For example add following parameter: hugepagesz=1G hugepages=0:1,1:3 It will allocate 1 hugepage in node0 and 3 hugepages in node1. Link: https://lkml.kernel.org/r/20211005054729.86457-1-yaozhenguo1@gmail.com Signed-off-by: Zhenguo Yao <yaozhenguo1(a)gmail.com> Reviewed-by: Mike Kravetz <mike.kravetz(a)oracle.com> Cc: Zhenguo Yao <yaozhenguo1(a)gmail.com> Cc: Dan Carpenter <dan.carpenter(a)oracle.com> Cc: Nathan Chancellor <nathan(a)kernel.org> Cc: Michael Ellerman <mpe(a)ellerman.id.au> Cc: Benjamin Herrenschmidt <benh(a)kernel.crashing.org> Cc: Paul Mackerras <paulus(a)samba.org> Cc: Jonathan Corbet <corbet(a)lwn.net> Cc: Mike Rapoport <rppt(a)kernel.org> Cc: Matthew Wilcox (Oracle) <willy(a)infradead.org> Signed-off-by: Andrew Morton <akpm(a)linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds(a)linux-foundation.org> Conflicts: Documentation/admin-guide/kernel-parameters.txt Documentation/admin-guide/mm/hugetlbpage.rst arch/powerpc/mm/hugetlbpage.c include/linux/hugetlb.h mm/hugetlb.c Signed-off-by: Liu Shixin <liushixin2(a)huawei.com> Reviewed-by: Kefeng Wang<wangkefeng.wang(a)huawei.com> Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com> --- .../admin-guide/kernel-parameters.txt | 5 + arch/powerpc/mm/hugetlbpage.c | 9 +- include/linux/hugetlb.h | 6 +- mm/hugetlb.c | 142 +++++++++++++++--- 4 files changed, 137 insertions(+), 25 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 420f7317aa6af..5623a9d1cf813 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1402,6 +1402,11 @@ registers. Default set by CONFIG_HPET_MMAP_DEFAULT. hugepages= [HW,X86-32,IA-64] HugeTLB pages to allocate at boot. + If using node format, the number of pages to allocate + per-node can be specified. + Format: <integer> or (node format) + <node>:<integer>[,<node>:<integer>] + hugepagesz= [HW,IA-64,PPC,X86-64] The size of the HugeTLB pages. On x86-64 and powerpc, this option can be specified multiple times interleaved with hugepages= to reserve diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index cef0b7ee10246..163b0ef7d156e 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -241,17 +241,22 @@ int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) m->hstate = hstate; return 1; } + +bool __init hugetlb_node_alloc_supported(void) +{ + return false; +} #endif -int __init alloc_bootmem_huge_page(struct hstate *h) +int __init alloc_bootmem_huge_page(struct hstate *h, int nid) { #ifdef CONFIG_PPC_BOOK3S_64 if (firmware_has_feature(FW_FEATURE_LPAR) && !radix_enabled()) return pseries_alloc_bootmem_huge_page(h); #endif - return __alloc_bootmem_huge_page(h); + return __alloc_bootmem_huge_page(h, nid); } #if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx) diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 561e7679fadc4..588f9f2a44fce 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -347,6 +347,7 @@ struct hstate { unsigned long nr_overcommit_huge_pages; struct list_head hugepage_activelist; struct list_head hugepage_freelists[MAX_NUMNODES]; + unsigned int max_huge_pages_node[MAX_NUMNODES]; unsigned int nr_huge_pages_node[MAX_NUMNODES]; unsigned int free_huge_pages_node[MAX_NUMNODES]; unsigned int surplus_huge_pages_node[MAX_NUMNODES]; @@ -409,8 +410,9 @@ int hugetlb_insert_hugepage(struct vm_area_struct *vma, unsigned long addr, struct page *hpage, pgprot_t prot); /* arch callback */ -int __init __alloc_bootmem_huge_page(struct hstate *h); -int __init alloc_bootmem_huge_page(struct hstate *h); +int __init __alloc_bootmem_huge_page(struct hstate *h, int nid); +int __init alloc_bootmem_huge_page(struct hstate *h, int nid); +bool __init hugetlb_node_alloc_supported(void); void __init hugetlb_bad_size(void); void __init hugetlb_add_hstate(unsigned order); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index b2ed9174bfd73..8286a6f8cb050 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -56,6 +56,7 @@ static struct hstate * __initdata parsed_hstate; static unsigned long __initdata default_hstate_max_huge_pages; static unsigned long __initdata default_hstate_size; static bool __initdata parsed_valid_hugepagesz = true; +static unsigned int default_hugepages_in_node[MAX_NUMNODES] __initdata; int enable_charge_mighp __read_mostly; /* @@ -2241,33 +2242,39 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, return ERR_PTR(-ENOSPC); } -int alloc_bootmem_huge_page(struct hstate *h) +int alloc_bootmem_huge_page(struct hstate *h, int nid) __attribute__ ((weak, alias("__alloc_bootmem_huge_page"))); -int __alloc_bootmem_huge_page(struct hstate *h) +int __alloc_bootmem_huge_page(struct hstate *h, int nid) { - struct huge_bootmem_page *m; + struct huge_bootmem_page *m = NULL; /* initialize for clang */ int nr_nodes, node; + if (nid >= nr_online_nodes) + return 0; + /* do node specific alloc */ + if (nid != NUMA_NO_NODE) { + m = memblock_virt_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h), + 0, MEMBLOCK_ALLOC_ACCESSIBLE, nid); + if (!m) + return 0; + goto found; + } + /* allocate from next node when distributing huge pages */ for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) { - void *addr; - - addr = memblock_virt_alloc_try_nid_raw( + m = memblock_virt_alloc_try_nid_raw( huge_page_size(h), huge_page_size(h), 0, BOOTMEM_ALLOC_ACCESSIBLE, node); - if (addr) { - /* - * Use the beginning of the huge page to store the - * huge_bootmem_page struct (until gather_bootmem - * puts them into the mem_map). - */ - m = addr; - goto found; - } + /* + * Use the beginning of the huge page to store the + * huge_bootmem_page struct (until gather_bootmem + * puts them into the mem_map). + */ + if (!m) + return 0; + goto found; } - return 0; found: - BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h))); /* Put them into a private list first because mem_map is not up yet */ INIT_LIST_HEAD(&m->list); list_add(&m->list, &huge_boot_pages); @@ -2310,12 +2317,55 @@ static void __init gather_bootmem_prealloc(void) cond_resched(); } } +static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid) +{ + unsigned long i; + char buf[32]; + + for (i = 0; i < h->max_huge_pages_node[nid]; ++i) { + if (hstate_is_gigantic(h)) { + if (!alloc_bootmem_huge_page(h, nid)) + break; + } else { + struct page *page; + gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE; + + page = alloc_fresh_huge_page(h, gfp_mask, nid, + &node_states[N_MEMORY], NULL); + if (!page) + break; + put_page(page); /* free it into the hugepage allocator */ + } + cond_resched(); + } + if (i == h->max_huge_pages_node[nid]) + return; + + string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32); + pr_warn("HugeTLB: allocating %u of page size %s failed node%d. Only allocated %lu hugepages.\n", + h->max_huge_pages_node[nid], buf, nid, i); + h->max_huge_pages -= (h->max_huge_pages_node[nid] - i); + h->max_huge_pages_node[nid] = i; +} static void __init hugetlb_hstate_alloc_pages(struct hstate *h) { unsigned long i; nodemask_t *node_alloc_noretry; + bool node_specific_alloc = false; + + /* do node specific alloc */ + for (i = 0; i < nr_online_nodes; i++) { + if (h->max_huge_pages_node[i] > 0) { + hugetlb_hstate_alloc_pages_onenode(h, i); + node_specific_alloc = true; + } + } + + if (node_specific_alloc) + return; + /* below will do all node balanced alloc */ if (!hstate_is_gigantic(h)) { /* * Bit mask controlling how hard we retry per-node allocations. @@ -2336,7 +2386,7 @@ static void __init hugetlb_hstate_alloc_pages(struct hstate *h) for (i = 0; i < h->max_huge_pages; ++i) { if (hstate_is_gigantic(h)) { - if (!alloc_bootmem_huge_page(h)) + if (!alloc_bootmem_huge_page(h, NUMA_NO_NODE)) break; } else if (!alloc_pool_huge_page(h, &node_states[N_MEMORY], @@ -2991,8 +3041,13 @@ static int __init hugetlb_init(void) } default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size)); if (default_hstate_max_huge_pages) { - if (!default_hstate.max_huge_pages) + if (!default_hstate.max_huge_pages) { default_hstate.max_huge_pages = default_hstate_max_huge_pages; + + for (i = 0; i < nr_online_nodes; i++) + default_hstate.max_huge_pages_node[i] = + default_hugepages_in_node[i]; + } } hugetlb_init_hstates(); @@ -3052,10 +3107,19 @@ void __init hugetlb_add_hstate(unsigned int order) parsed_hstate = h; } +bool __init __weak hugetlb_node_alloc_supported(void) +{ + return true; +} + static int __init hugetlb_nrpages_setup(char *s) { unsigned long *mhp; static unsigned long *last_mhp; + int node = NUMA_NO_NODE; + int count; + unsigned long tmp; + char *p = s; if (!parsed_valid_hugepagesz) { pr_warn("hugepages = %s preceded by " @@ -3077,8 +3141,40 @@ static int __init hugetlb_nrpages_setup(char *s) return 1; } - if (sscanf(s, "%lu", mhp) <= 0) - *mhp = 0; + while (*p) { + count = 0; + if (sscanf(p, "%lu%n", &tmp, &count) != 1) + goto invalid; + /* Parameter is node format */ + if (p[count] == ':') { + if (!hugetlb_node_alloc_supported()) { + pr_warn("HugeTLB: architecture can't support node specific alloc, ignoring!\n"); + return 0; + } + node = tmp; + p += count + 1; + if (node < 0 || node >= nr_online_nodes) + goto invalid; + /* Parse hugepages */ + if (sscanf(p, "%lu%n", &tmp, &count) != 1) + goto invalid; + if (!hugetlb_max_hstate) + default_hugepages_in_node[node] = tmp; + else + parsed_hstate->max_huge_pages_node[node] = tmp; + *mhp += tmp; + /* Go to parse next node*/ + if (p[count] == ',') + p += count + 1; + else + break; + } else { + if (p != s) + goto invalid; + *mhp = tmp; + break; + } + } /* * Global state is always initialized later in hugetlb_init. @@ -3091,6 +3187,10 @@ static int __init hugetlb_nrpages_setup(char *s) last_mhp = mhp; return 1; + +invalid: + pr_warn("HugeTLB: Invalid hugepages parameter %s\n", p); + return 0; } __setup("hugepages=", hugetlb_nrpages_setup); -- 2.25.1

1 1

回复:答复: openEuler kernel技术分享第十七期 & 双周例会
by Zhengyuan Liu 17 Jan '22

17 Jan '22

你好 rdma这个议题什么气候开始呢 ---------- 该邮件从移动设备发送 --------------原始邮件-------------- 发件人："tangjiayuan "<tangjiayuan(a)huawei.com>gt;; 发送时间：2022年1月17日(星期一) 下午3:10 收件人："kernel(a)openeuler.org" <kernel(a)openeuler.org>gt;;"kernel-discuss(a)openeuler.org" <kernel-discuss(a)openeuler.org>gt;; 抄送："xie xiuqi "<xiexiuqi(a)huawei.com>gt;;"chengjian (D) "<cj.chengjian(a)huawei.com>gt;;"qiulaibin "<qiulaibin(a)huawei.com>gt;; 主题：答复: openEuler kernel技术分享第十七期 & 双周例会 ----------------------------------- Kernel SIG增加一个议题：希望在内核态支持RDMA通信 -----邮件原件----- 发件人: openEuler conference [mailto:public@openeuler.org] 发送时间: 2021年12月27日 10:20 收件人: kernel(a)openeuler.org; tc(a)openeuler.org; dev(a)openeuler.org; kernel-discuss(a)openeuler.org 主题: openEuler kernel技术分享第十七期 & 双周例会您好！ Kernel SIG 邀请您参加 2021-12-31 14:00 召开的ZOOM会议(自动录制) 会议主题：openEuler kernel技术分享第十七期 & 双周例会会议内容： 14:10-15:30 openEuler kernel 技术分享第17期 - cgroup介绍会议链接：https://us06web.zoom.us/j/83118407147?pwd=elVzMVhGc2YzaDQ1TGorN2tPa1NhZz09 温馨提醒：建议接入会议后修改参会人的姓名，也可以使用您在gitee.com的ID 更多资讯尽在：https://openeuler.org/zh/ Hello! openEuler Kernel SIG invites you to attend the ZOOM conference(auto recording) will be held at 2021-12-31 14:00, The subject of the conference is openEuler kernel技术分享第十七期 & 双周例会, Summary: 14:10-15:30 openEuler kernel 技术分享第17期 - cgroup介绍 You can join the meeting at https://us06web.zoom.us/j/83118407147?pwd=elVzMVhGc2YzaDQ1TGorN2tPa1NhZz09. Note: You are advised to change the participant name after joining the conference or use your ID at gitee.com. More information: https://openeuler.org/en/ _______________________________________________ Kernel-discuss mailing list -- kernel-discuss(a)openeuler.org To unsubscribe send an email to kernel-discuss-leave(a)openeuler.org _______________________________________________ Kernel mailing list -- kernel(a)openeuler.org To unsubscribe send an email to kernel-leave(a)openeuler.org

1 0

[PATCH openEuler-1.0-LTS 1/2] share pool: use rwsem to protect sp group exit
by Yang Yingliang 17 Jan '22

17 Jan '22

From: Guo Mengqi <guomengqi3(a)huawei.com> ascend inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4ODMN CVE: NA ------------------------------------------------- Fix following situation: when the last process in a group exits, and a second process tries to add to this group. The second process may get a invalid spg. However the group's use_count is increased by 1, which caused the first process failed to free the group when it exits. And then second process called sp_group_drop --> free_sp_group and cause a double request of rwsem. Signed-off-by: Guo Mengqi <guomengqi3(a)huawei.com> Reviewed-by: Weilong Chen <chenweilong(a)huawei.com> Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com> --- mm/share_pool.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/mm/share_pool.c b/mm/share_pool.c index 428742f5d4044..c79ea4bb06026 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -711,20 +711,25 @@ static void free_new_spg_id(bool new, int spg_id) free_sp_group_id(spg_id); } -static void free_sp_group(struct sp_group *spg) +static void free_sp_group_locked(struct sp_group *spg) { fput(spg->file); fput(spg->file_hugetlb); free_spg_stat(spg->id); - down_write(&sp_group_sem); idr_remove(&sp_group_idr, spg->id); - up_write(&sp_group_sem); free_sp_group_id((unsigned int)spg->id); kfree(spg); system_group_count--; WARN(system_group_count < 0, "unexpected group count\n"); } +static void free_sp_group(struct sp_group *spg) +{ + down_write(&sp_group_sem); + free_sp_group_locked(spg); + up_write(&sp_group_sem); +} + static void sp_group_drop(struct sp_group *spg) { if (atomic_dec_and_test(&spg->use_count)) @@ -4460,14 +4465,15 @@ void sp_group_post_exit(struct mm_struct *mm) sp_proc_stat_drop(stat); } - /* lockless traverse */ + down_write(&sp_group_sem); list_for_each_entry_safe(spg_node, tmp, &master->node_list, group_node) { spg = spg_node->spg; /* match with refcount inc in sp_group_add_task */ - sp_group_drop(spg); + if (atomic_dec_and_test(&spg->use_count)) + free_sp_group_locked(spg); kfree(spg_node); } - + up_write(&sp_group_sem); kfree(master); } -- 2.25.1

1 1

[PATCH openEuler-1.0-LTS 1/9] Revert "svm: Add svm_set_user_mpam_en to enable/disable mpam for smmu"
by Yang Yingliang 17 Jan '22

17 Jan '22

From: Xingang Wang <wangxingang5(a)huawei.com> ascend inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I49RB2 CVE: NA ------------------------------------------------- This reverts commit c50ad40d63ef0cc9e0b158354af08f1f98419273. The commit "svm: Add svm_set_user_mpam_en to enable/disable mpam for smmu" add and export interface in svm module, this makes the mpam depend on the svm module, just revert this to avoid coupling. Signed-off-by: Xingang Wang <wangxingang5(a)huawei.com> Reviewed-by: Weilong Chen <chenweilong(a)huawei.com> Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com> --- drivers/char/svm.c | 64 ------------------------------------- include/linux/ascend_smmu.h | 3 -- 2 files changed, 67 deletions(-) diff --git a/drivers/char/svm.c b/drivers/char/svm.c index c877c75fdcd5f..000b9ad01c350 100644 --- a/drivers/char/svm.c +++ b/drivers/char/svm.c @@ -135,14 +135,11 @@ struct meminfo { struct svm_mpam { #define SVM_GET_DEV_MPAM (1 << 0) #define SVM_SET_DEV_MPAM (1 << 1) -#define SVM_GET_USER_MPAM_EN (1 << 2) -#define SVM_SET_USER_MPAM_EN (1 << 3) int flags; int pasid; int partid; int pmg; int s1mpam; - int user_mpam_en; }; struct phymeminfo { @@ -1505,14 +1502,6 @@ static int svm_get_core_mpam(struct device *dev, void *data) } } - if (mpam->flags & SVM_GET_USER_MPAM_EN) { - err = arm_smmu_get_dev_user_mpam_en(dev, &mpam->user_mpam_en); - if (err) { - dev_err(dev, "set user_mpam_en failed, %d\n", err); - return err; - } - } - return err; } @@ -1554,14 +1543,6 @@ static int svm_set_core_mpam(struct device *dev, void *data) } } - if (mpam->flags & SVM_SET_USER_MPAM_EN) { - err = arm_smmu_set_dev_user_mpam_en(dev, mpam->user_mpam_en); - if (err) { - dev_err(dev, "set user_mpam_en failed, %d\n", err); - return err; - } - } - return 0; } @@ -1655,51 +1636,6 @@ int svm_get_mpam(int pasid, int *partid, int *pmg, int *s1mpam) } EXPORT_SYMBOL_GPL(svm_get_mpam); -/** - * svm_set_user_mpam_en() - set user_mpam_en - * @user_mpam_en: 0 for smmu mpam, 1 for user mpam - */ -int svm_set_user_mpam_en(int user_mpam_en) -{ - int err; - struct svm_mpam mpam, old_mpam; - - old_mpam.flags = SVM_GET_USER_MPAM_EN; - err = __svm_get_mpam(&old_mpam); - - mpam.flags = SVM_SET_USER_MPAM_EN, - mpam.user_mpam_en = user_mpam_en, - err = __svm_set_mpam(&mpam); - if (err) - goto rollback; - - return 0; - -rollback: - __svm_set_mpam(&mpam); - return err; -} -EXPORT_SYMBOL_GPL(svm_set_user_mpam_en); - -/** - * svm_set_user_mpam_en() - set user_mpam_en - * @user_mpam_en: pointer to user_mpam_en - */ -int svm_get_user_mpam_en(int *user_mpam_en) -{ - int err; - struct svm_mpam mpam; - - mpam.flags = SVM_GET_USER_MPAM_EN; - err = __svm_get_mpam(&mpam); - if (err) - return err; - - *user_mpam_en = mpam.user_mpam_en; - return 0; -} -EXPORT_SYMBOL_GPL(svm_get_user_mpam_en); - static int svm_set_rc(unsigned long __user *arg) { unsigned long addr, size, rc; diff --git a/include/linux/ascend_smmu.h b/include/linux/ascend_smmu.h index 32241a02a8a89..e570acfc8f6ac 100644 --- a/include/linux/ascend_smmu.h +++ b/include/linux/ascend_smmu.h @@ -15,7 +15,4 @@ extern int arm_smmu_get_dev_user_mpam_en(struct device *dev, int *user_mpam_en); extern int svm_get_mpam(int pasid, int *partid, int *pmg, int *s1mpam); extern int svm_set_mpam(int pasid, int partid, int pmg, int s1mpam); -extern int svm_set_user_mpam_en(int user_mpam_en); -extern int svm_get_user_mpam_en(int *user_mpam_en); - #endif /* __LINUX_ASCEND_SMMU_H */ -- 2.25.1

1 8

[PATCH openEuler-1.0-LTS 1/3] Revert "svm: Add svm_set_user_mpam_en to enable/disable mpam for smmu"
by Yang Yingliang 17 Jan '22

17 Jan '22

From: Xingang Wang <wangxingang5(a)huawei.com> ascend inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I49RB2 CVE: NA ------------------------------------------------- This reverts commit c50ad40d63ef0cc9e0b158354af08f1f98419273. The commit "svm: Add svm_set_user_mpam_en to enable/disable mpam for smmu" add and export interface in svm module, this makes the mpam depend on the svm module, just revert this to avoid coupling. Signed-off-by: Xingang Wang <wangxingang5(a)huawei.com> Reviewed-by: Weilong Chen <chenweilong(a)huawei.com> Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com> --- drivers/char/svm.c | 64 ------------------------------------- include/linux/ascend_smmu.h | 3 -- 2 files changed, 67 deletions(-) diff --git a/drivers/char/svm.c b/drivers/char/svm.c index c877c75fdcd5f..000b9ad01c350 100644 --- a/drivers/char/svm.c +++ b/drivers/char/svm.c @@ -135,14 +135,11 @@ struct meminfo { struct svm_mpam { #define SVM_GET_DEV_MPAM (1 << 0) #define SVM_SET_DEV_MPAM (1 << 1) -#define SVM_GET_USER_MPAM_EN (1 << 2) -#define SVM_SET_USER_MPAM_EN (1 << 3) int flags; int pasid; int partid; int pmg; int s1mpam; - int user_mpam_en; }; struct phymeminfo { @@ -1505,14 +1502,6 @@ static int svm_get_core_mpam(struct device *dev, void *data) } } - if (mpam->flags & SVM_GET_USER_MPAM_EN) { - err = arm_smmu_get_dev_user_mpam_en(dev, &mpam->user_mpam_en); - if (err) { - dev_err(dev, "set user_mpam_en failed, %d\n", err); - return err; - } - } - return err; } @@ -1554,14 +1543,6 @@ static int svm_set_core_mpam(struct device *dev, void *data) } } - if (mpam->flags & SVM_SET_USER_MPAM_EN) { - err = arm_smmu_set_dev_user_mpam_en(dev, mpam->user_mpam_en); - if (err) { - dev_err(dev, "set user_mpam_en failed, %d\n", err); - return err; - } - } - return 0; } @@ -1655,51 +1636,6 @@ int svm_get_mpam(int pasid, int *partid, int *pmg, int *s1mpam) } EXPORT_SYMBOL_GPL(svm_get_mpam); -/** - * svm_set_user_mpam_en() - set user_mpam_en - * @user_mpam_en: 0 for smmu mpam, 1 for user mpam - */ -int svm_set_user_mpam_en(int user_mpam_en) -{ - int err; - struct svm_mpam mpam, old_mpam; - - old_mpam.flags = SVM_GET_USER_MPAM_EN; - err = __svm_get_mpam(&old_mpam); - - mpam.flags = SVM_SET_USER_MPAM_EN, - mpam.user_mpam_en = user_mpam_en, - err = __svm_set_mpam(&mpam); - if (err) - goto rollback; - - return 0; - -rollback: - __svm_set_mpam(&mpam); - return err; -} -EXPORT_SYMBOL_GPL(svm_set_user_mpam_en); - -/** - * svm_set_user_mpam_en() - set user_mpam_en - * @user_mpam_en: pointer to user_mpam_en - */ -int svm_get_user_mpam_en(int *user_mpam_en) -{ - int err; - struct svm_mpam mpam; - - mpam.flags = SVM_GET_USER_MPAM_EN; - err = __svm_get_mpam(&mpam); - if (err) - return err; - - *user_mpam_en = mpam.user_mpam_en; - return 0; -} -EXPORT_SYMBOL_GPL(svm_get_user_mpam_en); - static int svm_set_rc(unsigned long __user *arg) { unsigned long addr, size, rc; diff --git a/include/linux/ascend_smmu.h b/include/linux/ascend_smmu.h index 32241a02a8a89..e570acfc8f6ac 100644 --- a/include/linux/ascend_smmu.h +++ b/include/linux/ascend_smmu.h @@ -15,7 +15,4 @@ extern int arm_smmu_get_dev_user_mpam_en(struct device *dev, int *user_mpam_en); extern int svm_get_mpam(int pasid, int *partid, int *pmg, int *s1mpam); extern int svm_set_mpam(int pasid, int partid, int pmg, int s1mpam); -extern int svm_set_user_mpam_en(int user_mpam_en); -extern int svm_get_user_mpam_en(int *user_mpam_en); - #endif /* __LINUX_ASCEND_SMMU_H */ -- 2.25.1

1 2

[PATCH openEuler-1.0-LTS 1/7] cgroup: Use open-time credentials for process migraton perm checks
by Yang Yingliang 17 Jan '22

17 Jan '22

From: Tejun Heo <tj(a)kernel.org> mainline inclusion from mainline-v5.16-9 commit 1756d7994ad85c2479af6ae5a9750b92324685af category: bugfix bugzilla: NA CVE: CVE-2021-4197 --------------------------------------------------- cgroup process migration permission checks are performed at write time as whether a given operation is allowed or not is dependent on the content of the write - the PID. This currently uses current's credentials which is a potential security weakness as it may allow scenarios where a less privileged process tricks a more privileged one into writing into a fd that it created. This patch makes both cgroup2 and cgroup1 process migration interfaces to use the credentials saved at the time of open (file->f_cred) instead of current's. Reported-by: "Eric W. Biederman" <ebiederm(a)xmission.com> Suggested-by: Linus Torvalds <torvalds(a)linuxfoundation.org> Fixes: 187fe84067bd ("cgroup: require write perm on common ancestor when moving processes on the default hierarchy") Reviewed-by: Michal Koutný <mkoutny(a)suse.com> Signed-off-by: Tejun Heo <tj(a)kernel.org> Conflicts: kernel/cgroup/cgroup.c Signed-off-by: Lu Jialin <lujialin4(a)huawei.com> Reviewed-by: Xiu Jianfeng <xiujianfeng(a)huawei.com> Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com> --- kernel/cgroup/cgroup-v1.c | 7 ++++--- kernel/cgroup/cgroup.c | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/kernel/cgroup/cgroup-v1.c b/kernel/cgroup/cgroup-v1.c index 055063c9c2a38..7a3ea4d6b54cf 100644 --- a/kernel/cgroup/cgroup-v1.c +++ b/kernel/cgroup/cgroup-v1.c @@ -535,10 +535,11 @@ static ssize_t __cgroup1_procs_write(struct kernfs_open_file *of, goto out_unlock; /* - * Even if we're attaching all tasks in the thread group, we only - * need to check permissions on one of them. + * Even if we're attaching all tasks in the thread group, we only need + * to check permissions on one of them. Check permissions using the + * credentials from file open to protect against inherited fd attacks. */ - cred = current_cred(); + cred = of->file->f_cred; tcred = get_task_cred(task); if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) && !uid_eq(cred->euid, tcred->uid) && diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 6f4dcdd6f77b2..49e8bf7ee9c73 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -4490,6 +4490,7 @@ static ssize_t cgroup_procs_write(struct kernfs_open_file *of, { struct cgroup *src_cgrp, *dst_cgrp; struct task_struct *task; + const struct cred *saved_cred; ssize_t ret; dst_cgrp = cgroup_kn_lock_live(of->kn, false); @@ -4506,8 +4507,16 @@ static ssize_t cgroup_procs_write(struct kernfs_open_file *of, src_cgrp = task_cgroup_from_root(task, &cgrp_dfl_root); spin_unlock_irq(&css_set_lock); + /* + * Process and thread migrations follow same delegation rule. Check + * permissions using the credentials from file open to protect against + * inherited fd attacks. + */ + + saved_cred = override_creds(of->file->f_cred); ret = cgroup_procs_write_permission(src_cgrp, dst_cgrp, of->file->f_path.dentry->d_sb); + revert_creds(saved_cred); if (ret) goto out_finish; @@ -4531,6 +4540,7 @@ static ssize_t cgroup_threads_write(struct kernfs_open_file *of, { struct cgroup *src_cgrp, *dst_cgrp; struct task_struct *task; + const struct cred *saved_cred; ssize_t ret; buf = strstrip(buf); @@ -4549,9 +4559,16 @@ static ssize_t cgroup_threads_write(struct kernfs_open_file *of, src_cgrp = task_cgroup_from_root(task, &cgrp_dfl_root); spin_unlock_irq(&css_set_lock); - /* thread migrations follow the cgroup.procs delegation rule */ + /* + * Process and thread migrations follow same delegation rule. Check + * permissions using the credentials from file open to protect against + * inherited fd attacks. + */ + + saved_cred = override_creds(of->file->f_cred); ret = cgroup_procs_write_permission(src_cgrp, dst_cgrp, of->file->f_path.dentry->d_sb); + revert_creds(saved_cred); if (ret) goto out_finish; -- 2.25.1

1 6

[PATCH openEuler-5.10 1/2] x86: hugepage: use nt copy hugepage to AEP in x86
by Zheng Zengkai 15 Jan '22

15 Jan '22

From: Kemeng Shi <shikemeng(a)huawei.com> euleros inclusion category: feature feature: etmem bugzilla: https://gitee.com/openeuler/kernel/issues/I4OODH?from=project-issue CVE: NA ------------------------------------------------- Add proc/sys/vm/hugepage_nocache_copy switch. Set 1 to copy hugepage with movnt SSE instructoin if cpu support it. Set 0 to copy hugepage as usual. Signed-off-by: Kemeng Shi <shikemeng(a)huawei.com> Reviewed-by: louhongxiang <louhongxiang(a)huawei.com> Reviewed-by: Kefeng Wang <wangkefeng.wang(a)huawei.com> Signed-off-by: Zheng Zengkai <zhengzengkai(a)huawei.com> --- arch/x86/include/asm/page_64.h | 6 ++ arch/x86/lib/Makefile | 1 + arch/x86/lib/copy_highpages.c | 105 +++++++++++++++++++++++++++++++++ arch/x86/lib/copy_page_64.S | 73 +++++++++++++++++++++++ include/linux/highmem.h | 14 +++++ mm/migrate.c | 6 +- 6 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 arch/x86/lib/copy_highpages.c diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h index 939b1cff4a7b..d9fbc486728a 100644 --- a/arch/x86/include/asm/page_64.h +++ b/arch/x86/include/asm/page_64.h @@ -56,6 +56,12 @@ static inline void clear_page(void *page) void copy_page(void *to, void *from); +void copy_page_nocache(void *to, void *from); +void copy_page_nocache_barrir(void); + +struct page; +#define __HAVE_ARCH_COPY_HUGEPAGES 1 +void copy_highpages(struct page *to, struct page *from, int nr_pages); #endif /* !__ASSEMBLY__ */ #ifdef CONFIG_X86_VSYSCALL_EMULATION diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index bad4dee4f0e4..ab0d91b808be 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -70,4 +70,5 @@ else lib-y += memmove_64.o memset_64.o lib-y += copy_user_64.o lib-y += cmpxchg16b_emu.o + lib-y += copy_highpages.o endif diff --git a/arch/x86/lib/copy_highpages.c b/arch/x86/lib/copy_highpages.c new file mode 100644 index 000000000000..ed95b8a7b906 --- /dev/null +++ b/arch/x86/lib/copy_highpages.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * accelerate copying page to pmem with non-temproal stroes + */ +#include <linux/sched.h> +#include <linux/mmzone.h> +#include <linux/highmem.h> +#include <linux/sysctl.h> + +DEFINE_STATIC_KEY_FALSE(hugepage_nocache_copy); +#ifdef CONFIG_SYSCTL +static void set_hugepage_nocache_copy(bool enabled) +{ + if (enabled) + static_branch_enable(&hugepage_nocache_copy); + else + static_branch_disable(&hugepage_nocache_copy); +} + +int sysctl_hugepage_nocache_copy(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table t; + int err; + int state; + + if (write && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + state = static_branch_unlikely(&hugepage_nocache_copy); + t = *table; + t.data = &state; + err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); + if (err < 0) + return err; + if (write) + set_hugepage_nocache_copy(state); + return err; +} + +static struct ctl_table copy_highpages_table[] = { + { + .procname = "hugepage_nocache_copy", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0600, + .proc_handler = sysctl_hugepage_nocache_copy, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + {} +}; + +static struct ctl_table copy_highpages_root_table[] = { + { + .procname = "vm", + .mode = 0555, + .child = copy_highpages_table, + }, + {} +}; + +static __init int copy_highpages_init(void) +{ + return register_sysctl_table(copy_highpages_root_table) ? 0 : -ENOMEM; +} +__initcall(copy_highpages_init); +#endif + +static void copy_highpages_nocache(struct page *to, struct page *from, int nr_pages) +{ + char *vfrom, *vto; + int i; + + for (i = 0; i < nr_pages; i++) { + cond_resched(); + vfrom = kmap_atomic(from); + vto = kmap_atomic(to); + copy_page_nocache(vto, vfrom); + kunmap_atomic(vto); + kunmap_atomic(vfrom); + to++; + from++; + } + copy_page_nocache_barrir(); +} + +static void copy_highpages_cache(struct page *to, struct page *from, int nr_pages) +{ + int i; + + for (i = 0; i < nr_pages; i++) { + cond_resched(); + copy_highpage(to + i, from + i); + } +} + +void copy_highpages(struct page *to, struct page *from, int nr_pages) +{ + if (static_branch_unlikely(&hugepage_nocache_copy) && + get_node_type(page_to_nid(to)) == NODE_TYPE_PMEM) + return copy_highpages_nocache(to, from, nr_pages); + + return copy_highpages_cache(to, from, nr_pages); +} diff --git a/arch/x86/lib/copy_page_64.S b/arch/x86/lib/copy_page_64.S index 2402d4c489d2..16ca196d349b 100644 --- a/arch/x86/lib/copy_page_64.S +++ b/arch/x86/lib/copy_page_64.S @@ -87,3 +87,76 @@ SYM_FUNC_START_LOCAL(copy_page_regs) addq $2*8, %rsp ret SYM_FUNC_END(copy_page_regs) + +SYM_FUNC_START(copy_page_nocache) + ALTERNATIVE "jmp copy_page", "", X86_FEATURE_XMM2 + subq $2*8, %rsp + movq %rbx, (%rsp) + movq %r12, 1*8(%rsp) + + movl $(4096/64)-5, %ecx + .p2align 4 +.LoopNT64: + dec %rcx + movq 0x8*0(%rsi), %rax + movq 0x8*1(%rsi), %rbx + movq 0x8*2(%rsi), %rdx + movq 0x8*3(%rsi), %r8 + movq 0x8*4(%rsi), %r9 + movq 0x8*5(%rsi), %r10 + movq 0x8*6(%rsi), %r11 + movq 0x8*7(%rsi), %r12 + + prefetcht0 5*64(%rsi) + + movnti %rax, 0x8*0(%rdi) + movnti %rbx, 0x8*1(%rdi) + movnti %rdx, 0x8*2(%rdi) + movnti %r8, 0x8*3(%rdi) + movnti %r9, 0x8*4(%rdi) + movnti %r10, 0x8*5(%rdi) + movnti %r11, 0x8*6(%rdi) + movnti %r12, 0x8*7(%rdi) + + leaq 64 (%rsi), %rsi + leaq 64 (%rdi), %rdi + + jnz .LoopNT64 + + movl $5, %ecx + .p2align 4 +.LoopNT2: + decl %ecx + + movq 0x8*0(%rsi), %rax + movq 0x8*1(%rsi), %rbx + movq 0x8*2(%rsi), %rdx + movq 0x8*3(%rsi), %r8 + movq 0x8*4(%rsi), %r9 + movq 0x8*5(%rsi), %r10 + movq 0x8*6(%rsi), %r11 + movq 0x8*7(%rsi), %r12 + + movnti %rax, 0x8*0(%rdi) + movnti %rbx, 0x8*1(%rdi) + movnti %rdx, 0x8*2(%rdi) + movnti %r8, 0x8*3(%rdi) + movnti %r9, 0x8*4(%rdi) + movnti %r10, 0x8*5(%rdi) + movnti %r11, 0x8*6(%rdi) + movnti %r12, 0x8*7(%rdi) + + leaq 64(%rdi), %rdi + leaq 64(%rsi), %rsi + jnz .LoopNT2 + + movq (%rsp), %rbx + movq 1*8(%rsp), %r12 + addq $2*8, %rsp + ret +SYM_FUNC_END(copy_page_nocache) + +SYM_FUNC_START(copy_page_nocache_barrir) + ALTERNATIVE "", "sfence", X86_FEATURE_XMM2 + ret +SYM_FUNC_END(copy_page_nocache_barrir) diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 8e21fe82b3a3..6b27af8fe624 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -356,4 +356,18 @@ static inline void copy_highpage(struct page *to, struct page *from) #endif +#ifndef __HAVE_ARCH_COPY_HUGEPAGES + +static inline void copy_highpages(struct page *to, struct page *from, int nr_pages) +{ + int i; + + for (i = 0; i < nr_pages; i++) { + cond_resched(); + copy_highpage(to + i, from + i); + } +} + +#endif /* __HAVE_ARCH_COPY_HUGEPAGES */ + #endif /* _LINUX_HIGHMEM_H */ diff --git a/mm/migrate.c b/mm/migrate.c index 8a57a09ad665..14c4db23b3ed 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -578,7 +578,6 @@ static void __copy_gigantic_page(struct page *dst, struct page *src, static void copy_huge_page(struct page *dst, struct page *src) { - int i; int nr_pages; if (PageHuge(src)) { @@ -596,10 +595,7 @@ static void copy_huge_page(struct page *dst, struct page *src) nr_pages = thp_nr_pages(src); } - for (i = 0; i < nr_pages; i++) { - cond_resched(); - copy_highpage(dst + i, src + i); - } + copy_highpages(dst, src, nr_pages); } /* -- 2.20.1

1 1

2024

2023

2022

2021

2020

2019

Kernel January 2022