- Kernel - mailweb.openeuler.org

[PATCH openEuler-21.03 2/2] pid: add pid reserve method for checkpoint and recover
by hejingxian 02 Mar '21

02 Mar '21

From: Jingxian He <hejingxian(a)huawei.com> Date: Mon, 1 Mar 2021 17:44:59 +0800 Subject: [PATCH openEuler-21.03 2/2] pid: add pid reserve method for checkpoint and recover hulk inclusion category: feature bugzilla: 48159 CVE: N/A We record the pid of dump tasks in the reserved memory, and reserve the pids before init task start. In the recover process, free the reserved pids and realloc them for use. Signed-off-by: Jingxian He <hejingxian(a)huawei.com> Reviewed-by: Wenliang He <hewenliang4(a)huawei.com> Reviewed-by: Jing Xiangfeng <jingxiangfeng(a)huawei.com> --- arch/arm64/configs/openeuler_defconfig | 1 + include/linux/pin_mem.h | 6 ++++ kernel/pid.c | 10 +++++++ mm/Kconfig | 10 +++++++ mm/pin_mem.c | 51 ++++++++++++++++++++++++++++++++++ 5 files changed, 78 insertions(+) diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 76fda68..de6db02 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -1037,6 +1037,7 @@ CONFIG_FRAME_VECTOR=y # CONFIG_READ_ONLY_THP_FOR_FS is not set CONFIG_ARCH_HAS_PTE_SPECIAL=y CONFIG_PIN_MEMORY=y +CONFIG_PID_RESERVE=y # end of Memory Management options CONFIG_NET=y diff --git a/include/linux/pin_mem.h b/include/linux/pin_mem.h index bc8b03e..a9fe2ef 100644 --- a/include/linux/pin_mem.h +++ b/include/linux/pin_mem.h @@ -74,5 +74,11 @@ extern struct resource pin_memory_resource; #endif extern void init_reserve_page_map(unsigned long map_addr, unsigned long map_size); +#ifdef CONFIG_PID_RESERVE +extern bool is_need_reserve_pids(void); +extern void free_reserved_pid(struct idr *idr, int pid); +extern void reserve_pids(struct idr *idr, int pid_max); +#endif + #endif /* CONFIG_PIN_MEMORY */ #endif /* _LINUX_PIN_MEMORY_H */ diff --git a/kernel/pid.c b/kernel/pid.c index 4856818..32ab9ef 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -44,6 +44,9 @@ #include <linux/idr.h> #include <net/sock.h> #include <uapi/linux/pidfd.h> +#ifdef CONFIG_PID_RESERVE +#include <linux/pin_mem.h> +#endif struct pid init_struct_pid = { .count = REFCOUNT_INIT(1), @@ -209,6 +212,9 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid, spin_lock_irq(&pidmap_lock); if (tid) { +#ifdef CONFIG_PID_RESERVE + free_reserved_pid(&tmp->idr, tid); +#endif nr = idr_alloc(&tmp->idr, NULL, tid, tid + 1, GFP_ATOMIC); /* @@ -621,6 +627,10 @@ void __init pid_idr_init(void) init_pid_ns.pid_cachep = KMEM_CACHE(pid, SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT); +#ifdef CONFIG_PID_RESERVE + if (is_need_reserve_pids()) + reserve_pids(&init_pid_ns.idr, pid_max); +#endif } static struct file *__pidfd_fget(struct task_struct *task, int fd) diff --git a/mm/Kconfig b/mm/Kconfig index 930dc13..e27d2c6 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -868,3 +868,13 @@ config PIN_MEMORY the corresponding physical pages mapping info in checkpoint, and remap the physical pages to restore tasks in restore. endmenu + +config PID_RESERVE + bool "Support for reserve pid" + depends on PIN_MEMORY + help + Say y here to enable the pid reserved feature for checkpoint. + and restore. + We record the pid of dump task in the reserve memory, + and reserve the pids before init task start. In restore process, + free the reserved pids and realloc them for use. diff --git a/mm/pin_mem.c b/mm/pin_mem.c index 0a143b6..a040853 100644 --- a/mm/pin_mem.c +++ b/mm/pin_mem.c @@ -947,4 +947,55 @@ void clear_pin_memory_record(void) } EXPORT_SYMBOL_GPL(clear_pin_memory_record); +#ifdef CONFIG_PID_RESERVE +struct idr *reserve_idr; + +/* test if there exist pin memory tasks */ +bool is_need_reserve_pids(void) +{ + return (pin_pid_num > 0); +} + +void free_reserved_pid(struct idr *idr, int pid) +{ + unsigned int index; + struct page_map_info *pmi; + + if (!max_pin_pid_num || idr != reserve_idr) + return; + + for (index = 0; index < pin_pid_num; index++) { + pmi = &(user_space_reserve_start[index]); + if (pmi->pid == pid && pmi->pid_reserved) { + idr_remove(idr, pid); + return; + } + } +} + +/* reserve pids for check point tasks which pinned memory */ +void reserve_pids(struct idr *idr, int pid_max) +{ + int alloc_pid; + unsigned int index; + struct page_map_info *pmi; + + if (!max_pin_pid_num) + return; + reserve_idr = idr; + for (index = 0; index < pin_pid_num; index++) { + pmi = &(user_space_reserve_start[index]); + pmi->pid_reserved = true; + alloc_pid = idr_alloc(idr, NULL, pmi->pid, pid_max, GFP_ATOMIC); + if (alloc_pid != pmi->pid) { + if (alloc_pid > 0) + idr_remove(idr, alloc_pid); + pr_warn("Reserve pid (%d) fail, real pid is %d.\n", alloc_pid, pmi->pid); + pmi->pid_reserved = false; + continue; + } + } +} +#endif /* CONFIG_PID_RESERVE */ + #endif /* CONFIG_PIN_MEMORY */ -- 2.9.5

1 0

[PATCH openEuler-21.03 1/2] mm: add pin memory method for checkpoint add restore
by hejingxian＠huawei.com 02 Mar '21

02 Mar '21

From: Jingxian He <hejingxian(a)huawei.com> hulk inclusion category: feature bugzilla: 48159 CVE: N/A We can use the checkpoint and restore in userspace(criu) method to dump and restore tasks when updating the kernel. Currently, criu needs dump all memory data of tasks to files. When the memory size is very large(larger than 1G), the cost time of the dumping data will be very long(more than 1 min). By pin the memory data of tasks and collect the corresponding physical pages mapping info in checkpoint process, we can remap the physical pages to restore tasks after upgrading the kernel. This pin memory method can restore the task data within one second. The pin memory area info is saved in the reserved memblock, which can keep usable in the kernel update process. The pin memory driver provides the following ioctl command for criu: 1) SET_PIN_MEM_AREA: Set pin memory area, which can be remap to the restore task. 2) CLEAR_PIN_MEM_AREA: Clear the pin memory area info, which enable user reset the pin data. 3) REMAP_PIN_MEM_AREA: Remap the pages of the pin memory to the restore task. Signed-off-by: Jingxian He <hejingxian(a)huawei.com> Reviewed-by: Wenliang He <hewenliang4(a)huawei.com> Reviewed-by: Jing Xiangfeng <jingxiangfeng(a)huawei.com> --- arch/arm64/configs/openeuler_defconfig | 2 + arch/arm64/kernel/setup.c | 9 + arch/arm64/mm/init.c | 60 +++ drivers/char/Kconfig | 6 + drivers/char/Makefile | 1 + drivers/char/pin_memory.c | 208 ++++++++ include/linux/crash_core.h | 5 + include/linux/pin_mem.h | 78 +++ kernel/crash_core.c | 11 + mm/Kconfig | 8 + mm/Makefile | 1 + mm/huge_memory.c | 61 +++ mm/memory.c | 59 ++ mm/pin_mem.c | 950 +++++++++++++++++++++++++++++++++ 14 files changed, 1459 insertions(+) create mode 100644 drivers/char/pin_memory.c create mode 100644 include/linux/pin_mem.h create mode 100644 mm/pin_mem.c diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index c5271e7..76fda68 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -1036,6 +1036,7 @@ CONFIG_FRAME_VECTOR=y # CONFIG_GUP_BENCHMARK is not set # CONFIG_READ_ONLY_THP_FOR_FS is not set CONFIG_ARCH_HAS_PTE_SPECIAL=y +CONFIG_PIN_MEMORY=y # end of Memory Management options CONFIG_NET=y @@ -3282,6 +3283,7 @@ CONFIG_TCG_TIS_ST33ZP24_SPI=y # CONFIG_RANDOM_TRUST_CPU is not set # CONFIG_RANDOM_TRUST_BOOTLOADER is not set +CONFIG_PIN_MEMORY_DEV=m # # I2C support diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index c1f1fb9..5e282d3 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -50,6 +50,9 @@ #include <asm/efi.h> #include <asm/xen/hypervisor.h> #include <asm/mmu_context.h> +#ifdef CONFIG_PIN_MEMORY +#include <linux/pin_mem.h> +#endif static int num_standard_resources; static struct resource *standard_resources; @@ -260,6 +263,12 @@ static void __init request_standard_resources(void) quick_kexec_res.end <= res->end) request_resource(res, &quick_kexec_res); #endif +#ifdef CONFIG_PIN_MEMORY + if (pin_memory_resource.end && + pin_memory_resource.start >= res->start && + pin_memory_resource.end <= res->end) + request_resource(res, &pin_memory_resource); +#endif } } diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index f3e5a66..8ab5aac 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -42,6 +42,9 @@ #include <linux/sizes.h> #include <asm/tlb.h> #include <asm/alternative.h> +#ifdef CONFIG_PIN_MEMORY +#include <linux/pin_mem.h> +#endif #define ARM64_ZONE_DMA_BITS 30 @@ -78,6 +81,55 @@ static void __init reserve_crashkernel(void) */ #define MAX_USABLE_RANGES 2 +#ifdef CONFIG_PIN_MEMORY +struct resource pin_memory_resource = { + .name = "Pin memory", + .start = 0, + .end = 0, + .flags = IORESOURCE_MEM, + .desc = IORES_DESC_RESERVED +}; + +static void __init reserve_pin_memory_res(void) +{ + unsigned long long mem_start, mem_len; + int ret; + + ret = parse_pin_memory(boot_command_line, memblock_phys_mem_size(), + &mem_len, &mem_start); + if (ret || !mem_len) + return; + + mem_len = PAGE_ALIGN(mem_len); + + if (!memblock_is_region_memory(mem_start, mem_len)) { + pr_warn("cannot reserve for pin memory: region is not memory!\n"); + return; + } + + if (memblock_is_region_reserved(mem_start, mem_len)) { + pr_warn("cannot reserve for pin memory: region overlaps reserved memory!\n"); + return; + } + + if (!IS_ALIGNED(mem_start, SZ_2M)) { + pr_warn("cannot reserve for pin memory: base address is not 2MB aligned\n"); + return; + } + + memblock_reserve(mem_start, mem_len); + pr_debug("pin memory resource reserved: 0x%016llx - 0x%016llx (%lld MB)\n", + mem_start, mem_start + mem_len, mem_len >> 20); + + pin_memory_resource.start = mem_start; + pin_memory_resource.end = mem_start + mem_len - 1; +} +#else +static void __init reserve_pin_memory_res(void) +{ +} +#endif /* CONFIG_PIN_MEMORY */ + #ifdef CONFIG_CRASH_DUMP static int __init early_init_dt_scan_elfcorehdr(unsigned long node, const char *uname, int depth, void *data) @@ -455,6 +507,8 @@ void __init arm64_memblock_init(void) reserve_park_mem(); #endif + reserve_pin_memory_res(); + reserve_elfcorehdr(); high_memory = __va(memblock_end_of_DRAM() - 1) + 1; @@ -583,6 +637,12 @@ void __init mem_init(void) /* this will put all unused low memory onto the freelists */ memblock_free_all(); +#ifdef CONFIG_PIN_MEMORY + /* pre alloc the pages for pin memory */ + init_reserve_page_map((unsigned long)pin_memory_resource.start, + (unsigned long)(pin_memory_resource.end - pin_memory_resource.start + 1)); +#endif + mem_init_print_info(NULL); /* diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index d229a2d..fbb94b8 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -496,3 +496,9 @@ config RANDOM_TRUST_BOOTLOADER booloader is trustworthy so it will be added to the kernel's entropy pool. Otherwise, say N here so it will be regarded as device input that only mixes the entropy pool. + +config PIN_MEMORY_DEV + bool "/dev/pinmem character device" + default m + help + pin memory driver diff --git a/drivers/char/Makefile b/drivers/char/Makefile index ffce287..71d76fd 100644 --- a/drivers/char/Makefile +++ b/drivers/char/Makefile @@ -47,3 +47,4 @@ obj-$(CONFIG_PS3_FLASH) += ps3flash.o obj-$(CONFIG_XILLYBUS) += xillybus/ obj-$(CONFIG_POWERNV_OP_PANEL) += powernv-op-panel.o obj-$(CONFIG_ADI) += adi.o +obj-$(CONFIG_PIN_MEMORY_DEV) += pin_memory.o diff --git a/drivers/char/pin_memory.c b/drivers/char/pin_memory.c new file mode 100644 index 0000000..f46e056 --- /dev/null +++ b/drivers/char/pin_memory.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright @ Huawei Technologies Co., Ltd. 2020-2020. ALL rights reserved. + * Description: Euler pin memory driver + */ +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/kprobes.h> +#include <linux/spinlock.h> +#include <linux/workqueue.h> +#include <linux/sched.h> +#include <linux/mm.h> +#include <linux/init.h> +#include <linux/miscdevice.h> +#include <linux/fs.h> +#include <linux/mm_types.h> +#include <linux/processor.h> +#include <uapi/asm-generic/ioctl.h> +#include <uapi/asm-generic/mman-common.h> +#include <uapi/asm/setup.h> +#include <linux/pin_mem.h> +#include <linux/sched/mm.h> + +#define MAX_PIN_MEM_AREA_NUM 16 +struct _pin_mem_area { + unsigned long virt_start; + unsigned long virt_end; +}; + +struct pin_mem_area_set { + unsigned int pid; + unsigned int area_num; + struct _pin_mem_area mem_area[MAX_PIN_MEM_AREA_NUM]; +}; + +#define PIN_MEM_MAGIC 0x59 +#define _SET_PIN_MEM_AREA 1 +#define _CLEAR_PIN_MEM_AREA 2 +#define _REMAP_PIN_MEM_AREA 3 +#define _FINISH_PIN_MEM_DUMP 4 +#define _PIN_MEM_IOC_MAX_NR 4 +#define SET_PIN_MEM_AREA _IOW(PIN_MEM_MAGIC, _SET_PIN_MEM_AREA, struct pin_mem_area_set) +#define CLEAR_PIN_MEM_AREA _IOW(PIN_MEM_MAGIC, _CLEAR_PIN_MEM_AREA, int) +#define REMAP_PIN_MEM_AREA _IOW(PIN_MEM_MAGIC, _REMAP_PIN_MEM_AREA, int) +#define FINISH_PIN_MEM_DUMP _IOW(PIN_MEM_MAGIC, _FINISH_PIN_MEM_DUMP, int) +static int set_pin_mem(struct pin_mem_area_set *pmas) +{ + int i; + int ret = 0; + struct _pin_mem_area *pma; + struct mm_struct *mm; + struct task_struct *task; + struct pid *pid_s; + + pid_s = find_get_pid(pmas->pid); + if (!pid_s) { + pr_warn("Get pid struct fail:%d.\n", pmas->pid); + return -EFAULT; + } + rcu_read_lock(); + task = pid_task(pid_s, PIDTYPE_PID); + if (!task) { + pr_warn("Get task struct fail:%d.\n", pmas->pid); + goto fail; + } + mm = get_task_mm(task); + for (i = 0; i < pmas->area_num; i++) { + pma = &(pmas->mem_area[i]); + ret = pin_mem_area(task, mm, pma->virt_start, pma->virt_end); + if (ret) { + mmput(mm); + goto fail; + } + } + mmput(mm); + rcu_read_unlock(); + put_pid(pid_s); + return ret; + +fail: + rcu_read_unlock(); + put_pid(pid_s); + return -EFAULT; +} + +static int set_pin_mem_area(unsigned long arg) +{ + struct pin_mem_area_set pmas; + void __user *buf = (void __user *)arg; + + if (!access_ok(buf, sizeof(pmas))) + return -EFAULT; + if (copy_from_user(&pmas, buf, sizeof(pmas))) + return -EINVAL; + if (pmas.area_num > MAX_PIN_MEM_AREA_NUM) { + pr_warn("Input area_num is too large.\n"); + return -EINVAL; + } + + return set_pin_mem(&pmas); +} + +static int pin_mem_remap(unsigned long arg) +{ + int pid; + struct task_struct *task; + struct mm_struct *mm; + vm_fault_t ret; + void __user *buf = (void __user *)arg; + struct pid *pid_s; + + if (!access_ok(buf, sizeof(int))) + return -EINVAL; + if (copy_from_user(&pid, buf, sizeof(int))) + return -EINVAL; + + pid_s = find_get_pid(pid); + if (!pid_s) { + pr_warn("Get pid struct fail:%d.\n", pid); + return -EINVAL; + } + rcu_read_lock(); + task = pid_task(pid_s, PIDTYPE_PID); + if (!task) { + pr_warn("Get task struct fail:%d.\n", pid); + goto fault; + } + mm = get_task_mm(task); + ret = do_mem_remap(pid, mm); + if (ret) { + pr_warn("Handle pin memory remap fail.\n"); + mmput(mm); + goto fault; + } + mmput(mm); + rcu_read_unlock(); + put_pid(pid_s); + return 0; + +fault: + rcu_read_unlock(); + put_pid(pid_s); + return -EFAULT; +} + +static long pin_memory_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + long ret = 0; + + if (_IOC_TYPE(cmd) != PIN_MEM_MAGIC) + return -EINVAL; + if (_IOC_NR(cmd) > _PIN_MEM_IOC_MAX_NR) + return -EINVAL; + + switch (cmd) { + case SET_PIN_MEM_AREA: + ret = set_pin_mem_area(arg); + break; + case CLEAR_PIN_MEM_AREA: + clear_pin_memory_record(); + break; + case REMAP_PIN_MEM_AREA: + ret = pin_mem_remap(arg); + break; + case FINISH_PIN_MEM_DUMP: + ret = finish_pin_mem_dump(); + break; + default: + return -EINVAL; + } + return ret; +} + +static const struct file_operations pin_memory_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = pin_memory_ioctl, + .compat_ioctl = pin_memory_ioctl, +}; + +static struct miscdevice pin_memory_miscdev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "pinmem", + .fops = &pin_memory_fops, +}; + +static int pin_memory_init(void) +{ + int err = misc_register(&pin_memory_miscdev); + + if (!err) + pr_info("pin_memory init\n"); + else + pr_warn("pin_memory init failed!\n"); + return err; +} + +static void pin_memory_exit(void) +{ + misc_deregister(&pin_memory_miscdev); + pr_info("pin_memory ko exists!\n"); +} + +module_init(pin_memory_init); +module_exit(pin_memory_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Euler"); +MODULE_DESCRIPTION("pin memory"); diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h index fc0ef33..30f0df3 100644 --- a/include/linux/crash_core.h +++ b/include/linux/crash_core.h @@ -87,4 +87,9 @@ int parse_crashkernel_high(char *cmdline, unsigned long long system_ram, int parse_crashkernel_low(char *cmdline, unsigned long long system_ram, unsigned long long *crash_size, unsigned long long *crash_base); +#ifdef CONFIG_PIN_MEMORY +int __init parse_pin_memory(char *cmdline, unsigned long long system_ram, + unsigned long long *pin_size, unsigned long long *pin_base); +#endif + #endif /* LINUX_CRASH_CORE_H */ diff --git a/include/linux/pin_mem.h b/include/linux/pin_mem.h new file mode 100644 index 0000000..bc8b03e --- /dev/null +++ b/include/linux/pin_mem.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. + * Provide the pin memory method for check point and restore task. + */ +#ifndef _LINUX_PIN_MEMORY_H +#define _LINUX_PIN_MEMORY_H + +#ifdef CONFIG_PIN_MEMORY +#include <linux/errno.h> +#include <linux/mm_types.h> +#include <linux/err.h> +#ifdef CONFIG_ARM64 +#include <linux/ioport.h> +#endif + +#define PAGE_BUDDY_MAPCOUNT_VALUE (~PG_buddy) + +#define COLLECT_PAGES_FINISH 0 +#define COLLECT_PAGES_NEED_CONTINUE 1 +#define COLLECT_PAGES_FAIL -1 + +#define COMPOUND_PAD_MASK 0xffffffff +#define COMPOUND_PAD_START 0x88 +#define COMPOUND_PAD_DELTA 0x40 +#define LIST_POISON4 0xdead000000000400 +#define PAGE_FLAGS_CHECK_RESERVED (1UL << PG_reserved) +#define SHA256_DIGEST_SIZE 32 +#define next_pme(pme) ((unsigned long *)(pme + 1) + pme->nr_pages) +#define PIN_MEM_DUMP_MAGIC 0xfeab000000001acd +struct page_map_entry { + unsigned long virt_addr; + unsigned int nr_pages; + unsigned int is_huge_page; + unsigned long redirect_start; + unsigned long phy_addr_array[0]; +}; + +struct page_map_info { + int pid; + int pid_reserved; + unsigned int entry_num; + int disable_free_page; + struct page_map_entry *pme; +}; + +struct pin_mem_dump_info { + char sha_digest[SHA256_DIGEST_SIZE]; + unsigned long magic; + unsigned int pin_pid_num; + struct page_map_info pmi_array[0]; +}; + +struct redirect_info { + unsigned int redirect_pages; + unsigned int redirect_index[0]; +}; + +extern struct page_map_info *get_page_map_info(int pid); +extern struct page_map_info *create_page_map_info(int pid); +extern vm_fault_t do_mem_remap(int pid, struct mm_struct *mm); +extern vm_fault_t do_anon_page_remap(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmd, struct page *page); +extern void clear_pin_memory_record(void); +extern int pin_mem_area(struct task_struct *task, struct mm_struct *mm, + unsigned long start_addr, unsigned long end_addr); +extern vm_fault_t do_anon_huge_page_remap(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmd, struct page *page); +extern int finish_pin_mem_dump(void); + +/* reserve space for pin memory*/ +#ifdef CONFIG_ARM64 +extern struct resource pin_memory_resource; +#endif +extern void init_reserve_page_map(unsigned long map_addr, unsigned long map_size); + +#endif /* CONFIG_PIN_MEMORY */ +#endif /* _LINUX_PIN_MEMORY_H */ diff --git a/kernel/crash_core.c b/kernel/crash_core.c index bfed474..2407de3 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -450,6 +450,17 @@ void __init reserve_crashkernel(void) } #endif /* CONFIG_ARCH_WANT_RESERVE_CRASH_KERNEL */ +#ifdef CONFIG_PIN_MEMORY +int __init parse_pin_memory(char *cmdline, + unsigned long long system_ram, + unsigned long long *pin_size, + unsigned long long *pin_base) +{ + return __parse_crashkernel(cmdline, system_ram, pin_size, pin_base, + "pinmemory=", NULL); +} +#endif + Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type, void *data, size_t data_len) { diff --git a/mm/Kconfig b/mm/Kconfig index 390165f..930dc13 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -859,4 +859,12 @@ config ARCH_HAS_HUGEPD config MAPPING_DIRTY_HELPERS bool +config PIN_MEMORY + bool "Support for pin memory" + depends on CHECKPOINT_RESTORE + help + Say y here to enable the pin memory feature for checkpoint + and restore. We can pin the memory data of tasks and collect + the corresponding physical pages mapping info in checkpoint, + and remap the physical pages to restore tasks in restore. endmenu diff --git a/mm/Makefile b/mm/Makefile index d73aed0..4963827 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -120,3 +120,4 @@ obj-$(CONFIG_MEMFD_CREATE) += memfd.o obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o obj-$(CONFIG_PTDUMP_CORE) += ptdump.o obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o +obj-$(CONFIG_PIN_MEMORY) += pin_mem.o diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 0bc4a2c..8a11d30 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2996,3 +2996,64 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) update_mmu_cache_pmd(vma, address, pvmw->pmd); } #endif + +#ifdef CONFIG_PIN_MEMORY +vm_fault_t do_anon_huge_page_remap(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmd, struct page *page) +{ + gfp_t gfp; + pgtable_t pgtable; + spinlock_t *ptl; + pmd_t entry; + vm_fault_t ret = 0; + + if (unlikely(anon_vma_prepare(vma))) + return VM_FAULT_OOM; + if (unlikely(khugepaged_enter(vma, vma->vm_flags))) + return VM_FAULT_OOM; + gfp = alloc_hugepage_direct_gfpmask(vma); + prep_transhuge_page(page); + if (mem_cgroup_charge(page, vma->vm_mm, gfp)) { + put_page(page); + count_vm_event(THP_FAULT_FALLBACK); + count_vm_event(THP_FAULT_FALLBACK_CHARGE); + return VM_FAULT_FALLBACK; + } + cgroup_throttle_swaprate(page, gfp); + + pgtable = pte_alloc_one(vma->vm_mm); + if (unlikely(!pgtable)) { + ret = VM_FAULT_OOM; + goto release; + } + __SetPageUptodate(page); + ptl = pmd_lock(vma->vm_mm, pmd); + if (unlikely(!pmd_none(*pmd))) { + goto unlock_release; + } else { + ret = check_stable_address_space(vma->vm_mm); + if (ret) + goto unlock_release; + entry = mk_huge_pmd(page, vma->vm_page_prot); + entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); + page_add_new_anon_rmap(page, vma, address, true); + lru_cache_add_inactive_or_unevictable(page, vma); + pgtable_trans_huge_deposit(vma->vm_mm, pmd, pgtable); + set_pmd_at(vma->vm_mm, address, pmd, entry); + add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); + mm_inc_nr_ptes(vma->vm_mm); + spin_unlock(ptl); + count_vm_event(THP_FAULT_ALLOC); + count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC); + } + + return 0; +unlock_release: + spin_unlock(ptl); +release: + if (pgtable) + pte_free(vma->vm_mm, pgtable); + put_page(page); + return ret; +} +#endif diff --git a/mm/memory.c b/mm/memory.c index 50632c4..7b7f1a7 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -5248,3 +5248,62 @@ void ptlock_free(struct page *page) kmem_cache_free(page_ptl_cachep, page->ptl); } #endif + +#ifdef CONFIG_PIN_MEMORY +vm_fault_t do_anon_page_remap(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmd, struct page *page) +{ + pte_t entry; + spinlock_t *ptl; + pte_t *pte; + vm_fault_t ret = 0; + + if (pte_alloc(vma->vm_mm, pmd)) + return VM_FAULT_OOM; + + /* See the comment in pte_alloc_one_map() */ + if (unlikely(pmd_trans_unstable(pmd))) + return 0; + + /* Allocate our own private page. */ + if (unlikely(anon_vma_prepare(vma))) + goto oom; + + if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL)) + goto oom_free_page; + cgroup_throttle_swaprate(page, GFP_KERNEL); + + __SetPageUptodate(page); + + entry = mk_pte(page, vma->vm_page_prot); + if (vma->vm_flags & VM_WRITE) + entry = pte_mkwrite(pte_mkdirty(entry)); + pte = pte_offset_map_lock(vma->vm_mm, pmd, address, + &ptl); + if (!pte_none(*pte)) { + ret = VM_FAULT_FALLBACK; + goto release; + } + + ret = check_stable_address_space(vma->vm_mm); + if (ret) + goto release; + inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); + page_add_new_anon_rmap(page, vma, address, false); + lru_cache_add_inactive_or_unevictable(page, vma); + + set_pte_at(vma->vm_mm, address, pte, entry); + /* No need to invalidate - it was non-present before */ + update_mmu_cache(vma, address, pte); +unlock: + pte_unmap_unlock(pte, ptl); + return ret; +release: + put_page(page); + goto unlock; +oom_free_page: + put_page(page); +oom: + return VM_FAULT_OOM; +} +#endif diff --git a/mm/pin_mem.c b/mm/pin_mem.c new file mode 100644 index 0000000..0a143b6 --- /dev/null +++ b/mm/pin_mem.c @@ -0,0 +1,950 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. + * Provide the pin memory method for check point and restore task. + */ +#ifdef CONFIG_PIN_MEMORY +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/time.h> +#include <linux/sched/cputime.h> +#include <linux/tick.h> +#include <linux/mm.h> +#include <linux/pin_mem.h> +#include <linux/idr.h> +#include <linux/page-isolation.h> +#include <linux/sched/mm.h> +#include <linux/ctype.h> +#include <linux/highmem.h> +#include <crypto/sha.h> + +#define MAX_PIN_PID_NUM 128 +static DEFINE_SPINLOCK(page_map_entry_lock); + +struct pin_mem_dump_info *pin_mem_dump_start; +unsigned int pin_pid_num; +static unsigned int *pin_pid_num_addr; +static unsigned long __page_map_entry_start; +static unsigned long page_map_entry_end; +static struct page_map_info *user_space_reserve_start; +static struct page_map_entry *page_map_entry_start; +unsigned int max_pin_pid_num __read_mostly; +unsigned long redirect_space_size; +unsigned long redirect_space_start; +#define DEFAULT_REDIRECT_SPACE_SIZE 0x100000 + +static int __init setup_max_pin_pid_num(char *str) +{ + int ret = 0; + + if (!str) + goto out; + + ret = kstrtouint(str, 10, &max_pin_pid_num); +out: + if (ret) { + pr_warn("Unable to parse max pin pid num.\n"); + } else { + if (max_pin_pid_num > MAX_PIN_PID_NUM) { + max_pin_pid_num = 0; + pr_warn("Input max_pin_pid_num is too large.\n"); + } + } + return ret; +} +early_param("max_pin_pid_num", setup_max_pin_pid_num); + +static int __init setup_redirect_space_size(char *str) +{ + if (!str) + goto out; + + redirect_space_size = memparse(str, NULL); +out: + if (!redirect_space_size) { + pr_warn("Unable to parse redirect space size, use the default value.\n"); + redirect_space_size = DEFAULT_REDIRECT_SPACE_SIZE; + } + return 0; +} +early_param("redirect_space_size", setup_redirect_space_size); + +struct page_map_info *create_page_map_info(int pid) +{ + struct page_map_info *new; + + if (!user_space_reserve_start) + return NULL; + + if (pin_pid_num >= max_pin_pid_num) { + pr_warn("Pin pid num too large than max_pin_pid_num, fail create: %d!", pid); + return NULL; + } + new = (struct page_map_info *)(user_space_reserve_start + pin_pid_num); + new->pid = pid; + new->pme = NULL; + new->entry_num = 0; + new->pid_reserved = false; + new->disable_free_page = false; + (*pin_pid_num_addr)++; + pin_pid_num++; + return new; +} +EXPORT_SYMBOL_GPL(create_page_map_info); + +struct page_map_info *get_page_map_info(int pid) +{ + int i; + + if (!user_space_reserve_start) + return NULL; + + for (i = 0; i < pin_pid_num; i++) { + if (user_space_reserve_start[i].pid == pid) + return &(user_space_reserve_start[i]); + } + return NULL; +} +EXPORT_SYMBOL_GPL(get_page_map_info); + +static struct page *find_head_page(struct page *page) +{ + struct page *p = page; + + while (!PageBuddy(p)) { + if (PageLRU(p)) + return NULL; + p--; + } + return p; +} + +static void spilt_page_area_left(struct zone *zone, struct free_area *area, struct page *page, + unsigned long size, int order) +{ + unsigned long cur_size = 1 << order; + unsigned long total_size = 0; + + while (size && cur_size > size) { + cur_size >>= 1; + order--; + area--; + if (cur_size <= size) { + list_add(&page[total_size].lru, &area->free_list[MIGRATE_MOVABLE]); + atomic_set(&(page[total_size]._mapcount), PAGE_BUDDY_MAPCOUNT_VALUE); + set_page_private(&page[total_size], order); + set_pageblock_migratetype(&page[total_size], MIGRATE_MOVABLE); + area->nr_free++; + total_size += cur_size; + size -= cur_size; + } + } +} + +static void spilt_page_area_right(struct zone *zone, struct free_area *area, struct page *page, + unsigned long size, int order) +{ + unsigned long cur_size = 1 << order; + struct page *right_page, *head_page; + + right_page = page + size; + while (size && cur_size > size) { + cur_size >>= 1; + order--; + area--; + if (cur_size <= size) { + head_page = right_page - cur_size; + list_add(&head_page->lru, &area->free_list[MIGRATE_MOVABLE]); + atomic_set(&(head_page->_mapcount), PAGE_BUDDY_MAPCOUNT_VALUE); + set_page_private(head_page, order); + set_pageblock_migratetype(head_page, MIGRATE_MOVABLE); + area->nr_free++; + size -= cur_size; + right_page = head_page; + } + } +} + +void reserve_page_from_buddy(unsigned long nr_pages, struct page *page) +{ + unsigned int current_order; + struct page *page_end; + struct free_area *area; + struct zone *zone; + struct page *head_page; + + head_page = find_head_page(page); + if (!head_page) { + pr_warn("Find page head fail."); + return; + } + current_order = head_page->private; + page_end = head_page + (1 << current_order); + zone = page_zone(head_page); + area = &(zone->free_area[current_order]); + list_del(&head_page->lru); + atomic_set(&head_page->_mapcount, -1); + set_page_private(head_page, 0); + area->nr_free--; + if (head_page != page) + spilt_page_area_left(zone, area, head_page, + (unsigned long)(page - head_page), current_order); + page = page + nr_pages; + if (page < page_end) { + spilt_page_area_right(zone, area, page, + (unsigned long)(page_end - page), current_order); + } else if (page > page_end) { + pr_warn("Find page end smaller than page."); + } +} + +static inline void reserve_user_normal_pages(struct page *page) +{ + atomic_inc(&page->_refcount); + reserve_page_from_buddy(1, page); +} + +static void init_huge_pmd_pages(struct page *head_page) +{ + int i = 0; + struct page *page = head_page; + + __set_bit(PG_head, &page->flags); + __set_bit(PG_active, &page->flags); + atomic_set(&page->_refcount, 1); + page++; + i++; + page->compound_head = (unsigned long)head_page + 1; + page->compound_dtor = HUGETLB_PAGE_DTOR + 1; + page->compound_order = HPAGE_PMD_ORDER; + page++; + i++; + page->compound_head = (unsigned long)head_page + 1; + i++; + INIT_LIST_HEAD(&(page->deferred_list)); + for (; i < HPAGE_PMD_NR; i++) { + page = head_page + i; + page->compound_head = (unsigned long)head_page + 1; + } +} + +static inline void reserve_user_huge_pmd_pages(struct page *page) +{ + atomic_inc(&page->_refcount); + reserve_page_from_buddy((1 << HPAGE_PMD_ORDER), page); + init_huge_pmd_pages(page); +} + +int reserve_user_map_pages_fail; + +void free_user_map_pages(unsigned int pid_index, unsigned int entry_index, unsigned int page_index) +{ + unsigned int i, j, index, order; + struct page_map_info *pmi; + struct page_map_entry *pme; + struct page *page; + unsigned long phy_addr; + + for (index = 0; index < pid_index; index++) { + pmi = &(user_space_reserve_start[index]); + pme = pmi->pme; + for (i = 0; i < pmi->entry_num; i++) { + for (j = 0; j < pme->nr_pages; j++) { + order = pme->is_huge_page ? HPAGE_PMD_ORDER : 0; + phy_addr = pme->phy_addr_array[j]; + if (phy_addr) { + page = phys_to_page(phy_addr); + if (!(page->flags & PAGE_FLAGS_CHECK_RESERVED)) { + __free_pages(page, order); + pme->phy_addr_array[j] = 0; + } + } + } + pme = (struct page_map_entry *)next_pme(pme); + } + } + pmi = &(user_space_reserve_start[index]); + pme = pmi->pme; + for (i = 0; i < entry_index; i++) { + for (j = 0; j < pme->nr_pages; j++) { + order = pme->is_huge_page ? HPAGE_PMD_ORDER : 0; + phy_addr = pme->phy_addr_array[j]; + if (phy_addr) { + page = phys_to_page(phy_addr); + if (!(page->flags & PAGE_FLAGS_CHECK_RESERVED)) { + __free_pages(page, order); + pme->phy_addr_array[j] = 0; + } + } + } + pme = (struct page_map_entry *)next_pme(pme); + } + for (j = 0; j < page_index; j++) { + order = pme->is_huge_page ? HPAGE_PMD_ORDER : 0; + phy_addr = pme->phy_addr_array[j]; + if (phy_addr) { + page = phys_to_page(phy_addr); + if (!(page->flags & PAGE_FLAGS_CHECK_RESERVED)) { + __free_pages(page, order); + pme->phy_addr_array[j] = 0; + } + } + } +} + +bool check_redirect_end_valid(struct redirect_info *redirect_start, + unsigned long max_redirect_page_num) +{ + unsigned long redirect_end; + + redirect_end = ((unsigned long)(redirect_start + 1) + + max_redirect_page_num * sizeof(unsigned int)); + if (redirect_end > redirect_space_start + redirect_space_size) + return false; + return false; +} + +static void reserve_user_space_map_pages(void) +{ + struct page_map_info *pmi; + struct page_map_entry *pme; + unsigned int i, j, index; + struct page *page; + unsigned long flags; + unsigned long phy_addr; + unsigned long redirect_pages = 0; + struct redirect_info *redirect_start = (struct redirect_info *)redirect_space_start; + + if (!user_space_reserve_start || !redirect_start) + return; + spin_lock_irqsave(&page_map_entry_lock, flags); + for (index = 0; index < pin_pid_num; index++) { + pmi = &(user_space_reserve_start[index]); + pme = pmi->pme; + for (i = 0; i < pmi->entry_num; i++) { + redirect_pages = 0; + if (!check_redirect_end_valid(redirect_start, pme->nr_pages)) + redirect_start = NULL; + for (j = 0; j < pme->nr_pages; j++) { + phy_addr = pme->phy_addr_array[j]; + if (!phy_addr) + continue; + page = phys_to_page(phy_addr); + if (atomic_read(&page->_refcount)) { + if ((page->flags & PAGE_FLAGS_CHECK_RESERVED) + && !pme->redirect_start) + pme->redirect_start = + (unsigned long)redirect_start; + if (redirect_start && + (page->flags & PAGE_FLAGS_CHECK_RESERVED)) { + redirect_start->redirect_index[redirect_pages] = j; + redirect_pages++; + continue; + } else { + reserve_user_map_pages_fail = 1; + pr_warn("Page %pK refcount %d large than zero, no need reserve.\n", + page, atomic_read(&page->_refcount)); + goto free_pages; + } + } + if (!pme->is_huge_page) + reserve_user_normal_pages(page); + else + reserve_user_huge_pmd_pages(page); + } + pme = (struct page_map_entry *)next_pme(pme); + if (redirect_pages && redirect_start) { + redirect_start->redirect_pages = redirect_pages; + redirect_start = (struct redirect_info *)( + (unsigned long)(redirect_start + 1) + + redirect_start->redirect_pages * sizeof(unsigned int)); + } + } + } + spin_unlock(&page_map_entry_lock); + return; +free_pages: + free_user_map_pages(index, i, j); + spin_unlock(&page_map_entry_lock); +} + + +int calculate_pin_mem_digest(struct pin_mem_dump_info *pmdi, char *digest) +{ + int i; + struct sha256_state sctx; + + if (!digest) + digest = pmdi->sha_digest; + sha256_init(&sctx); + sha256_update(&sctx, (unsigned char *)(&(pmdi->magic)), + sizeof(struct pin_mem_dump_info) - SHA256_DIGEST_SIZE); + for (i = 0; i < pmdi->pin_pid_num; i++) { + sha256_update(&sctx, (unsigned char *)(&(pmdi->pmi_array[i])), + sizeof(struct page_map_info)); + } + sha256_final(&sctx, digest); + return 0; +} + +static int check_sha_digest(struct pin_mem_dump_info *pmdi) +{ + int ret = 0; + char digest[SHA256_DIGEST_SIZE] = {0}; + + ret = calculate_pin_mem_digest(pmdi, digest); + if (ret) { + pr_warn("calculate pin mem digest fail:%d\n", ret); + return ret; + } + if (memcmp(pmdi->sha_digest, digest, SHA256_DIGEST_SIZE)) { + pr_warn("pin mem dump info sha256 digest match error!\n"); + return -EFAULT; + } + return ret; +} + +/* + * The whole page map entry collect process must be Sequentially. + * The user_space_reserve_start points to the first page map info for + * the first dump task. And the page_map_entry_start points to + * the first page map entry of the first dump vma. + */ +static void init_page_map_info(struct pin_mem_dump_info *pmdi, unsigned long map_len) +{ + if (pin_mem_dump_start || !max_pin_pid_num) { + pr_warn("pin page map already init or max_pin_pid_num not set.\n"); + return; + } + if (map_len < sizeof(struct pin_mem_dump_info) + + max_pin_pid_num * sizeof(struct page_map_info) + redirect_space_size) { + pr_warn("pin memory reserved memblock too small.\n"); + return; + } + if ((pmdi->magic != PIN_MEM_DUMP_MAGIC) || (pmdi->pin_pid_num > max_pin_pid_num) || + check_sha_digest(pmdi)) + memset(pmdi, 0, sizeof(struct pin_mem_dump_info)); + pin_mem_dump_start = pmdi; + pin_pid_num = pmdi->pin_pid_num; + pr_info("pin_pid_num: %d\n", pin_pid_num); + pin_pid_num_addr = &(pmdi->pin_pid_num); + user_space_reserve_start = + (struct page_map_info *)pmdi->pmi_array; + page_map_entry_start = + (struct page_map_entry *)(user_space_reserve_start + max_pin_pid_num); + page_map_entry_end = (unsigned long)pmdi + map_len - redirect_space_size; + redirect_space_start = page_map_entry_end; + if (pin_pid_num > 0) + reserve_user_space_map_pages(); +} + +int finish_pin_mem_dump(void) +{ + int ret; + + pin_mem_dump_start->magic = PIN_MEM_DUMP_MAGIC; + memset(pin_mem_dump_start->sha_digest, 0, SHA256_DIGEST_SIZE); + ret = calculate_pin_mem_digest(pin_mem_dump_start, NULL); + if (ret) { + pr_warn("calculate pin mem digest fail:%d\n", ret); + return ret; + } + return ret; +} + +int collect_pmd_huge_pages(struct task_struct *task, + unsigned long start_addr, unsigned long end_addr, struct page_map_entry *pme) +{ + long res; + int index = 0; + unsigned long start = start_addr; + struct page *temp_page; + + while (start < end_addr) { + temp_page = NULL; + res = get_user_pages_remote(task->mm, start, 1, + FOLL_TOUCH | FOLL_GET, &temp_page, NULL, NULL); + if (!res) { + pr_warn("Get huge page for addr(%lx) fail.", start); + return COLLECT_PAGES_FAIL; + } + if (PageHead(temp_page)) { + start += HPAGE_PMD_SIZE; + pme->phy_addr_array[index] = page_to_phys(temp_page); + index++; + } else { + pme->nr_pages = index; + atomic_dec(&((temp_page)->_refcount)); + return COLLECT_PAGES_NEED_CONTINUE; + } + } + pme->nr_pages = index; + return COLLECT_PAGES_FINISH; +} + +int collect_normal_pages(struct task_struct *task, + unsigned long start_addr, unsigned long end_addr, struct page_map_entry *pme) +{ + int res; + unsigned long next; + unsigned long i, nr_pages; + struct page *tmp_page; + unsigned long *phy_addr_array = pme->phy_addr_array; + struct page **page_array = (struct page **)pme->phy_addr_array; + + next = (start_addr & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE; + next = (next > end_addr) ? end_addr : next; + pme->nr_pages = 0; + while (start_addr < next) { + nr_pages = (PAGE_ALIGN(next) - start_addr) / PAGE_SIZE; + res = get_user_pages_remote(task->mm, start_addr, 1, + FOLL_TOUCH | FOLL_GET, &tmp_page, NULL, NULL); + if (!res) { + pr_warn("Get user page of %lx fail.\n", start_addr); + return COLLECT_PAGES_FAIL; + } + if (PageHead(tmp_page)) { + atomic_dec(&(tmp_page->_refcount)); + return COLLECT_PAGES_NEED_CONTINUE; + } + atomic_dec(&(tmp_page->_refcount)); + if (PageTail(tmp_page)) { + start_addr = next; + pme->virt_addr = start_addr; + next = (next + HPAGE_PMD_SIZE) > end_addr ? + end_addr : (next + HPAGE_PMD_SIZE); + continue; + } + res = get_user_pages_remote(task->mm, start_addr, nr_pages, + FOLL_TOUCH | FOLL_GET, page_array, NULL, NULL); + if (!res) { + pr_warn("Get user pages of %lx fail.\n", start_addr); + return COLLECT_PAGES_FAIL; + } + for (i = 0; i < nr_pages; i++) + phy_addr_array[i] = page_to_phys(page_array[i]); + pme->nr_pages += nr_pages; + page_array += nr_pages; + phy_addr_array += nr_pages; + start_addr = next; + next = (next + HPAGE_PMD_SIZE) > end_addr ? end_addr : (next + HPAGE_PMD_SIZE); + } + return COLLECT_PAGES_FINISH; +} + +/* Users make sure that the pin memory belongs to anonymous vma. */ +int pin_mem_area(struct task_struct *task, struct mm_struct *mm, + unsigned long start_addr, unsigned long end_addr) +{ + int pid, ret; + int is_huge_page = false; + unsigned int page_size; + unsigned long nr_pages, flags; + struct page_map_entry *pme; + struct page_map_info *pmi; + struct vm_area_struct *vma; + unsigned long i; + struct page *tmp_page; + + if (!page_map_entry_start + || !task || !mm + || start_addr >= end_addr) + return -EFAULT; + + pid = task->pid; + spin_lock_irqsave(&page_map_entry_lock, flags); + nr_pages = ((end_addr - start_addr) / PAGE_SIZE); + if ((unsigned long)page_map_entry_start + nr_pages * sizeof(struct page *) >= + page_map_entry_end) { + pr_warn("Page map entry use up!\n"); + ret = -EFAULT; + goto finish; + } + vma = find_extend_vma(mm, start_addr); + if (!vma) { + pr_warn("Find no match vma!\n"); + ret = -EFAULT; + goto finish; + } + if (start_addr == (start_addr & HPAGE_PMD_MASK) && + transparent_hugepage_enabled(vma)) { + page_size = HPAGE_PMD_SIZE; + is_huge_page = true; + } else { + page_size = PAGE_SIZE; + } + pme = page_map_entry_start; + pme->virt_addr = start_addr; + pme->redirect_start = 0; + pme->is_huge_page = is_huge_page; + memset(pme->phy_addr_array, 0, nr_pages * sizeof(unsigned long)); + down_write(&mm->mmap_lock); + if (!is_huge_page) { + ret = collect_normal_pages(task, start_addr, end_addr, pme); + if (ret != COLLECT_PAGES_FAIL && !pme->nr_pages) { + if (ret == COLLECT_PAGES_FINISH) { + ret = 0; + up_write(&mm->mmap_lock); + goto finish; + } + pme->is_huge_page = true; + page_size = HPAGE_PMD_SIZE; + ret = collect_pmd_huge_pages(task, pme->virt_addr, end_addr, pme); + } + } else { + ret = collect_pmd_huge_pages(task, start_addr, end_addr, pme); + if (ret != COLLECT_PAGES_FAIL && !pme->nr_pages) { + if (ret == COLLECT_PAGES_FINISH) { + ret = 0; + up_write(&mm->mmap_lock); + goto finish; + } + pme->is_huge_page = false; + page_size = PAGE_SIZE; + ret = collect_normal_pages(task, pme->virt_addr, end_addr, pme); + } + } + up_write(&mm->mmap_lock); + if (ret == COLLECT_PAGES_FAIL) { + ret = -EFAULT; + goto finish; + } + + /* check for zero pages */ + for (i = 0; i < pme->nr_pages; i++) { + tmp_page = phys_to_page(pme->phy_addr_array[i]); + if (!pme->is_huge_page) { + if (page_to_pfn(tmp_page) == my_zero_pfn(pme->virt_addr + i * PAGE_SIZE)) + pme->phy_addr_array[i] = 0; + } else if (is_huge_zero_page(tmp_page)) + pme->phy_addr_array[i] = 0; + } + + page_map_entry_start = (struct page_map_entry *)(next_pme(pme)); + pmi = get_page_map_info(pid); + if (!pmi) + pmi = create_page_map_info(pid); + if (!pmi) { + pr_warn("Create page map info fail for pid: %d!\n", pid); + ret = -EFAULT; + goto finish; + } + if (!pmi->pme) + pmi->pme = pme; + pmi->entry_num++; + spin_unlock_irqrestore(&page_map_entry_lock, flags); + if (ret == COLLECT_PAGES_NEED_CONTINUE) + ret = pin_mem_area(task, mm, pme->virt_addr + pme->nr_pages * page_size, end_addr); + return ret; +finish: + spin_unlock_irqrestore(&page_map_entry_lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(pin_mem_area); + +vm_fault_t remap_normal_pages(struct mm_struct *mm, struct vm_area_struct *vma, + struct page_map_entry *pme) +{ + int ret; + unsigned int j, i; + pgd_t *pgd; + p4d_t *p4d; + pmd_t *pmd; + pud_t *pud; + struct page *page, *new; + unsigned long address; + unsigned long phy_addr; + unsigned int redirect_pages = 0; + struct redirect_info *redirect_start; + + redirect_start = (struct redirect_info *)pme->redirect_start; + for (j = 0; j < pme->nr_pages; j++) { + address = pme->virt_addr + j * PAGE_SIZE; + phy_addr = pme->phy_addr_array[j]; + if (!phy_addr) + continue; + page = phys_to_page(phy_addr); + if (page_to_pfn(page) == my_zero_pfn(address)) { + pme->phy_addr_array[j] = 0; + continue; + } + pme->phy_addr_array[j] = 0; + if (redirect_start && (redirect_pages < redirect_start->redirect_pages) && + (j == redirect_start->redirect_index[redirect_pages])) { + new = alloc_zeroed_user_highpage_movable(vma, address); + if (!new) { + pr_warn("Redirect alloc page fail\n"); + continue; + } + copy_page(page_to_virt(new), phys_to_virt(phy_addr)); + page = new; + redirect_pages++; + } + page->mapping = NULL; + pgd = pgd_offset(mm, address); + p4d = p4d_alloc(mm, pgd, address); + if (!p4d) { + ret = VM_FAULT_OOM; + goto free; + } + pud = pud_alloc(mm, p4d, address); + if (!pud) { + ret = VM_FAULT_OOM; + goto free; + } + pmd = pmd_alloc(mm, pud, address); + if (!pmd) { + ret = VM_FAULT_OOM; + goto free; + } + ret = do_anon_page_remap(vma, address, pmd, page); + if (ret) + goto free; + } + return 0; +free: + for (i = j; i < pme->nr_pages; i++) { + phy_addr = pme->phy_addr_array[i]; + if (phy_addr) { + __free_page(phys_to_page(phy_addr)); + pme->phy_addr_array[i] = 0; + } + } + return ret; +} + +static inline gfp_t get_hugepage_gfpmask(struct vm_area_struct *vma) +{ + const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE); + + if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) + return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); + if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) + return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM; + if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) + return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM : + __GFP_KSWAPD_RECLAIM); + if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) + return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM : + 0); + return GFP_TRANSHUGE_LIGHT; +} + +vm_fault_t remap_huge_pmd_pages(struct mm_struct *mm, struct vm_area_struct *vma, + struct page_map_entry *pme) +{ + int ret; + unsigned int j, i; + pgd_t *pgd; + p4d_t *p4d; + pmd_t *pmd; + pud_t *pud; + gfp_t gfp; + struct page *page, *new; + unsigned long address; + unsigned long phy_addr; + unsigned int redirect_pages = 0; + struct redirect_info *redirect_start; + + redirect_start = (struct redirect_info *)pme->redirect_start; + for (j = 0; j < pme->nr_pages; j++) { + address = pme->virt_addr + j * HPAGE_PMD_SIZE; + phy_addr = pme->phy_addr_array[j]; + if (!phy_addr) + continue; + page = phys_to_page(phy_addr); + if (is_huge_zero_page(page)) { + pme->phy_addr_array[j] = 0; + continue; + } + pme->phy_addr_array[j] = 0; + if (redirect_start && (redirect_pages < redirect_start->redirect_pages) && + (j == redirect_start->redirect_index[redirect_pages])) { + gfp = get_hugepage_gfpmask(vma); + new = alloc_hugepage_vma(gfp, vma, address, HPAGE_PMD_ORDER); + if (!new) { + pr_warn("Redirect alloc huge page fail\n"); + continue; + } + memcpy(page_to_virt(new), phys_to_virt(phy_addr), HPAGE_PMD_SIZE); + page = new; + redirect_pages++; + } + pgd = pgd_offset(mm, address); + p4d = p4d_alloc(mm, pgd, address); + if (!p4d) { + ret = VM_FAULT_OOM; + goto free; + } + pud = pud_alloc(mm, p4d, address); + if (!pud) { + ret = VM_FAULT_OOM; + goto free; + } + pmd = pmd_alloc(mm, pud, address); + if (!pmd) { + ret = VM_FAULT_OOM; + goto free; + } + ret = do_anon_huge_page_remap(vma, address, pmd, page); + if (ret) + goto free; + } + return 0; +free: + for (i = j; i < pme->nr_pages; i++) { + phy_addr = pme->phy_addr_array[i]; + if (phy_addr) { + page = phys_to_page(phy_addr); + if (!(page->flags & PAGE_FLAGS_CHECK_RESERVED)) { + __free_pages(page, HPAGE_PMD_ORDER); + pme->phy_addr_array[i] = 0; + } + } + } + return ret; +} + +static void free_unmap_pages(struct page_map_info *pmi, + struct page_map_entry *pme, + unsigned int index) +{ + unsigned int i, j; + unsigned long phy_addr; + unsigned int order; + struct page *page; + + pme = (struct page_map_entry *)(next_pme(pme)); + for (i = index; i < pmi->entry_num; i++) { + for (j = 0; j < pme->nr_pages; j++) { + phy_addr = pme->phy_addr_array[i]; + if (phy_addr) { + page = phys_to_page(phy_addr); + order = pme->is_huge_page ? HPAGE_PMD_ORDER : 0; + if (!(page->flags & PAGE_FLAGS_CHECK_RESERVED)) { + __free_pages(page, order); + pme->phy_addr_array[i] = 0; + } + } + } + pme = (struct page_map_entry *)(next_pme(pme)); + } +} + +vm_fault_t do_mem_remap(int pid, struct mm_struct *mm) +{ + unsigned int i = 0; + vm_fault_t ret = 0; + struct vm_area_struct *vma; + struct page_map_info *pmi; + struct page_map_entry *pme; + unsigned long flags; + + if (reserve_user_map_pages_fail) + return -EFAULT; + pmi = get_page_map_info(pid); + if (!pmi) + return -EFAULT; + + spin_lock_irqsave(&page_map_entry_lock, flags); + pmi->disable_free_page = true; + spin_unlock(&page_map_entry_lock); + down_write(&mm->mmap_lock); + pme = pmi->pme; + vma = mm->mmap; + while ((i < pmi->entry_num) && (vma != NULL)) { + if (pme->virt_addr >= vma->vm_start && pme->virt_addr < vma->vm_end) { + i++; + if (!vma_is_anonymous(vma)) { + pme = (struct page_map_entry *)(next_pme(pme)); + continue; + } + if (!pme->is_huge_page) { + ret = remap_normal_pages(mm, vma, pme); + if (ret < 0) + goto free; + } else { + ret = remap_huge_pmd_pages(mm, vma, pme); + if (ret < 0) + goto free; + } + pme = (struct page_map_entry *)(next_pme(pme)); + } else { + vma = vma->vm_next; + } + } + up_write(&mm->mmap_lock); + return 0; +free: + free_unmap_pages(pmi, pme, i); + up_write(&mm->mmap_lock); + return ret; +} +EXPORT_SYMBOL_GPL(do_mem_remap); + +#if defined(CONFIG_ARM64) +void init_reserve_page_map(unsigned long map_addr, unsigned long map_size) +{ + void *addr; + + if (!map_addr || !map_size) + return; + addr = phys_to_virt(map_addr); + init_page_map_info((struct pin_mem_dump_info *)addr, map_size); +} +#else +void init_reserve_page_map(unsigned long map_addr, unsigned long map_size) +{ +} +#endif + +static void free_all_reserved_pages(void) +{ + unsigned int i, j, index, order; + struct page_map_info *pmi; + struct page_map_entry *pme; + struct page *page; + unsigned long phy_addr; + + if (!user_space_reserve_start || reserve_user_map_pages_fail) + return; + + for (index = 0; index < pin_pid_num; index++) { + pmi = &(user_space_reserve_start[index]); + if (pmi->disable_free_page) + continue; + pme = pmi->pme; + for (i = 0; i < pmi->entry_num; i++) { + for (j = 0; j < pme->nr_pages; j++) { + order = pme->is_huge_page ? HPAGE_PMD_ORDER : 0; + phy_addr = pme->phy_addr_array[j]; + if (phy_addr) { + page = phys_to_page(phy_addr); + if (!(page->flags & PAGE_FLAGS_CHECK_RESERVED)) { + __free_pages(page, order); + pme->phy_addr_array[j] = 0; + } + } + } + pme = (struct page_map_entry *)next_pme(pme); + } + } +} + +/* Clear all pin memory record. */ +void clear_pin_memory_record(void) +{ + unsigned long flags; + + spin_lock_irqsave(&page_map_entry_lock, flags); + free_all_reserved_pages(); + if (pin_pid_num_addr) { + *pin_pid_num_addr = 0; + pin_pid_num = 0; + page_map_entry_start = (struct page_map_entry *)__page_map_entry_start; + } + spin_unlock(&page_map_entry_lock); +} +EXPORT_SYMBOL_GPL(clear_pin_memory_record); + +#endif /* CONFIG_PIN_MEMORY */ -- 2.9.5

1 1

[PATCH OLK-5.10 v1] arm64: Declare var of local_cpu_stop only on PARK
by sangyan＠huawei.com 01 Mar '21

01 Mar '21

From: Sang Yan <sangyan(a)huawei.com> hulk inclusion category: feature bugzilla: 48159 CVE: N/A Fix compile warning: unused variable 'ops' 'cpu' while CONFIG_ARM64_CPU_PARK=n. Put declaration of 'ops' and 'cpu' under CONFIG_ARM64_CPU_PARK in local_cpu_stop. Signed-off-by: Sang Yan <sangyan(a)huawei.com> --- arch/arm64/kernel/smp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 644bbd7..d7b750a 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -1024,8 +1024,10 @@ void arch_irq_work_raise(void) static void local_cpu_stop(void) { +#ifdef CONFIG_ARM64_CPU_PARK int cpu; const struct cpu_operations *ops = NULL; +#endif set_cpu_online(smp_processor_id(), false); -- 2.9.5

1 0

[PATCH OLK-5.10 v2 1/2] arm64: smp: Add support for cpu park
by sangyan＠huawei.com 26 Feb '21

26 Feb '21

From: Sang Yan <sangyan(a)huawei.com> hulk inclusion category: feature bugzilla: 48159 CVE: N/A Introducing a feature of CPU PARK in order to save time of cpus down and up during kexec, which may cost 250ms of per cpu's down and 30ms of up. As a result, for 128 cores, it costs more than 30 seconds to down and up cpus during kexec. Think about 256 cores and more. CPU PARK is a state that cpu power-on and staying in spin loop, polling for exit chances, such as writing exit address. Reserving a block of memory, to fill with cpu park text section, exit address and park-magic-flag of each cpu. In implementation, reserved one page for one cpu core. Cpus going to park state instead of down in machine_shutdown(). Cpus going out of park state in smp_init instead of brought up. One of cpu park sections in pre-reserved memory blocks,: +--------------+ + exit address + +--------------+ + park magic + +--------------+ + park codes + + . + + . + + . + +--------------+ Signed-off-by: Sang Yan <sangyan(a)huawei.com> Reviewed-by: Jing Xiangfeng <jingxiangfeng(a)huawei.com> --- arch/arm64/Kconfig | 12 ++ arch/arm64/include/asm/kexec.h | 6 + arch/arm64/include/asm/smp.h | 15 +++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/cpu-park.S | 59 ++++++++++ arch/arm64/kernel/machine_kexec.c | 2 +- arch/arm64/kernel/process.c | 4 + arch/arm64/kernel/smp.c | 230 ++++++++++++++++++++++++++++++++++++++ arch/arm64/mm/init.c | 55 +++++++++ 9 files changed, 383 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/kernel/cpu-park.S diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b9c5654..0885668 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -345,6 +345,18 @@ config KASAN_SHADOW_OFFSET default 0xeffffff900000000 if ARM64_VA_BITS_36 && KASAN_SW_TAGS default 0xffffffffffffffff +config ARM64_CPU_PARK + bool "Support CPU PARK on kexec" + depends on SMP + depends on KEXEC_CORE + help + This enables support for CPU PARK feature in + order to save time of cpu down to up. + CPU park is a state through kexec, spin loop + instead of cpu die before jumping to new kernel, + jumping out from loop to new kernel entry in + smp_init. + source "arch/arm64/Kconfig.platforms" menu "Kernel Features" diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h index 79909ae..a133889 100644 --- a/arch/arm64/include/asm/kexec.h +++ b/arch/arm64/include/asm/kexec.h @@ -36,6 +36,11 @@ #define CRASH_ADDR_HIGH_MAX MEMBLOCK_ALLOC_ACCESSIBLE +#ifdef CONFIG_ARM64_CPU_PARK +/* CPU park state flag: "park" */ +#define PARK_MAGIC 0x7061726b +#endif + #ifndef __ASSEMBLY__ /** @@ -104,6 +109,7 @@ static inline void crash_post_resume(void) {} #ifdef CONFIG_KEXEC_CORE extern void __init reserve_crashkernel(void); #endif +void machine_kexec_mask_interrupts(void); #ifdef CONFIG_KEXEC_FILE #define ARCH_HAS_KIMAGE_ARCH diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index 2e7f529..8c5d2d6 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -145,6 +145,21 @@ bool cpus_are_stuck_in_kernel(void); extern void crash_smp_send_stop(void); extern bool smp_crash_stop_failed(void); +#ifdef CONFIG_ARM64_CPU_PARK +#define PARK_SECTION_SIZE 1024 +struct cpu_park_info { + /* Physical address of reserved park memory. */ + unsigned long start; + /* park reserve mem len should be PARK_SECTION_SIZE * NR_CPUS */ + unsigned long len; + /* Virtual address of reserved park memory. */ + unsigned long start_v; +}; +extern struct cpu_park_info park_info; +extern void enter_cpu_park(unsigned long text, unsigned long exit); +extern void do_cpu_park(unsigned long exit); +extern int kexec_smp_send_park(void); +#endif #endif /* ifndef __ASSEMBLY__ */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 2621d5c..60478d2 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -54,6 +54,7 @@ obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o \ cpu-reset.o +obj-$(CONFIG_ARM64_CPU_PARK) += cpu-park.o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_ARM64_RELOC_TEST) += arm64-reloc-test.o arm64-reloc-test-y := reloc_test_core.o reloc_test_syms.o diff --git a/arch/arm64/kernel/cpu-park.S b/arch/arm64/kernel/cpu-park.S new file mode 100644 index 0000000..10c685c --- /dev/null +++ b/arch/arm64/kernel/cpu-park.S @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * CPU park routines + * + * Copyright (C) 2020 Huawei Technologies., Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> +#include <asm/kexec.h> +#include <asm/sysreg.h> +#include <asm/virt.h> + +.text +.pushsection .idmap.text, "awx" + +/* cpu park helper in idmap section */ +SYM_CODE_START(enter_cpu_park) + /* Clear sctlr_el1 flags. */ + mrs x12, sctlr_el1 + mov_q x13, SCTLR_ELx_FLAGS + bic x12, x12, x13 + pre_disable_mmu_workaround + msr sctlr_el1, x12 /* disable mmu */ + isb + + mov x18, x0 + mov x0, x1 /* secondary_entry addr */ + br x18 /* call do_cpu_park of each cpu */ +SYM_CODE_END(enter_cpu_park) + +.popsection + +SYM_CODE_START(do_cpu_park) + ldr x18, =PARK_MAGIC /* magic number "park" */ + add x1, x0, #8 + str x18, [x1] /* set on-park flag */ + dc civac, x1 /* flush cache of "park" */ + dsb nsh + isb + +.Lloop: + wfe + isb + ldr x19, [x0] + cmp x19, #0 /* test secondary_entry */ + b.eq .Lloop + + ic iallu /* invalidate the local I-cache */ + dsb nsh + isb + + br x19 /* jump to secondary_entry */ +SYM_CODE_END(do_cpu_park) + diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index a0b144c..f47ce96 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -213,7 +213,7 @@ void machine_kexec(struct kimage *kimage) BUG(); /* Should never get here. */ } -static void machine_kexec_mask_interrupts(void) +void machine_kexec_mask_interrupts(void) { unsigned int i; struct irq_desc *desc; diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 73e3b32..10cffee 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -146,6 +146,10 @@ void arch_cpu_idle_dead(void) */ void machine_shutdown(void) { +#ifdef CONFIG_ARM64_CPU_PARK + if (kexec_smp_send_park() == 0) + return; +#endif smp_shutdown_nonboot_cpus(reboot_cpu); } diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 18e9727..dea67d0 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -32,6 +32,7 @@ #include <linux/irq_work.h> #include <linux/kernel_stat.h> #include <linux/kexec.h> + #include <linux/kvm_host.h> #include <asm/alternative.h> @@ -93,6 +94,167 @@ static inline int op_cpu_kill(unsigned int cpu) } #endif +#ifdef CONFIG_ARM64_CPU_PARK +struct cpu_park_section { + unsigned long exit; /* exit address of park look */ + unsigned long magic; /* maigc represent park state */ + char text[0]; /* text section of park */ +}; + +static int mmap_cpu_park_mem(void) +{ + if (!park_info.start) + return -ENOMEM; + + if (park_info.start_v) + return 0; + + park_info.start_v = (unsigned long)__ioremap(park_info.start, + park_info.len, + PAGE_KERNEL_EXEC); + if (!park_info.start_v) { + pr_warn("map park memory failed."); + return -ENOMEM; + } + + return 0; +} + +static inline unsigned long cpu_park_section_v(unsigned int cpu) +{ + return park_info.start_v + PARK_SECTION_SIZE * (cpu - 1); +} + +static inline unsigned long cpu_park_section_p(unsigned int cpu) +{ + return park_info.start + PARK_SECTION_SIZE * (cpu - 1); +} + +/* + * Write the secondary_entry to exit section of park state. + * Then the secondary cpu will jump straight into the kernel + * by the secondary_entry. + */ +static int write_park_exit(unsigned int cpu) +{ + struct cpu_park_section *park_section; + unsigned long *park_exit; + unsigned long *park_text; + + if (mmap_cpu_park_mem() != 0) + return -EPERM; + + park_section = (struct cpu_park_section *)cpu_park_section_v(cpu); + park_exit = &park_section->exit; + park_text = (unsigned long *)park_section->text; + pr_debug("park_text 0x%lx : 0x%lx, do_cpu_park text 0x%lx : 0x%lx", + (unsigned long)park_text, *park_text, + (unsigned long)do_cpu_park, + *(unsigned long *)do_cpu_park); + + /* + * Test first 8 bytes to determine + * whether needs to write cpu park exit. + */ + if (*park_text == *(unsigned long *)do_cpu_park) { + writeq_relaxed(__pa_symbol(secondary_entry), park_exit); + __flush_dcache_area((__force void *)park_exit, + sizeof(unsigned long)); + flush_icache_range((unsigned long)park_exit, + (unsigned long)(park_exit + 1)); + sev(); + dsb(sy); + isb(); + + pr_debug("Write cpu %u secondary entry 0x%lx to 0x%lx.", + cpu, *park_exit, (unsigned long)park_exit); + pr_info("Boot cpu %u from PARK state.", cpu); + return 0; + } + + return -EPERM; +} + +/* Install cpu park sections for the specific cpu. */ +static int install_cpu_park(unsigned int cpu) +{ + struct cpu_park_section *park_section; + unsigned long *park_exit; + unsigned long *park_magic; + unsigned long park_text_len; + + park_section = (struct cpu_park_section *)cpu_park_section_v(cpu); + pr_debug("Install cpu park on cpu %u park exit 0x%lx park text 0x%lx", + cpu, (unsigned long)park_section, + (unsigned long)(park_section->text)); + + park_exit = &park_section->exit; + park_magic = &park_section->magic; + park_text_len = PARK_SECTION_SIZE - sizeof(struct cpu_park_section); + + *park_exit = 0UL; + *park_magic = 0UL; + memcpy((void *)park_section->text, do_cpu_park, park_text_len); + __flush_dcache_area((void *)park_section, PARK_SECTION_SIZE); + + return 0; +} + +static int uninstall_cpu_park(unsigned int cpu) +{ + unsigned long park_section; + + if (mmap_cpu_park_mem() != 0) + return -EPERM; + + park_section = cpu_park_section_v(cpu); + memset((void *)park_section, 0, PARK_SECTION_SIZE); + __flush_dcache_area((void *)park_section, PARK_SECTION_SIZE); + + return 0; +} + +static int cpu_wait_park(unsigned int cpu) +{ + long timeout; + struct cpu_park_section *park_section; + + volatile unsigned long *park_magic; + + park_section = (struct cpu_park_section *)cpu_park_section_v(cpu); + park_magic = &park_section->magic; + + timeout = USEC_PER_SEC; + while (*park_magic != PARK_MAGIC && timeout--) + udelay(1); + + if (timeout > 0) + pr_debug("cpu %u park done.", cpu); + else + pr_err("cpu %u park failed.", cpu); + + return *park_magic == PARK_MAGIC; +} + +static void cpu_park(unsigned int cpu) +{ + unsigned long park_section_p; + unsigned long park_exit_phy; + unsigned long do_park; + typeof(enter_cpu_park) *park; + + park_section_p = cpu_park_section_p(cpu); + park_exit_phy = park_section_p; + pr_debug("Go to park cpu %u exit address 0x%lx", cpu, park_exit_phy); + + do_park = park_section_p + sizeof(struct cpu_park_section); + park = (void *)__pa_symbol(enter_cpu_park); + + cpu_install_idmap(); + park(do_park, park_exit_phy); + unreachable(); +} +#endif /* * Boot a secondary CPU, and assign it the specified idle task. @@ -102,6 +264,10 @@ static int boot_secondary(unsigned int cpu, struct task_struct *idle) { const struct cpu_operations *ops = get_cpu_ops(cpu); +#ifdef CONFIG_ARM64_CPU_PARK + if (write_park_exit(cpu) == 0) + return 0; +#endif if (ops->cpu_boot) return ops->cpu_boot(cpu); @@ -131,6 +297,9 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) return ret; } +#ifdef CONFIG_ARM64_CPU_PARK + uninstall_cpu_park(cpu); +#endif /* * CPU was successfully started, wait for it to come online or * time out. @@ -844,10 +1013,32 @@ void arch_irq_work_raise(void) static void local_cpu_stop(void) { +#ifdef CONFIG_ARM64_CPU_PARK + int cpu; + const struct cpu_operations *ops = NULL; +#endif + set_cpu_online(smp_processor_id(), false); local_daif_mask(); sdei_mask_local_cpu(); + +#ifdef CONFIG_ARM64_CPU_PARK + /* + * Go to cpu park state. + * Otherwise go to cpu die. + */ + cpu = smp_processor_id(); + if (kexec_in_progress && park_info.start_v) { + machine_kexec_mask_interrupts(); + cpu_park(cpu); + + ops = get_cpu_ops(cpu); + if (ops && ops->cpu_die) + ops->cpu_die(cpu); + } +#endif + cpu_park_loop(); } @@ -1053,6 +1244,45 @@ void smp_send_stop(void) sdei_mask_local_cpu(); } +#ifdef CONFIG_ARM64_CPU_PARK +int kexec_smp_send_park(void) +{ + unsigned long cpu; + + if (WARN_ON(!kexec_in_progress)) { + pr_crit("%s called not in kexec progress.", __func__); + return -EPERM; + } + + if (mmap_cpu_park_mem() != 0) { + pr_info("no cpuparkmem, goto normal way."); + return -EPERM; + } + + local_irq_disable(); + + if (num_online_cpus() > 1) { + cpumask_t mask; + + cpumask_copy(&mask, cpu_online_mask); + cpumask_clear_cpu(smp_processor_id(), &mask); + + for_each_cpu(cpu, &mask) + install_cpu_park(cpu); + smp_cross_call(&mask, IPI_CPU_STOP); + + /* Wait for other CPUs to park */ + for_each_cpu(cpu, &mask) + cpu_wait_park(cpu); + pr_info("smp park other cpus done\n"); + } + + sdei_mask_local_cpu(); + + return 0; +} +#endif + #ifdef CONFIG_KEXEC_CORE void crash_smp_send_stop(void) { diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 794f992..d01259c 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -236,6 +236,57 @@ static void __init fdt_enforce_memory_region(void) memblock_add(usable_rgns[1].base, usable_rgns[1].size); } +#ifdef CONFIG_ARM64_CPU_PARK +struct cpu_park_info park_info = { + .start = 0, + .len = PARK_SECTION_SIZE * NR_CPUS, + .start_v = 0, +}; + +static int __init parse_park_mem(char *p) +{ + if (!p) + return 0; + + park_info.start = PAGE_ALIGN(memparse(p, NULL)); + if (park_info.start == 0) + pr_info("cpu park mem params[%s]", p); + + return 0; +} +early_param("cpuparkmem", parse_park_mem); + +static int __init reserve_park_mem(void) +{ + if (park_info.start == 0 || park_info.len == 0) + return 0; + + park_info.start = PAGE_ALIGN(park_info.start); + park_info.len = PAGE_ALIGN(park_info.len); + + if (!memblock_is_region_memory(park_info.start, park_info.len)) { + pr_warn("cannot reserve park mem: region is not memory!"); + goto out; + } + + if (memblock_is_region_reserved(park_info.start, park_info.len)) { + pr_warn("cannot reserve park mem: region overlaps reserved memory!"); + goto out; + } + + memblock_remove(park_info.start, park_info.len); + pr_info("cpu park mem reserved: 0x%016lx - 0x%016lx (%ld MB)", + park_info.start, park_info.start + park_info.len, + park_info.len >> 20); + + return 0; +out: + park_info.start = 0; + park_info.len = 0; + return -EINVAL; +} +#endif + void __init arm64_memblock_init(void) { const s64 linear_region_size = BIT(vabits_actual - 1); @@ -357,6 +408,10 @@ void __init arm64_memblock_init(void) reserve_crashkernel(); +#ifdef CONFIG_ARM64_CPU_PARK + reserve_park_mem(); +#endif + reserve_elfcorehdr(); high_memory = __va(memblock_end_of_DRAM() - 1) + 1; -- 2.9.5

1 1

[PATCH openEuler-1.0-LTS 00/65] MPAM bugfix @ 20210224
by Cheng Jian 24 Feb '21

24 Feb '21

MPAM bugfix @ 20210224 James Morse (10): arm64/mpam: Add mpam driver discovery phase and kbuild boiler plate cacheinfo: Provide a helper to find a cacheinfo leaf arm64/mpam: Probe supported partid/pmg ranges from devices arm64/mpam: Supplement MPAM MSC register layout definitions arm64/mpam: Probe the features resctrl supports arm64/mpam: Reset controls when CPUs come online arm64/mpam: Summarize feature support during mpam_enable() arm64/mpam: resctrl: Re-synchronise resctrl's view of online CPUs drivers: base: cacheinfo: Add helper to search cacheinfo by of_node arm64/mpam: Enabling registering and logging error interrupts Wang ShaoBo (55): arm64/mpam: Preparing for MPAM refactoring arm64/mpam: Add helper for getting mpam sysprops arm64/mpam: Allocate mpam component configuration arrays arm64/mpam: Pick MPAM resources and events for resctrl_res exported arm64/mpam: Init resctrl resources' info from resctrl_res selected arm64/mpam: resctrl: Handle cpuhp and resctrl_dom allocation arm64/mpam: Implement helpers for handling configuration and monitoring arm64/mpam: Migrate old MSCs' discovery process to new branch arm64/mpam: Add helper for getting MSCs' configuration arm64/mpam: Probe partid,pmg and feature capabilities' ranges from classes arm64/mpam: resctrl: Rebuild configuration and monitoring pipeline arm64/mpam: resctrl: Append schemata CDP definitions arm64/mpam: resctrl: Supplement cdpl2,cdpl3 for mount options arm64/mpam: resctrl: Add helpers for init and destroy schemata list arm64/mpam: resctrl: Use resctrl_group_init_alloc() to init schema list arm64/mpam: resctrl: Write and read schemata by schema_list arm64/mpam: Support cdp in mpam_sched_in() arm64/mpam: resctrl: Update resources reset process arm64/mpam: resctrl: Update closid alloc and free process with bitmap arm64/mpam: resctrl: Move ctrlmon sysfile write/read function to mpam_ctrlmon.c arm64/mpam: Support cdp on allocating monitors arm64/mpam: resctrl: Support cdp on monitoring data arm64/mpam: Clean up header files and rearrange declarations arm64/mpam: resctrl: Remove ctrlmon sysfile arm64/mpam: resctrl: Remove unnecessary CONFIG_ARM64 arm64/mpam: Implement intpartid narrowing process arm64/mpam: Using software-defined id for rdtgroup instead of 32-bit integer arm64/mpam: resctrl: collect child mon group's monitor data arm64/mpam: resctrl: Support cpus' monitoring for mon group arm64/mpam: resctrl: Support priority and hardlimit(Memory bandwidth) configuration arm64/mpam: Store intpri and dspri for mpam device reset arm64/mpam: Squash default priority from mpam device to class arm64/mpam: Restore extend ctrls' max width for checking schemata input arm64/mpam: Re-plan intpartid narrowing process arm64/mpam: Add hook-events id for ctrl features arm64/mpam: Integrate monitor data for Memory Bandwidth if cdp enabled arm64/mpam: Fix MPAM_ESR intPARTID_range error arm64/mpam: Separate internal and downstream priority event arm64/mpam: Remap reqpartid,pmg to rmid and intpartid to closid arm64/mpam: Add wait queue for monitor alloc and free arm64/mpam: Add resctrl_ctrl_feature structure to manage ctrl features arm64/mpam: resctrl: Export resource's properties to info directory arm64/mpam: Split header files into suitable location arm64/mpam: resctrl: Add rmid file in resctrl sysfs arm64/mpam: Filter schema control type with ctrl features arm64/mpam: Simplify mpamid cdp mapping process arm64/mpam: Set per-cpu's closid to none zero for cdp ACPI/MPAM: Use acpi_map_pxm_to_node() to get node id for memory node arm64/mpam: Supplement additional useful ctrl features for mount options arm64/mpam: resctrl: Add proper error handling to resctrl_mount() arm64/mpam: resctrl: Use resctrl_group_init_alloc() for default group arm64/mpam: resctrl: Allow setting register MPAMCFG_MBW_MIN to 0 arm64/mpam: resctrl: Refresh cpu mask for handling cpuhp arm64/mpam: Sort domains when cpu online arm64/mpam: Fix compile warning arch/arm64/include/asm/mpam.h | 324 +--- arch/arm64/include/asm/mpam_resource.h | 129 -- arch/arm64/include/asm/mpam_sched.h | 8 - arch/arm64/include/asm/resctrl.h | 514 +++++- arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/mpam.c | 1499 ---------------- arch/arm64/kernel/mpam/Makefile | 3 + arch/arm64/kernel/mpam/mpam_ctrlmon.c | 961 ++++++++++ arch/arm64/kernel/mpam/mpam_device.c | 1706 ++++++++++++++++++ arch/arm64/kernel/mpam/mpam_device.h | 140 ++ arch/arm64/kernel/mpam/mpam_internal.h | 345 ++++ arch/arm64/kernel/mpam/mpam_mon.c | 334 ++++ arch/arm64/kernel/mpam/mpam_resctrl.c | 2240 ++++++++++++++++++++++++ arch/arm64/kernel/mpam/mpam_resource.h | 228 +++ arch/arm64/kernel/mpam/mpam_setup.c | 608 +++++++ arch/arm64/kernel/mpam_ctrlmon.c | 623 ------- arch/arm64/kernel/mpam_mon.c | 124 -- drivers/acpi/arm64/mpam.c | 87 +- drivers/base/cacheinfo.c | 38 + fs/resctrlfs.c | 396 +++-- include/linux/arm_mpam.h | 118 ++ include/linux/cacheinfo.h | 36 + include/linux/resctrlfs.h | 30 - 23 files changed, 7521 insertions(+), 2972 deletions(-) delete mode 100644 arch/arm64/include/asm/mpam_resource.h delete mode 100644 arch/arm64/kernel/mpam.c create mode 100644 arch/arm64/kernel/mpam/Makefile create mode 100644 arch/arm64/kernel/mpam/mpam_ctrlmon.c create mode 100644 arch/arm64/kernel/mpam/mpam_device.c create mode 100644 arch/arm64/kernel/mpam/mpam_device.h create mode 100644 arch/arm64/kernel/mpam/mpam_internal.h create mode 100644 arch/arm64/kernel/mpam/mpam_mon.c create mode 100644 arch/arm64/kernel/mpam/mpam_resctrl.c create mode 100644 arch/arm64/kernel/mpam/mpam_resource.h create mode 100644 arch/arm64/kernel/mpam/mpam_setup.c delete mode 100644 arch/arm64/kernel/mpam_ctrlmon.c delete mode 100644 arch/arm64/kernel/mpam_mon.c create mode 100644 include/linux/arm_mpam.h -- 2.25.1

1 65

[PATCH kernel-4.19] drivers: TCM: add GMJS(Nationz Tech) TCM driver.
by Xie XiuQi 23 Feb '21

23 Feb '21

From: Li ZhiGang <lizhigang(a)kylinos.cn> Nationz Tech TCM are used for trusted computing, the chip attached via SPI or LPC. We have a brief verify/test with this driver on KunPeng920 + openEuler system, with externally compiled module. Signed-off-by: Li ZhiGang <lizhigang(a)kylinos.cn> --- drivers/staging/Kconfig | 2 + drivers/staging/Makefile | 2 + drivers/staging/gmjstcm/Kconfig | 21 + drivers/staging/gmjstcm/Makefile | 5 + drivers/staging/gmjstcm/tcm.c | 949 ++++++++++++++++++++++++++ drivers/staging/gmjstcm/tcm.h | 123 ++++ drivers/staging/gmjstcm/tcm_tis_spi.c | 868 +++++++++++++++++++++++ 7 files changed, 1970 insertions(+) create mode 100644 drivers/staging/gmjstcm/Kconfig create mode 100644 drivers/staging/gmjstcm/Makefile create mode 100644 drivers/staging/gmjstcm/tcm.c create mode 100644 drivers/staging/gmjstcm/tcm.h create mode 100644 drivers/staging/gmjstcm/tcm_tis_spi.c diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig index 1abf76be2aa8..d51fa4f4e7ca 100644 --- a/drivers/staging/Kconfig +++ b/drivers/staging/Kconfig @@ -126,4 +126,6 @@ source "drivers/staging/axis-fifo/Kconfig" source "drivers/staging/erofs/Kconfig" +source "drivers/staging/gmjstcm/Kconfig" + endif # STAGING diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile index ab0cbe8815b1..6d41915dad5b 100644 --- a/drivers/staging/Makefile +++ b/drivers/staging/Makefile @@ -53,3 +53,5 @@ obj-$(CONFIG_SOC_MT7621) += mt7621-dts/ obj-$(CONFIG_STAGING_GASKET_FRAMEWORK) += gasket/ obj-$(CONFIG_XIL_AXIS_FIFO) += axis-fifo/ obj-$(CONFIG_EROFS_FS) += erofs/ +obj-$(CONFIG_GMJS_TCM) += gmjstcm/ + diff --git a/drivers/staging/gmjstcm/Kconfig b/drivers/staging/gmjstcm/Kconfig new file mode 100644 index 000000000000..5b5397ae1832 --- /dev/null +++ b/drivers/staging/gmjstcm/Kconfig @@ -0,0 +1,21 @@ +menu "GMJS TCM support" + +config GMJS_TCM + bool + +config GMJS_TCM_CORE + tristate "GMJS TCM core support" + depends on ARM64 || MIPS + default m + select GMJS_TCM + help + GMJS TCM core support. + +config GMJS_TCM_SPI + tristate "GMJS TCM support on SPI interface" + depends on GMJS_TCM_CORE && SPI_MASTER + default m + help + GMJS TCM support on SPI interface. + +endmenu diff --git a/drivers/staging/gmjstcm/Makefile b/drivers/staging/gmjstcm/Makefile new file mode 100644 index 000000000000..601c78e44793 --- /dev/null +++ b/drivers/staging/gmjstcm/Makefile @@ -0,0 +1,5 @@ + +obj-$(CONFIG_GMJS_TCM_CORE) += tcm_core.o +tcm_core-objs := tcm.o +obj-$(CONFIG_GMJS_TCM_SPI) += tcm_tis_spi.o + diff --git a/drivers/staging/gmjstcm/tcm.c b/drivers/staging/gmjstcm/tcm.c new file mode 100644 index 000000000000..5c41bfa8b423 --- /dev/null +++ b/drivers/staging/gmjstcm/tcm.c @@ -0,0 +1,949 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2009 Nationz Technologies Inc. + * + * Description: Exprot symbol for tcm_tis module + * + * Major Function: public write read register function etc. + * + */ + +#include <linux/sched.h> +#include <linux/poll.h> +#include <linux/spinlock.h> +#include <linux/timer.h> +#include "tcm.h" + +/* + * const var + */ +enum tcm_const { + TCM_MINOR = 224, /* officially assigned */ + TCM_BUFSIZE = 2048, /* Buffer Size */ + TCM_NUM_DEVICES = 256, /* Max supporting tcm device number */ +}; + +/* + * CMD duration + */ +enum tcm_duration { + TCM_SHORT = 0, + TCM_MEDIUM = 1, + TCM_LONG = 2, + TCM_UNDEFINED, +}; + +/* Max Total of Command Number */ +#define TCM_MAX_ORDINAL 88 /*243*/ + +static LIST_HEAD(tcm_chip_list); +static DEFINE_SPINLOCK(driver_lock); /* spin lock */ +static DECLARE_BITMAP(dev_mask, TCM_NUM_DEVICES); + +typedef struct tagTCM_Command { + u8 ordinal; + u8 DURATION; +} TCM_Command; + +static const TCM_Command TCM_Command_List[TCM_MAX_ORDINAL + 1] = { + {/*TCM_ORD_ActivateIdentity, */122, 1}, + {/*TCM_ORD_CertifyKey, */50, 1}, + {/*TCM_ORD_CertifyKeyM, */51, 1}, + {/*TCM_ORD_ChangeAuth, */12, 1}, + {/*TCM_ORD_ChangeAuthOwner, */16, 0}, + {/*TCM_ORD_ContinueSelfTeSt, */83, 2}, + {/*TCM_ORD_CreateCounter, */220, 0}, + {/*TCM_ORD_CreateWrapKey, */31, 2}, + {/*TCM_ORD_DiSableForceClear, */94, 0}, + {/*TCM_ORD_DiSableOwnerClear, */92, 0}, + {/*TCM_ORD_EStabliShTranSport, */230, 0}, + {/*TCM_ORD_ExecuteTranSport, */231, 2}, + {/*TCM_ORD_Extend, */20, 0}, + {/*TCM_ORD_FieldUpgrade, */170, 2}, + {/*TCM_ORD_FluShSpecific, */186, 0}, + {/*TCM_ORD_ForceClear, */93, 0}, + {/*TCM_ORD_GetAuditDigeSt, */133, 0}, + {/*TCM_ORD_GetAuditDigeStSigned, */134, 1}, + {/*TCM_ORD_GetCapability, */101, 0}, + {/*TCM_ORD_GetPubKey, */33, 0}, + {/*TCM_ORD_GetRandoM, */70, 0}, + {/*TCM_ORD_GetTeStReSult, */84, 0}, + {/*TCM_ORD_GetTickS, */241, 0}, + {/*TCM_ORD_IncreMentCounter, */221, 0}, + {/*TCM_ORD_LoadContext, */185, 1}, + {/*TCM_ORD_MakeIdentity, */121, 2}, + {/*TCM_ORD_NV_DefineSpace, */204, 0}, + {/*TCM_ORD_NV_ReadValue, */207, 0}, + {/*TCM_ORD_NV_ReadValueAuth, */208, 0}, + {/*TCM_ORD_NV_WriteValue, */205, 0}, + {/*TCM_ORD_NV_WriteValueAuth, */206, 0}, + {/*TCM_ORD_OwnerClear, */91, 0}, + {/*TCM_ORD_OwnerReadInternalPub, */129, 0}, + {/*TCM_ORD_OwnerSetDiSable, */110, 0}, + {/*TCM_ORD_PCR_ReSet, */200, 0}, + {/*TCM_ORD_PcrRead, */21, 0}, + {/*TCM_ORD_PhySicalDiSable, */112, 0}, + {/*TCM_ORD_PhySicalEnable, */111, 0}, + {/*TCM_ORD_PhySicalSetDeactivated, */114, 0}, + {/*TCM_ORD_Quote, */22, 1}, + {/*TCM_ORD_QuoteM, */62, 1}, + {/*TCM_ORD_ReadCounter, */222, 0}, + {/*TCM_ORD_ReadPubek, */124, 0}, + {/*TCM_ORD_ReleaSeCounter, */223, 0}, + {/*TCM_ORD_ReleaSeCounterOwner, */224, 0}, + {/*TCM_ORD_ReleaSeTranSportSigned, */232, 1}, + {/*TCM_ORD_ReSetLockValue, */64, 0}, + {/*TCM_ORD_RevokeTruSt, */128, 0}, + {/*TCM_ORD_SaveContext, */184, 1}, + {/*TCM_ORD_SaveState, */152, 1}, + {/*TCM_ORD_Seal, */23, 1}, + {/*TCM_ORD_Sealx, */61, 1}, + {/*TCM_ORD_SelfTeStFull, */80, 2}, + {/*TCM_ORD_SetCapability, */63, 0}, + {/*TCM_ORD_SetOperatorAuth, */116, 0}, + {/*TCM_ORD_SetOrdinalAuditStatuS, */141, 0}, + {/*TCM_ORD_SetOwnerInStall, */113, 0}, + {/*TCM_ORD_SetTeMpDeactivated, */115, 0}, + {/*TCM_ORD_Sign, */60, 1}, + {/*TCM_ORD_Startup, */153, 0}, + {/*TCM_ORD_TakeOwnerShip, */13, 1}, + {/*TCM_ORD_TickStaMpBlob, */242, 1}, + {/*TCM_ORD_UnSeal, */24, 1}, + {/*TSC_ORD_PhySicalPreSence, */10, 0}, + {/*TSC_ORD_ReSetEStabliShMentBit, */11, 0}, + {/*TCM_ORD_WrapKey, */189, 2}, + {/*TCM_ORD_APcreate, */191, 0}, + {/*TCM_ORD_APTerMinate, */192, 0}, + {/*TCM_ORD_CreateMigratedBlob, */193, 1}, + {/*TCM_ORD_ConvertMigratedBlob, */194, 1}, + {/*TCM_ORD_AuthorizeMigrationKey, */195, 0}, + {/*TCM_ORD_SMS4Encrypt, */197, 1}, + {/*TCM_ORD_SMS4Decrypt, */198, 1}, + {/*TCM_ORD_ReadEKCert, */199, 1}, + {/*TCM_ORD_WriteEKCert, */233, 1}, + {/*TCM_ORD_SCHStart, */234, 0}, + {/*TCM_ORD_SCHUpdata, */235, 0}, + {/*TCM_ORD_SCHCoMplete, */236, 0}, + {/*TCM_ORD_SCHCoMpleteExtend, */237, 0}, + {/*TCM_ORD_ECCDecrypt, */238, 1}, + {/*TCM_ORD_LoadKey, */239, 1}, + {/*TCM_ORD_CreateEndorSeMentKeyPair, */120, 2}, + {/*TCM_ORD_CreateRevocableEK, */127, 2}, + {/*TCM_ORD_ReleaSeECCExchangeSeSSion, */174, 1}, + {/*TCM_ORD_CreateECCExchangeSeSSion, */175, 1}, + {/*TCM_ORD_GetKeyECCExchangeSeSSion, */176, 1}, + {/*TCM_ORD_ActivatePEK, */217, 1}, + {/*TCM_ORD_ActivatePEKCert, */218, 1}, + {0, 0} +}; + +static void user_reader_timeout(struct timer_list *t) +{ + struct tcm_chip *chip = from_timer(chip, t, user_read_timer); + + schedule_work(&chip->work); +} + +static void timeout_work(struct work_struct *work) +{ + struct tcm_chip *chip = container_of(work, struct tcm_chip, work); + + mutex_lock(&chip->buffer_mutex); + atomic_set(&chip->data_pending, 0); + memset(chip->data_buffer, 0, TCM_BUFSIZE); + mutex_unlock(&chip->buffer_mutex); +} + +unsigned long tcm_calc_ordinal_duration(struct tcm_chip *chip, + u32 ordinal) +{ + int duration_idx = TCM_UNDEFINED; + int duration = 0; + int i = 0; + + for (i = 0; i < TCM_MAX_ORDINAL; i++) { + if (ordinal == TCM_Command_List[i].ordinal) { + duration_idx = TCM_Command_List[i].DURATION; + break; + } + } + + if (duration_idx != TCM_UNDEFINED) + duration = chip->vendor.duration[duration_idx]; + if (duration <= 0) + return 2 * 60 * HZ; + else + return duration; +} +EXPORT_SYMBOL_GPL(tcm_calc_ordinal_duration); + +/* + * Internal kernel interface to transmit TCM commands + * buff format: TAG(2 bytes) + Total Size(4 bytes ) + + * Command Ordinal(4 bytes ) + ...... + */ +static ssize_t tcm_transmit(struct tcm_chip *chip, const char *buf, + size_t bufsiz) +{ + ssize_t rc = 0; + u32 count = 0, ordinal = 0; + unsigned long stop = 0; + + count = be32_to_cpu(*((__be32 *)(buf + 2))); /* buff size */ + ordinal = be32_to_cpu(*((__be32 *)(buf + 6))); /* command ordinal */ + + if (count == 0) + return -ENODATA; + if (count > bufsiz) { /* buff size err ,invalid buff stream */ + dev_err(chip->dev, "invalid count value %x, %zx\n", + count, bufsiz); + return -E2BIG; + } + + mutex_lock(&chip->tcm_mutex); /* enter mutex */ + + rc = chip->vendor.send(chip, (u8 *)buf, count); + if (rc < 0) { + dev_err(chip->dev, "%s: tcm_send: error %zd\n", + __func__, rc); + goto out; + } + + if (chip->vendor.irq) + goto out_recv; + + stop = jiffies + tcm_calc_ordinal_duration(chip, + ordinal); /* cmd duration */ + do { + u8 status = chip->vendor.status(chip); + + if ((status & chip->vendor.req_complete_mask) == + chip->vendor.req_complete_val) + goto out_recv; + + if ((status == chip->vendor.req_canceled)) { + dev_err(chip->dev, "Operation Canceled\n"); + rc = -ECANCELED; + goto out; + } + + msleep(TCM_TIMEOUT); /* CHECK */ + rmb(); + } while (time_before(jiffies, stop)); + /* time out */ + chip->vendor.cancel(chip); + dev_err(chip->dev, "Operation Timed out\n"); + rc = -ETIME; + goto out; + +out_recv: + rc = chip->vendor.recv(chip, (u8 *)buf, bufsiz); + if (rc < 0) + dev_err(chip->dev, "%s: tcm_recv: error %zd\n", + __func__, rc); +out: + mutex_unlock(&chip->tcm_mutex); + return rc; +} + +#define TCM_DIGEST_SIZE 32 +#define TCM_ERROR_SIZE 10 +#define TCM_RET_CODE_IDX 6 +#define TCM_GET_CAP_RET_SIZE_IDX 10 +#define TCM_GET_CAP_RET_UINT32_1_IDX 14 +#define TCM_GET_CAP_RET_UINT32_2_IDX 18 +#define TCM_GET_CAP_RET_UINT32_3_IDX 22 +#define TCM_GET_CAP_RET_UINT32_4_IDX 26 +#define TCM_GET_CAP_PERM_DISABLE_IDX 16 +#define TCM_GET_CAP_PERM_INACTIVE_IDX 18 +#define TCM_GET_CAP_RET_BOOL_1_IDX 14 +#define TCM_GET_CAP_TEMP_INACTIVE_IDX 16 + +#define TCM_CAP_IDX 13 +#define TCM_CAP_SUBCAP_IDX 21 + +enum tcm_capabilities { + TCM_CAP_FLAG = 4, + TCM_CAP_PROP = 5, +}; + +enum tcm_sub_capabilities { + TCM_CAP_PROP_PCR = 0x1, /* tcm 0x101 */ + TCM_CAP_PROP_MANUFACTURER = 0x3, /* tcm 0x103 */ + TCM_CAP_FLAG_PERM = 0x8, /* tcm 0x108 */ + TCM_CAP_FLAG_VOL = 0x9, /* tcm 0x109 */ + TCM_CAP_PROP_OWNER = 0x11, /* tcm 0x101 */ + TCM_CAP_PROP_TIS_TIMEOUT = 0x15, /* tcm 0x115 */ + TCM_CAP_PROP_TIS_DURATION = 0x20, /* tcm 0x120 */ +}; + +/* + * This is a semi generic GetCapability command for use + * with the capability type TCM_CAP_PROP or TCM_CAP_FLAG + * and their associated sub_capabilities. + */ + +static const u8 tcm_cap[] = { + 0, 193, /* TCM_TAG_RQU_COMMAND 0xc1*/ + 0, 0, 0, 22, /* length */ + 0, 0, 128, 101, /* TCM_ORD_GetCapability */ + 0, 0, 0, 0, /* TCM_CAP_<TYPE> */ + 0, 0, 0, 4, /* TCM_CAP_SUB_<TYPE> size */ + 0, 0, 1, 0 /* TCM_CAP_SUB_<TYPE> */ +}; + +static ssize_t transmit_cmd(struct tcm_chip *chip, u8 *data, int len, + char *desc) +{ + int err = 0; + + len = tcm_transmit(chip, data, len); + if (len < 0) + return len; + if (len == TCM_ERROR_SIZE) { + err = be32_to_cpu(*((__be32 *)(data + TCM_RET_CODE_IDX))); + dev_dbg(chip->dev, "A TCM error (%d) occurred %s\n", err, desc); + return err; + } + return 0; +} + +/* + * Get default timeouts value form tcm by GetCapability with TCM_CAP_PROP_TIS_TIMEOUT prop + */ +void tcm_get_timeouts(struct tcm_chip *chip) +{ + u8 data[max_t(int, ARRAY_SIZE(tcm_cap), 30)]; + ssize_t rc = 0; + u32 timeout = 0; + + memcpy(data, tcm_cap, sizeof(tcm_cap)); + data[TCM_CAP_IDX] = TCM_CAP_PROP; + data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_PROP_TIS_TIMEOUT; + + rc = transmit_cmd(chip, data, sizeof(data), + "attempting to determine the timeouts"); + if (rc) + goto duration; + + if (be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_SIZE_IDX))) != + 4 * sizeof(u32)) + goto duration; + + /* Don't overwrite default if value is 0 */ + timeout = be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_UINT32_1_IDX))); + if (timeout) + chip->vendor.timeout_a = msecs_to_jiffies(timeout); + timeout = be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_UINT32_2_IDX))); + if (timeout) + chip->vendor.timeout_b = msecs_to_jiffies(timeout); + timeout = be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_UINT32_3_IDX))); + if (timeout) + chip->vendor.timeout_c = msecs_to_jiffies(timeout); + timeout = be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_UINT32_4_IDX))); + if (timeout) + chip->vendor.timeout_d = msecs_to_jiffies(timeout); + +duration: + memcpy(data, tcm_cap, sizeof(tcm_cap)); + data[TCM_CAP_IDX] = TCM_CAP_PROP; + data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_PROP_TIS_DURATION; + + rc = transmit_cmd(chip, data, sizeof(data), + "attempting to determine the durations"); + if (rc) + return; + + if (be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_SIZE_IDX))) != + 3 * sizeof(u32)) + return; + + chip->vendor.duration[TCM_SHORT] = + msecs_to_jiffies(be32_to_cpu(*((__be32 *)(data + + TCM_GET_CAP_RET_UINT32_1_IDX)))); + chip->vendor.duration[TCM_MEDIUM] = + msecs_to_jiffies(be32_to_cpu(*((__be32 *)(data + + TCM_GET_CAP_RET_UINT32_2_IDX)))); + chip->vendor.duration[TCM_LONG] = + msecs_to_jiffies(be32_to_cpu(*((__be32 *)(data + + TCM_GET_CAP_RET_UINT32_3_IDX)))); +} +EXPORT_SYMBOL_GPL(tcm_get_timeouts); + +ssize_t tcm_show_enabled(struct device *dev, struct device_attribute *attr, + char *buf) +{ + u8 data[max_t(int, ARRAY_SIZE(tcm_cap), 35)]; + ssize_t rc = 0; + struct tcm_chip *chip = dev_get_drvdata(dev); + + if (chip == NULL) + return -ENODEV; + + memcpy(data, tcm_cap, sizeof(tcm_cap)); + data[TCM_CAP_IDX] = TCM_CAP_FLAG; + data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_FLAG_PERM; + + rc = transmit_cmd(chip, data, sizeof(data), + "attemtping to determine the permanent state"); + if (rc) + return 0; + if (data[TCM_GET_CAP_PERM_DISABLE_IDX]) + return sprintf(buf, "disable\n"); + else + return sprintf(buf, "enable\n"); +} +EXPORT_SYMBOL_GPL(tcm_show_enabled); + +ssize_t tcm_show_active(struct device *dev, struct device_attribute *attr, + char *buf) +{ + u8 data[max_t(int, ARRAY_SIZE(tcm_cap), 35)]; + ssize_t rc = 0; + struct tcm_chip *chip = dev_get_drvdata(dev); + + if (chip == NULL) + return -ENODEV; + + memcpy(data, tcm_cap, sizeof(tcm_cap)); + data[TCM_CAP_IDX] = TCM_CAP_FLAG; + data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_FLAG_PERM; + + rc = transmit_cmd(chip, data, sizeof(data), + "attemtping to determine the permanent state"); + if (rc) + return 0; + if (data[TCM_GET_CAP_PERM_INACTIVE_IDX]) + return sprintf(buf, "deactivated\n"); + else + return sprintf(buf, "activated\n"); +} +EXPORT_SYMBOL_GPL(tcm_show_active); + +ssize_t tcm_show_owned(struct device *dev, struct device_attribute *attr, + char *buf) +{ + u8 data[sizeof(tcm_cap)]; + ssize_t rc = 0; + struct tcm_chip *chip = dev_get_drvdata(dev); + + if (chip == NULL) + return -ENODEV; + + memcpy(data, tcm_cap, sizeof(tcm_cap)); + data[TCM_CAP_IDX] = TCM_CAP_PROP; + data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_PROP_OWNER; + + rc = transmit_cmd(chip, data, sizeof(data), + "attempting to determine the owner state"); + if (rc) + return 0; + if (data[TCM_GET_CAP_RET_BOOL_1_IDX]) + return sprintf(buf, "Owner installed\n"); + else + return sprintf(buf, "Owner have not installed\n"); +} +EXPORT_SYMBOL_GPL(tcm_show_owned); + +ssize_t tcm_show_temp_deactivated(struct device *dev, + struct device_attribute *attr, char *buf) +{ + u8 data[sizeof(tcm_cap)]; + ssize_t rc = 0; + struct tcm_chip *chip = dev_get_drvdata(dev); + + if (chip == NULL) + return -ENODEV; + + memcpy(data, tcm_cap, sizeof(tcm_cap)); + data[TCM_CAP_IDX] = TCM_CAP_FLAG; + data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_FLAG_VOL; + + rc = transmit_cmd(chip, data, sizeof(data), + "attempting to determine the temporary state"); + if (rc) + return 0; + if (data[TCM_GET_CAP_TEMP_INACTIVE_IDX]) + return sprintf(buf, "Temp deactivated\n"); + else + return sprintf(buf, "activated\n"); +} +EXPORT_SYMBOL_GPL(tcm_show_temp_deactivated); + +static const u8 pcrread[] = { + 0, 193, /* TCM_TAG_RQU_COMMAND */ + 0, 0, 0, 14, /* length */ + 0, 0, 128, 21, /* TCM_ORD_PcrRead */ + 0, 0, 0, 0 /* PCR index */ +}; + +ssize_t tcm_show_pcrs(struct device *dev, struct device_attribute *attr, + char *buf) +{ + u8 data[1024]; + ssize_t rc = 0; + int i = 0, j = 0, num_pcrs = 0; + __be32 index = 0; + char *str = buf; + struct tcm_chip *chip = dev_get_drvdata(dev); + + if (chip == NULL) + return -ENODEV; + + memcpy(data, tcm_cap, sizeof(tcm_cap)); + data[TCM_CAP_IDX] = TCM_CAP_PROP; + data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_PROP_PCR; + + rc = transmit_cmd(chip, data, sizeof(data), + "attempting to determine the number of PCRS"); + if (rc) + return 0; + + num_pcrs = be32_to_cpu(*((__be32 *)(data + 14))); + for (i = 0; i < num_pcrs; i++) { + memcpy(data, pcrread, sizeof(pcrread)); + index = cpu_to_be32(i); + memcpy(data + 10, &index, 4); + rc = transmit_cmd(chip, data, sizeof(data), + "attempting to read a PCR"); + if (rc) + goto out; + str += sprintf(str, "PCR-%02d: ", i); + for (j = 0; j < TCM_DIGEST_SIZE; j++) + str += sprintf(str, "%02X ", *(data + 10 + j)); + str += sprintf(str, "\n"); + memset(data, 0, 1024); + } +out: + return str - buf; +} +EXPORT_SYMBOL_GPL(tcm_show_pcrs); + +#define READ_PUBEK_RESULT_SIZE 128 +static const u8 readpubek[] = { + 0, 193, /* TCM_TAG_RQU_COMMAND */ + 0, 0, 0, 42, /* length */ + 0, 0, 128, 124, /* TCM_ORD_ReadPubek */ + 0, 0, 0, 0, 0, 0, 0, 0, /* NONCE */ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 +}; + +ssize_t tcm_show_pubek(struct device *dev, struct device_attribute *attr, + char *buf) +{ + u8 data[READ_PUBEK_RESULT_SIZE] = {0}; + ssize_t err = 0; + int i = 0, rc = 0; + char *str = buf; + struct tcm_chip *chip = dev_get_drvdata(dev); + + if (chip == NULL) + return -ENODEV; + + memcpy(data, readpubek, sizeof(readpubek)); + + err = transmit_cmd(chip, data, sizeof(data), + "attempting to read the PUBEK"); + if (err) + goto out; + + str += sprintf(str, "PUBEK:"); + for (i = 0 ; i < 65 ; i++) { + if ((i) % 16 == 0) + str += sprintf(str, "\n"); + str += sprintf(str, "%02X ", data[i+10]); + } + + str += sprintf(str, "\n"); +out: + rc = str - buf; + return rc; +} +EXPORT_SYMBOL_GPL(tcm_show_pubek); + +#define CAP_VERSION_1_1 6 +#define CAP_VERSION_1_2 0x1A +#define CAP_VERSION_IDX 13 +static const u8 cap_version[] = { + 0, 193, /* TCM_TAG_RQU_COMMAND */ + 0, 0, 0, 18, /* length */ + 0, 0, 128, 101, /* TCM_ORD_GetCapability */ + 0, 0, 0, 0, + 0, 0, 0, 0 +}; + +ssize_t tcm_show_caps(struct device *dev, struct device_attribute *attr, + char *buf) +{ + u8 data[max_t(int, max(ARRAY_SIZE(tcm_cap), ARRAY_SIZE(cap_version)), 30)]; + ssize_t rc = 0; + char *str = buf; + struct tcm_chip *chip = dev_get_drvdata(dev); + + if (chip == NULL) + return -ENODEV; + + memcpy(data, tcm_cap, sizeof(tcm_cap)); + data[TCM_CAP_IDX] = TCM_CAP_PROP; + data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_PROP_MANUFACTURER; + + rc = transmit_cmd(chip, data, sizeof(data), + "attempting to determine the manufacturer"); + if (rc) + return 0; + + str += sprintf(str, "Manufacturer: 0x%x\n", + be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_UINT32_1_IDX)))); + + memcpy(data, cap_version, sizeof(cap_version)); + data[CAP_VERSION_IDX] = CAP_VERSION_1_1; + rc = transmit_cmd(chip, data, sizeof(data), + "attempting to determine the 1.1 version"); + if (rc) + goto out; + + str += sprintf(str, "Firmware version: %02X.%02X.%02X.%02X\n", + (int)data[14], (int)data[15], (int)data[16], + (int)data[17]); + +out: + return str - buf; +} +EXPORT_SYMBOL_GPL(tcm_show_caps); + +ssize_t tcm_store_cancel(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct tcm_chip *chip = dev_get_drvdata(dev); + + if (chip == NULL) + return 0; + + chip->vendor.cancel(chip); + return count; +} +EXPORT_SYMBOL_GPL(tcm_store_cancel); + +/* + * Device file system interface to the TCM + * when App call file open in usr space ,this func will respone + */ +int tcm_open(struct inode *inode, struct file *file) +{ + int rc = 0, minor = iminor(inode); + struct tcm_chip *chip = NULL, *pos = NULL; + + spin_lock(&driver_lock); + + list_for_each_entry(pos, &tcm_chip_list, list) { + if (pos->vendor.miscdev.minor == minor) { + chip = pos; + break; + } + } + + if (chip == NULL) { + rc = -ENODEV; + goto err_out; + } + + if (chip->num_opens) { + dev_dbg(chip->dev, "Another process owns this TCM\n"); + rc = -EBUSY; + goto err_out; + } + + chip->num_opens++; + get_device(chip->dev); + + spin_unlock(&driver_lock); + + chip->data_buffer = kmalloc(TCM_BUFSIZE * sizeof(u8), GFP_KERNEL); + if (chip->data_buffer == NULL) { + chip->num_opens--; + put_device(chip->dev); + return -ENOMEM; + } + + atomic_set(&chip->data_pending, 0); + + file->private_data = chip; + return 0; + +err_out: + spin_unlock(&driver_lock); + return rc; +} +EXPORT_SYMBOL_GPL(tcm_open); + +int tcm_release(struct inode *inode, struct file *file) +{ + struct tcm_chip *chip = file->private_data; + + spin_lock(&driver_lock); + file->private_data = NULL; + chip->num_opens--; + del_singleshot_timer_sync(&chip->user_read_timer); + flush_work(&chip->work); + atomic_set(&chip->data_pending, 0); + put_device(chip->dev); + kfree(chip->data_buffer); + spin_unlock(&driver_lock); + return 0; +} +EXPORT_SYMBOL_GPL(tcm_release); + +ssize_t tcm_write(struct file *file, const char __user *buf, + size_t size, loff_t *off) +{ + struct tcm_chip *chip = file->private_data; + int in_size = size, out_size; + + /* + * cannot perform a write until the read has cleared + * either via tcm_read or a user_read_timer timeout + */ + while (atomic_read(&chip->data_pending) != 0) + msleep(TCM_TIMEOUT); + + mutex_lock(&chip->buffer_mutex); + + if (in_size > TCM_BUFSIZE) + in_size = TCM_BUFSIZE; + + if (copy_from_user(chip->data_buffer, (void __user *)buf, in_size)) { + mutex_unlock(&chip->buffer_mutex); + return -EFAULT; + } + + /* atomic tcm command send and result receive */ + out_size = tcm_transmit(chip, chip->data_buffer, TCM_BUFSIZE); + + if (out_size >= 0) { + atomic_set(&chip->data_pending, out_size); + mutex_unlock(&chip->buffer_mutex); + + /* Set a timeout by which the reader must come claim the result */ + mod_timer(&chip->user_read_timer, jiffies + (60 * HZ)); + } else + mutex_unlock(&chip->buffer_mutex); + + return in_size; +} +EXPORT_SYMBOL_GPL(tcm_write); + +ssize_t tcm_read(struct file *file, char __user *buf, + size_t size, loff_t *off) +{ + struct tcm_chip *chip = file->private_data; + int ret_size = 0; + + del_singleshot_timer_sync(&chip->user_read_timer); + flush_work(&chip->work); + ret_size = atomic_read(&chip->data_pending); + atomic_set(&chip->data_pending, 0); + if (ret_size > 0) { /* relay data */ + if (size < ret_size) + ret_size = size; + + mutex_lock(&chip->buffer_mutex); + if (copy_to_user(buf, chip->data_buffer, ret_size)) + ret_size = -EFAULT; + mutex_unlock(&chip->buffer_mutex); + } + + return ret_size; +} +EXPORT_SYMBOL_GPL(tcm_read); + +void tcm_remove_hardware(struct device *dev) +{ + struct tcm_chip *chip = dev_get_drvdata(dev); + + if (chip == NULL) { + dev_err(dev, "No device data found\n"); + return; + } + + spin_lock(&driver_lock); + list_del(&chip->list); + spin_unlock(&driver_lock); + + dev_set_drvdata(dev, NULL); + misc_deregister(&chip->vendor.miscdev); + kfree(chip->vendor.miscdev.name); + + sysfs_remove_group(&dev->kobj, chip->vendor.attr_group); + /* tcm_bios_log_teardown(chip->bios_dir); */ + + clear_bit(chip->dev_num, dev_mask); + kfree(chip); + put_device(dev); +} +EXPORT_SYMBOL_GPL(tcm_remove_hardware); + +static u8 savestate[] = { + 0, 193, /* TCM_TAG_RQU_COMMAND */ + 0, 0, 0, 10, /* blob length (in bytes) */ + 0, 0, 128, 152 /* TCM_ORD_SaveState */ +}; + +/* + * We are about to suspend. Save the TCM state + * so that it can be restored. + */ +int tcm_pm_suspend(struct device *dev, pm_message_t pm_state) +{ + struct tcm_chip *chip = dev_get_drvdata(dev); + + if (chip == NULL) + return -ENODEV; + + tcm_transmit(chip, savestate, sizeof(savestate)); + return 0; +} +EXPORT_SYMBOL_GPL(tcm_pm_suspend); + +int tcm_pm_suspend_p(struct device *dev) +{ + struct tcm_chip *chip = dev_get_drvdata(dev); + + if (chip == NULL) + return -ENODEV; + + tcm_transmit(chip, savestate, sizeof(savestate)); + return 0; +} +EXPORT_SYMBOL_GPL(tcm_pm_suspend_p); + +void tcm_startup(struct tcm_chip *chip) +{ + u8 start_up[] = { + 0, 193, /* TCM_TAG_RQU_COMMAND */ + 0, 0, 0, 12, /* blob length (in bytes) */ + 0, 0, 128, 153, /* TCM_ORD_SaveState */ + 0, 1 + }; + if (chip == NULL) + return; + tcm_transmit(chip, start_up, sizeof(start_up)); +} +EXPORT_SYMBOL_GPL(tcm_startup); + +/* + * Resume from a power safe. The BIOS already restored + * the TCM state. + */ +int tcm_pm_resume(struct device *dev) +{ + u8 start_up[] = { + 0, 193, /* TCM_TAG_RQU_COMMAND */ + 0, 0, 0, 12, /* blob length (in bytes) */ + 0, 0, 128, 153, /* TCM_ORD_SaveState */ + 0, 1 + }; + struct tcm_chip *chip = dev_get_drvdata(dev); + /* dev_info(chip->dev ,"--call tcm_pm_resume\n"); */ + if (chip == NULL) + return -ENODEV; + + tcm_transmit(chip, start_up, sizeof(start_up)); + return 0; +} +EXPORT_SYMBOL_GPL(tcm_pm_resume); + +/* + * Called from tcm_<specific>.c probe function only for devices + * the driver has determined it should claim. Prior to calling + * this function the specific probe function has called pci_enable_device + * upon errant exit from this function specific probe function should call + * pci_disable_device + */ +struct tcm_chip *tcm_register_hardware(struct device *dev, + const struct tcm_vendor_specific *entry) +{ + int rc; +#define DEVNAME_SIZE 7 + + char *devname = NULL; + struct tcm_chip *chip = NULL; + + /* Driver specific per-device data */ + chip = kzalloc(sizeof(*chip), GFP_KERNEL); + if (chip == NULL) { + dev_err(dev, "chip kzalloc err\n"); + return NULL; + } + + mutex_init(&chip->buffer_mutex); + mutex_init(&chip->tcm_mutex); + INIT_LIST_HEAD(&chip->list); + + INIT_WORK(&chip->work, timeout_work); + timer_setup(&chip->user_read_timer, user_reader_timeout, 0); + + memcpy(&chip->vendor, entry, sizeof(struct tcm_vendor_specific)); + + chip->dev_num = find_first_zero_bit(dev_mask, TCM_NUM_DEVICES); + + if (chip->dev_num >= TCM_NUM_DEVICES) { + dev_err(dev, "No available tcm device numbers\n"); + kfree(chip); + return NULL; + } else if (chip->dev_num == 0) + chip->vendor.miscdev.minor = TCM_MINOR; + else + chip->vendor.miscdev.minor = MISC_DYNAMIC_MINOR; + + set_bit(chip->dev_num, dev_mask); + + devname = kmalloc(DEVNAME_SIZE, GFP_KERNEL); + scnprintf(devname, DEVNAME_SIZE, "%s%d", "tcm", chip->dev_num); + chip->vendor.miscdev.name = devname; + + /* chip->vendor.miscdev.dev = dev; */ + + chip->dev = get_device(dev); + + if (misc_register(&chip->vendor.miscdev)) { + dev_err(chip->dev, + "unable to misc_register %s, minor %d\n", + chip->vendor.miscdev.name, + chip->vendor.miscdev.minor); + put_device(dev); + clear_bit(chip->dev_num, dev_mask); + kfree(chip); + kfree(devname); + return NULL; + } + + spin_lock(&driver_lock); + dev_set_drvdata(dev, chip); + list_add(&chip->list, &tcm_chip_list); + spin_unlock(&driver_lock); + + rc = sysfs_create_group(&dev->kobj, chip->vendor.attr_group); + /* chip->bios_dir = tcm_bios_log_setup(devname); */ + + return chip; +} +EXPORT_SYMBOL_GPL(tcm_register_hardware); + +static int __init tcm_init_module(void) +{ + return 0; +} + +static void __exit tcm_exit_module(void) +{ +} + +module_init(tcm_init_module); +module_exit(tcm_exit_module); + +MODULE_AUTHOR("Nationz Technologies Inc."); +MODULE_DESCRIPTION("TCM Driver"); +MODULE_VERSION("1.1.1.0"); +MODULE_LICENSE("GPL"); diff --git a/drivers/staging/gmjstcm/tcm.h b/drivers/staging/gmjstcm/tcm.h new file mode 100644 index 000000000000..b8cafe78d590 --- /dev/null +++ b/drivers/staging/gmjstcm/tcm.h @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2009 Nationz Technologies Inc. + * + */ +#include <linux/module.h> +#include <linux/types.h> +#include <linux/pci.h> +#include <linux/delay.h> +#include <linux/fs.h> +#include <linux/miscdevice.h> +#include <linux/platform_device.h> +#include <linux/io.h> + +struct device; +struct tcm_chip; + +enum tcm_timeout { + TCM_TIMEOUT = 5, +}; + +/* TCM addresses */ +enum tcm_addr { + TCM_SUPERIO_ADDR = 0x2E, + TCM_ADDR = 0x4E, +}; + +extern ssize_t tcm_show_pubek(struct device *, struct device_attribute *attr, + char *); +extern ssize_t tcm_show_pcrs(struct device *, struct device_attribute *attr, + char *); +extern ssize_t tcm_show_caps(struct device *, struct device_attribute *attr, + char *); +extern ssize_t tcm_store_cancel(struct device *, struct device_attribute *attr, + const char *, size_t); +extern ssize_t tcm_show_enabled(struct device *, struct device_attribute *attr, + char *); +extern ssize_t tcm_show_active(struct device *, struct device_attribute *attr, + char *); +extern ssize_t tcm_show_owned(struct device *, struct device_attribute *attr, + char *); +extern ssize_t tcm_show_temp_deactivated(struct device *, + struct device_attribute *attr, char *); + +struct tcm_vendor_specific { + const u8 req_complete_mask; + const u8 req_complete_val; + const u8 req_canceled; + void __iomem *iobase; /* ioremapped address */ + void __iomem *iolbc; + unsigned long base; /* TCM base address */ + + int irq; + + int region_size; + int have_region; + + int (*recv)(struct tcm_chip *, u8 *, size_t); + int (*send)(struct tcm_chip *, u8 *, size_t); + void (*cancel)(struct tcm_chip *); + u8 (*status)(struct tcm_chip *); + struct miscdevice miscdev; + struct attribute_group *attr_group; + struct list_head list; + int locality; + unsigned long timeout_a, timeout_b, timeout_c, timeout_d; /* jiffies */ + unsigned long duration[3]; /* jiffies */ + + wait_queue_head_t read_queue; + wait_queue_head_t int_queue; +}; + +struct tcm_chip { + struct device *dev; /* Device stuff */ + + int dev_num; /* /dev/tcm# */ + int num_opens; /* only one allowed */ + int time_expired; + + /* Data passed to and from the tcm via the read/write calls */ + u8 *data_buffer; + atomic_t data_pending; + struct mutex buffer_mutex; + + struct timer_list user_read_timer; /* user needs to claim result */ + struct work_struct work; + struct mutex tcm_mutex; /* tcm is processing */ + + struct tcm_vendor_specific vendor; + + struct dentry **bios_dir; + + struct list_head list; +}; + +#define to_tcm_chip(n) container_of(n, struct tcm_chip, vendor) + +static inline int tcm_read_index(int base, int index) +{ + outb(index, base); + return inb(base+1) & 0xFF; +} + +static inline void tcm_write_index(int base, int index, int value) +{ + outb(index, base); + outb(value & 0xFF, base+1); +} +extern void tcm_startup(struct tcm_chip *); +extern void tcm_get_timeouts(struct tcm_chip *); +extern unsigned long tcm_calc_ordinal_duration(struct tcm_chip *, u32); +extern struct tcm_chip *tcm_register_hardware(struct device *, + const struct tcm_vendor_specific *); +extern int tcm_open(struct inode *, struct file *); +extern int tcm_release(struct inode *, struct file *); +extern ssize_t tcm_write(struct file *, const char __user *, size_t, + loff_t *); +extern ssize_t tcm_read(struct file *, char __user *, size_t, loff_t *); +extern void tcm_remove_hardware(struct device *); +extern int tcm_pm_suspend(struct device *, pm_message_t); +extern int tcm_pm_suspend_p(struct device *); +extern int tcm_pm_resume(struct device *); + diff --git a/drivers/staging/gmjstcm/tcm_tis_spi.c b/drivers/staging/gmjstcm/tcm_tis_spi.c new file mode 100644 index 000000000000..db30a5b4c47d --- /dev/null +++ b/drivers/staging/gmjstcm/tcm_tis_spi.c @@ -0,0 +1,868 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Kylin Tech. Co., Ltd. + */ + +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/interrupt.h> +#include <linux/wait.h> +#include <linux/acpi.h> +#include <linux/spi/spi.h> + +#include "tcm.h" + +#if !defined(CONFIG_KYLINOS_SERVER) && !defined(CONFIG_KYLINOS_DESKTOP) +static int is_ft_all(void) { + return 0; +} +#endif + +#define TCM_HEADER_SIZE 10 + +static bool tcm_debug; +module_param_named(debug, tcm_debug, bool, 0600); +MODULE_PARM_DESC(debug, "Turn TCM debugging mode on and off"); + +#define tcm_dbg(fmt, args...) \ +{ \ + if (tcm_debug) \ + pr_err(fmt, ## args); \ +} + +enum tis_access { + TCM_ACCESS_VALID = 0x80, + TCM_ACCESS_ACTIVE_LOCALITY = 0x20, + TCM_ACCESS_REQUEST_PENDING = 0x04, + TCM_ACCESS_REQUEST_USE = 0x02, +}; + +enum tis_status { + TCM_STS_VALID = 0x80, + TCM_STS_COMMAND_READY = 0x40, + TCM_STS_GO = 0x20, + TCM_STS_DATA_AVAIL = 0x10, + TCM_STS_DATA_EXPECT = 0x08, +}; + +enum tis_int_flags { + TCM_GLOBAL_INT_ENABLE = 0x80000000, + TCM_INTF_BURST_COUNT_STATIC = 0x100, + TCM_INTF_CMD_READY_INT = 0x080, + TCM_INTF_INT_EDGE_FALLING = 0x040, + TCM_INTF_INT_EDGE_RISING = 0x020, + TCM_INTF_INT_LEVEL_LOW = 0x010, + TCM_INTF_INT_LEVEL_HIGH = 0x008, + TCM_INTF_LOCALITY_CHANGE_INT = 0x004, + TCM_INTF_STS_VALID_INT = 0x002, + TCM_INTF_DATA_AVAIL_INT = 0x001, +}; + +enum tis_defaults { + TIS_SHORT_TIMEOUT = 750, /* ms */ + TIS_LONG_TIMEOUT = 2000, /* 2 sec */ +}; + +#define TCM_ACCESS(l) (0x0000 | ((l) << 12)) +#define TCM_INT_ENABLE(l) (0x0008 | ((l) << 12)) /* interperet */ +#define TCM_INT_VECTOR(l) (0x000C | ((l) << 12)) +#define TCM_INT_STATUS(l) (0x0010 | ((l) << 12)) +#define TCM_INTF_CAPS(l) (0x0014 | ((l) << 12)) +#define TCM_STS(l) (0x0018 | ((l) << 12)) +#define TCM_DATA_FIFO(l) (0x0024 | ((l) << 12)) + +#define TCM_DID_VID(l) (0x0F00 | ((l) << 12)) +#define TCM_RID(l) (0x0F04 | ((l) << 12)) + +#define TIS_MEM_BASE_huawei 0x3fed40000LL + +#define MAX_SPI_FRAMESIZE 64 + +// +#define _CPU_FT2000A4 +#define REUSE_CONF_REG_BASE 0x28180208 +#define REUSE_GPIO1_A5_BASE 0x28005000 + +static void *__iomem reuse_conf_reg; +static void *__iomem gpio1_a5; + +// +static LIST_HEAD(tis_chips); +static DEFINE_SPINLOCK(tis_lock); + +struct chip_data { + u8 cs; + u8 tmode; + u8 type; + u8 poll_mode; + u16 clk_div; + u32 speed_hz; + void (*cs_control)(u32 command); +}; + +struct tcm_tis_spi_phy { + struct spi_device *spi_device; + struct completion ready; + u8 *iobuf; +}; + +int tcm_tis_spi_transfer(struct device *dev, u32 addr, u16 len, + u8 *in, const u8 *out) +{ + struct tcm_tis_spi_phy *phy = dev_get_drvdata(dev); + int ret = 0; + struct spi_message m; + struct spi_transfer spi_xfer; + u8 transfer_len; + + tcm_dbg("TCM-dbg: %s, addr: 0x%x, len: %x, %s\n", + __func__, addr, len, (in) ? "in" : "out"); + + spi_bus_lock(phy->spi_device->master); + + /* set gpio1_a5 to LOW */ + if (is_ft_all() && (phy->spi_device->chip_select == 0)) { + iowrite32(0x0, gpio1_a5); + } + + while (len) { + transfer_len = min_t(u16, len, MAX_SPI_FRAMESIZE); + + phy->iobuf[0] = (in ? 0x80 : 0) | (transfer_len - 1); + phy->iobuf[1] = 0xd4; + phy->iobuf[2] = addr >> 8; + phy->iobuf[3] = addr; + + memset(&spi_xfer, 0, sizeof(spi_xfer)); + spi_xfer.tx_buf = phy->iobuf; + spi_xfer.rx_buf = phy->iobuf; + spi_xfer.len = 4; + spi_xfer.cs_change = 1; + + spi_message_init(&m); + spi_message_add_tail(&spi_xfer, &m); + ret = spi_sync_locked(phy->spi_device, &m); + if (ret < 0) + goto exit; + + spi_xfer.cs_change = 0; + spi_xfer.len = transfer_len; + spi_xfer.delay_usecs = 5; + + if (in) { + spi_xfer.tx_buf = NULL; + } else if (out) { + spi_xfer.rx_buf = NULL; + memcpy(phy->iobuf, out, transfer_len); + out += transfer_len; + } + + spi_message_init(&m); + spi_message_add_tail(&spi_xfer, &m); + reinit_completion(&phy->ready); + ret = spi_sync_locked(phy->spi_device, &m); + if (ret < 0) + goto exit; + + if (in) { + memcpy(in, phy->iobuf, transfer_len); + in += transfer_len; + } + + len -= transfer_len; + } + +exit: + /* set gpio1_a5 to HIGH */ + if (is_ft_all() && (phy->spi_device->chip_select == 0)) { + iowrite32(0x20, gpio1_a5); + } + + spi_bus_unlock(phy->spi_device->master); + tcm_dbg("TCM-dbg: ret: %d\n", ret); + return ret; +} + +static int tcm_tis_read8(struct device *dev, + u32 addr, u16 len, u8 *result) +{ + return tcm_tis_spi_transfer(dev, addr, len, result, NULL); +} + +static int tcm_tis_write8(struct device *dev, + u32 addr, u16 len, u8 *value) +{ + return tcm_tis_spi_transfer(dev, addr, len, NULL, value); +} + +static int tcm_tis_readb(struct device *dev, u32 addr, u8 *value) +{ + return tcm_tis_read8(dev, addr, sizeof(u8), value); +} + +static int tcm_tis_writeb(struct device *dev, u32 addr, u8 value) +{ + return tcm_tis_write8(dev, addr, sizeof(u8), &value); +} + +static int tcm_tis_readl(struct device *dev, u32 addr, u32 *result) +{ + int rc; + __le32 result_le; + + rc = tcm_tis_read8(dev, addr, sizeof(u32), (u8 *)&result_le); + tcm_dbg("TCM-dbg: result_le: 0x%x\n", result_le); + if (!rc) + *result = le32_to_cpu(result_le); + + return rc; +} + +static int tcm_tis_writel(struct device *dev, u32 addr, u32 value) +{ + int rc; + __le32 value_le; + + value_le = cpu_to_le32(value); + rc = tcm_tis_write8(dev, addr, sizeof(u32), (u8 *)&value_le); + + return rc; +} + +static int request_locality(struct tcm_chip *chip, int l); +static void release_locality(struct tcm_chip *chip, int l, int force); +static void cleanup_tis(void) +{ + int ret; + u32 inten; + struct tcm_vendor_specific *i, *j; + struct tcm_chip *chip; + + spin_lock(&tis_lock); + list_for_each_entry_safe(i, j, &tis_chips, list) { + chip = to_tcm_chip(i); + ret = tcm_tis_readl(chip->dev, + TCM_INT_ENABLE(chip->vendor.locality), &inten); + if (ret < 0) + return; + + tcm_tis_writel(chip->dev, TCM_INT_ENABLE(chip->vendor.locality), + ~TCM_GLOBAL_INT_ENABLE & inten); + release_locality(chip, chip->vendor.locality, 1); + } + spin_unlock(&tis_lock); +} + +static void tcm_tis_init(struct tcm_chip *chip) +{ + int ret; + u8 rid; + u32 vendor, intfcaps; + + ret = tcm_tis_readl(chip->dev, TCM_DID_VID(0), &vendor); + + if ((vendor & 0xffff) != 0x19f5 && (vendor & 0xffff) != 0x1B4E) + pr_info("there is no Nationz TCM on you computer\n"); + + ret = tcm_tis_readb(chip->dev, TCM_RID(0), &rid); + if (ret < 0) + return; + pr_info("kylin: 2019-09-21 1.2 TCM (device-id 0x%X, rev-id %d)\n", + vendor >> 16, rid); + + /* Figure out the capabilities */ + ret = tcm_tis_readl(chip->dev, + TCM_INTF_CAPS(chip->vendor.locality), &intfcaps); + if (ret < 0) + return; + + if (request_locality(chip, 0) != 0) + pr_err("tcm request_locality err\n"); + + atomic_set(&chip->data_pending, 0); +} + +static void tcm_handle_err(struct tcm_chip *chip) +{ + cleanup_tis(); + tcm_tis_init(chip); +} + +static bool check_locality(struct tcm_chip *chip, int l) +{ + int ret; + u8 access; + + ret = tcm_tis_readb(chip->dev, TCM_ACCESS(l), &access); + tcm_dbg("TCM-dbg: access: 0x%x\n", access); + if (ret < 0) + return false; + + if ((access & (TCM_ACCESS_ACTIVE_LOCALITY | TCM_ACCESS_VALID)) == + (TCM_ACCESS_ACTIVE_LOCALITY | TCM_ACCESS_VALID)) { + chip->vendor.locality = l; + return true; + } + + return false; +} + +static int request_locality(struct tcm_chip *chip, int l) +{ + unsigned long stop; + + if (check_locality(chip, l)) + return l; + + tcm_tis_writeb(chip->dev, TCM_ACCESS(l), TCM_ACCESS_REQUEST_USE); + + /* wait for burstcount */ + stop = jiffies + chip->vendor.timeout_a; + do { + if (check_locality(chip, l)) + return l; + msleep(TCM_TIMEOUT); + } while (time_before(jiffies, stop)); + + return -1; +} + +static void release_locality(struct tcm_chip *chip, int l, int force) +{ + int ret; + u8 access; + + ret = tcm_tis_readb(chip->dev, TCM_ACCESS(l), &access); + if (ret < 0) + return; + if (force || (access & (TCM_ACCESS_REQUEST_PENDING | TCM_ACCESS_VALID)) == + (TCM_ACCESS_REQUEST_PENDING | TCM_ACCESS_VALID)) + tcm_tis_writeb(chip->dev, + TCM_ACCESS(l), TCM_ACCESS_ACTIVE_LOCALITY); +} + +static u8 tcm_tis_status(struct tcm_chip *chip) +{ + int ret; + u8 status; + + ret = tcm_tis_readb(chip->dev, + TCM_STS(chip->vendor.locality), &status); + tcm_dbg("TCM-dbg: status: 0x%x\n", status); + if (ret < 0) + return 0; + + return status; +} + +static void tcm_tis_ready(struct tcm_chip *chip) +{ + /* this causes the current command to be aboreted */ + tcm_tis_writeb(chip->dev, TCM_STS(chip->vendor.locality), + TCM_STS_COMMAND_READY); +} + +static int get_burstcount(struct tcm_chip *chip) +{ + int ret; + unsigned long stop; + u8 tmp, tmp1; + int burstcnt = 0; + + /* wait for burstcount */ + /* which timeout value, spec has 2 answers (c & d) */ + stop = jiffies + chip->vendor.timeout_d; + do { + ret = tcm_tis_readb(chip->dev, + TCM_STS(chip->vendor.locality) + 1, + &tmp); + tcm_dbg("TCM-dbg: burstcnt: 0x%x\n", burstcnt); + if (ret < 0) + return -EINVAL; + ret = tcm_tis_readb(chip->dev, + (TCM_STS(chip->vendor.locality) + 2), + &tmp1); + tcm_dbg("TCM-dbg: burstcnt: 0x%x\n", burstcnt); + if (ret < 0) + return -EINVAL; + + burstcnt = tmp | (tmp1 << 8); + if (burstcnt) + return burstcnt; + msleep(TCM_TIMEOUT); + } while (time_before(jiffies, stop)); + + return -EBUSY; +} + +static int wait_for_stat(struct tcm_chip *chip, u8 mask, + unsigned long timeout, + wait_queue_head_t *queue) +{ + unsigned long stop; + u8 status; + + /* check current status */ + status = tcm_tis_status(chip); + if ((status & mask) == mask) + return 0; + + stop = jiffies + timeout; + do { + msleep(TCM_TIMEOUT); + status = tcm_tis_status(chip); + if ((status & mask) == mask) + return 0; + } while (time_before(jiffies, stop)); + + return -ETIME; +} + +static int recv_data(struct tcm_chip *chip, u8 *buf, size_t count) +{ + int ret; + int size = 0, burstcnt; + + while (size < count && wait_for_stat(chip, + TCM_STS_DATA_AVAIL | TCM_STS_VALID, + chip->vendor.timeout_c, + &chip->vendor.read_queue) == 0) { + burstcnt = get_burstcount(chip); + + if (burstcnt < 0) { + dev_err(chip->dev, "Unable to read burstcount\n"); + return burstcnt; + } + + for (; burstcnt > 0 && size < count; burstcnt--) { + ret = tcm_tis_readb(chip->dev, + TCM_DATA_FIFO(chip->vendor.locality), + &buf[size]); + tcm_dbg("TCM-dbg: buf[%d]: 0x%x\n", size, buf[size]); + size++; + } + } + + return size; +} + +static int tcm_tis_recv(struct tcm_chip *chip, u8 *buf, size_t count) +{ + int size = 0; + int expected, status; + unsigned long stop; + + if (count < TCM_HEADER_SIZE) { + dev_err(chip->dev, "read size is to small: %d\n", (u32)(count)); + size = -EIO; + goto out; + } + + /* read first 10 bytes, including tag, paramsize, and result */ + size = recv_data(chip, buf, TCM_HEADER_SIZE); + if (size < TCM_HEADER_SIZE) { + dev_err(chip->dev, "Unable to read header\n"); + goto out; + } + + expected = be32_to_cpu(*(__be32 *)(buf + 2)); + if (expected > count) { + dev_err(chip->dev, "Expected data count\n"); + size = -EIO; + goto out; + } + + size += recv_data(chip, &buf[TCM_HEADER_SIZE], + expected - TCM_HEADER_SIZE); + if (size < expected) { + dev_err(chip->dev, "Unable to read remainder of result\n"); + size = -ETIME; + goto out; + } + + wait_for_stat(chip, TCM_STS_VALID, chip->vendor.timeout_c, + &chip->vendor.int_queue); + + stop = jiffies + chip->vendor.timeout_c; + do { + msleep(TCM_TIMEOUT); + status = tcm_tis_status(chip); + if ((status & TCM_STS_DATA_AVAIL) == 0) + break; + + } while (time_before(jiffies, stop)); + + status = tcm_tis_status(chip); + if (status & TCM_STS_DATA_AVAIL) { /* retry? */ + dev_err(chip->dev, "Error left over data\n"); + size = -EIO; + goto out; + } + +out: + tcm_tis_ready(chip); + release_locality(chip, chip->vendor.locality, 0); + if (size < 0) + tcm_handle_err(chip); + return size; +} + +/* + * If interrupts are used (signaled by an irq set in the vendor structure) + * tcm.c can skip polling for the data to be available as the interrupt is + * waited for here + */ +static int tcm_tis_send(struct tcm_chip *chip, u8 *buf, size_t len) +{ + int rc, status, burstcnt; + size_t count = 0; + u32 ordinal; + unsigned long stop; + int send_again = 0; + +tcm_tis_send_again: + count = 0; + if (request_locality(chip, 0) < 0) { + dev_err(chip->dev, "send, tcm is busy\n"); + return -EBUSY; + } + status = tcm_tis_status(chip); + + if ((status & TCM_STS_COMMAND_READY) == 0) { + tcm_tis_ready(chip); + if (wait_for_stat(chip, TCM_STS_COMMAND_READY, + chip->vendor.timeout_b, &chip->vendor.int_queue) < 0) { + dev_err(chip->dev, "send, tcm wait time out1\n"); + rc = -ETIME; + goto out_err; + } + } + + while (count < len - 1) { + burstcnt = get_burstcount(chip); + if (burstcnt < 0) { + dev_err(chip->dev, "Unable to read burstcount\n"); + rc = burstcnt; + goto out_err; + } + for (; burstcnt > 0 && count < len - 1; burstcnt--) { + tcm_tis_writeb(chip->dev, + TCM_DATA_FIFO(chip->vendor.locality), buf[count]); + count++; + } + + wait_for_stat(chip, TCM_STS_VALID, chip->vendor.timeout_c, + &chip->vendor.int_queue); + } + + /* write last byte */ + tcm_tis_writeb(chip->dev, + TCM_DATA_FIFO(chip->vendor.locality), buf[count]); + + wait_for_stat(chip, TCM_STS_VALID, + chip->vendor.timeout_c, &chip->vendor.int_queue); + stop = jiffies + chip->vendor.timeout_c; + do { + msleep(TCM_TIMEOUT); + status = tcm_tis_status(chip); + if ((status & TCM_STS_DATA_EXPECT) == 0) + break; + + } while (time_before(jiffies, stop)); + + if ((status & TCM_STS_DATA_EXPECT) != 0) { + dev_err(chip->dev, "send, tcm expect data\n"); + rc = -EIO; + goto out_err; + } + + /* go and do it */ + tcm_tis_writeb(chip->dev, TCM_STS(chip->vendor.locality), TCM_STS_GO); + + ordinal = be32_to_cpu(*((__be32 *)(buf + 6))); + if (wait_for_stat(chip, TCM_STS_DATA_AVAIL | TCM_STS_VALID, + tcm_calc_ordinal_duration(chip, ordinal), + &chip->vendor.read_queue) < 0) { + dev_err(chip->dev, "send, tcm wait time out2\n"); + rc = -ETIME; + goto out_err; + } + + return len; + +out_err: + tcm_tis_ready(chip); + release_locality(chip, chip->vendor.locality, 0); + tcm_handle_err(chip); + if (send_again++ < 3) { + goto tcm_tis_send_again; + } + + dev_err(chip->dev, "kylin send, err: %d\n", rc); + return rc; +} + +static struct file_operations tis_ops = { + .owner = THIS_MODULE, + .llseek = no_llseek, + .open = tcm_open, + .read = tcm_read, + .write = tcm_write, + .release = tcm_release, +}; + +static DEVICE_ATTR(pubek, S_IRUGO, tcm_show_pubek, NULL); +static DEVICE_ATTR(pcrs, S_IRUGO, tcm_show_pcrs, NULL); +static DEVICE_ATTR(enabled, S_IRUGO, tcm_show_enabled, NULL); +static DEVICE_ATTR(active, S_IRUGO, tcm_show_active, NULL); +static DEVICE_ATTR(owned, S_IRUGO, tcm_show_owned, NULL); +static DEVICE_ATTR(temp_deactivated, S_IRUGO, tcm_show_temp_deactivated, + NULL); +static DEVICE_ATTR(caps, S_IRUGO, tcm_show_caps, NULL); +static DEVICE_ATTR(cancel, S_IWUSR | S_IWGRP, NULL, tcm_store_cancel); + +static struct attribute *tis_attrs[] = { + &dev_attr_pubek.attr, + &dev_attr_pcrs.attr, + &dev_attr_enabled.attr, + &dev_attr_active.attr, + &dev_attr_owned.attr, + &dev_attr_temp_deactivated.attr, + &dev_attr_caps.attr, + &dev_attr_cancel.attr, NULL, +}; + +static struct attribute_group tis_attr_grp = { + .attrs = tis_attrs +}; + +static struct tcm_vendor_specific tcm_tis = { + .status = tcm_tis_status, + .recv = tcm_tis_recv, + .send = tcm_tis_send, + .cancel = tcm_tis_ready, + .req_complete_mask = TCM_STS_DATA_AVAIL | TCM_STS_VALID, + .req_complete_val = TCM_STS_DATA_AVAIL | TCM_STS_VALID, + .req_canceled = TCM_STS_COMMAND_READY, + .attr_group = &tis_attr_grp, + .miscdev = { + .fops = &tis_ops, + }, +}; + +static struct tcm_chip *chip; +static int tcm_tis_spi_probe(struct spi_device *spi) +{ + int ret; + u8 revid; + u32 vendor, intfcaps; + struct tcm_tis_spi_phy *phy; + struct chip_data *spi_chip; + + pr_info("TCM(ky): __func__(v=%d) ..\n", + 10); + + tcm_dbg("TCM-dbg: %s/%d, enter\n", __func__, __LINE__); + phy = devm_kzalloc(&spi->dev, sizeof(struct tcm_tis_spi_phy), + GFP_KERNEL); + if (!phy) + return -ENOMEM; + + phy->iobuf = devm_kmalloc(&spi->dev, MAX_SPI_FRAMESIZE, GFP_KERNEL); + if (!phy->iobuf) + return -ENOMEM; + + phy->spi_device = spi; + init_completion(&phy->ready); + + tcm_dbg("TCM-dbg: %s/%d\n", __func__, __LINE__); + /* init spi dev */ + spi->chip_select = 0; /* cs0 */ + spi->mode = SPI_MODE_0; + spi->bits_per_word = 8; + spi->max_speed_hz = spi->max_speed_hz ? : 24000000; + spi_setup(spi); + + spi_chip = spi_get_ctldata(spi); + if (!spi_chip) { + pr_err("There was wrong in spi master\n"); + return -ENODEV; + } + /* tcm does not support interrupt mode, we use poll mode instead. */ + spi_chip->poll_mode = 1; + + tcm_dbg("TCM-dbg: %s/%d\n", __func__, __LINE__); + /* regiter tcm hw */ + chip = tcm_register_hardware(&spi->dev, &tcm_tis); + if (!chip) { + dev_err(chip->dev, "tcm register hardware err\n"); + return -ENODEV; + } + + dev_set_drvdata(chip->dev, phy); + + /** + * phytium2000a4 spi controller's clk clk level is unstable, + * so it is solved by using the low level of gpio output. + **/ + if (is_ft_all() && (spi->chip_select == 0)) { + /* reuse conf reg base */ + reuse_conf_reg = ioremap(REUSE_CONF_REG_BASE, 0x10); + if (!reuse_conf_reg) { + dev_err(&spi->dev, "Failed to ioremap reuse conf reg\n"); + ret = -ENOMEM; + goto out_err; + } + + /* gpio1 a5 base addr */ + gpio1_a5 = ioremap(REUSE_GPIO1_A5_BASE, 0x10); + if (!gpio1_a5) { + dev_err(&spi->dev, "Failed to ioremap gpio1 a5\n"); + ret = -ENOMEM; + goto out_err; + } + + /* reuse cs0 to gpio1_a5 */ + iowrite32((ioread32(reuse_conf_reg) | 0xFFFF0) & 0xFFF9004F, + reuse_conf_reg); + /* set gpio1 a5 to output */ + iowrite32(0x20, gpio1_a5 + 0x4); + } + + tcm_dbg("TCM-dbg: %s/%d\n", + __func__, __LINE__); + ret = tcm_tis_readl(chip->dev, TCM_DID_VID(0), &vendor); + if (ret < 0) + goto out_err; + + tcm_dbg("TCM-dbg: %s/%d, vendor: 0x%x\n", + __func__, __LINE__, vendor); + if ((vendor & 0xffff) != 0x19f5 && (vendor & 0xffff) != 0x1B4E) { + dev_err(chip->dev, "there is no Nationz TCM on you computer\n"); + goto out_err; + } + + ret = tcm_tis_readb(chip->dev, TCM_RID(0), &revid); + tcm_dbg("TCM-dbg: %s/%d, revid: 0x%x\n", + __func__, __LINE__, revid); + if (ret < 0) + goto out_err; + dev_info(chip->dev, "kylin: 2019-09-21 1.2 TCM " + "(device-id 0x%X, rev-id %d)\n", + vendor >> 16, revid); + + /* Default timeouts */ + chip->vendor.timeout_a = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + chip->vendor.timeout_b = msecs_to_jiffies(TIS_LONG_TIMEOUT); + chip->vendor.timeout_c = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + chip->vendor.timeout_d = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + + tcm_dbg("TCM-dbg: %s/%d\n", + __func__, __LINE__); + /* Figure out the capabilities */ + ret = tcm_tis_readl(chip->dev, + TCM_INTF_CAPS(chip->vendor.locality), &intfcaps); + if (ret < 0) + goto out_err; + + tcm_dbg("TCM-dbg: %s/%d, intfcaps: 0x%x\n", + __func__, __LINE__, intfcaps); + if (request_locality(chip, 0) != 0) { + dev_err(chip->dev, "tcm request_locality err\n"); + ret = -ENODEV; + goto out_err; + } + + INIT_LIST_HEAD(&chip->vendor.list); + spin_lock(&tis_lock); + list_add(&chip->vendor.list, &tis_chips); + spin_unlock(&tis_lock); + + tcm_get_timeouts(chip); + tcm_startup(chip); + + tcm_dbg("TCM-dbg: %s/%d, exit\n", __func__, __LINE__); + return 0; + +out_err: + if (is_ft_all()) { + if (reuse_conf_reg) + iounmap(reuse_conf_reg); + if (gpio1_a5) + iounmap(gpio1_a5); + } + tcm_dbg("TCM-dbg: %s/%d, error\n", __func__, __LINE__); + dev_set_drvdata(chip->dev, chip); + tcm_remove_hardware(chip->dev); + + return ret; +} + +static int tcm_tis_spi_remove(struct spi_device *dev) +{ + if (is_ft_all()) { + if (reuse_conf_reg) + iounmap(reuse_conf_reg); + if (gpio1_a5) + iounmap(gpio1_a5); + } + + dev_info(&dev->dev, "%s\n", __func__); + dev_set_drvdata(chip->dev, chip); + tcm_remove_hardware(&dev->dev); + + return 0; +} + +static const struct acpi_device_id tcm_tis_spi_acpi_match[] = { + {"TCMS0001", 0}, + {"SMO0768", 0}, + {"ZIC0601", 0}, + {} +}; +MODULE_DEVICE_TABLE(acpi, tcm_tis_spi_acpi_match); + +static const struct spi_device_id tcm_tis_spi_id_table[] = { + {"SMO0768", 0}, + {"ZIC0601", 0}, + {} +}; +MODULE_DEVICE_TABLE(spi, tcm_tis_spi_id_table); + +static struct spi_driver tcm_tis_spi_drv = { + .driver = { + .name = "tcm_tis_spi", + .acpi_match_table = ACPI_PTR(tcm_tis_spi_acpi_match), + }, + .id_table = tcm_tis_spi_id_table, + .probe = tcm_tis_spi_probe, + .remove = tcm_tis_spi_remove, +}; + +#if 1 +module_spi_driver(tcm_tis_spi_drv); +#else/*0*/ + +static int __init __spi_driver_init(void) +{ + pr_info("TCM(ky): __init __func__(ver=%2d)\n", + 10); + return spi_register_driver(&tcm_tis_spi_drv); +} + +static void __exit __spi_driver_exit(void) +{ + pr_info("TCM(ky): __exit __func__\n"); + spi_unregister_driver(&tcm_tis_spi_drv); +} + +module_init(__spi_driver_init); +module_exit(__spi_driver_exit); +#endif/*0*/ + +MODULE_AUTHOR("xiongxin<xiongxin(a)tj.kylinos.cn>"); +MODULE_DESCRIPTION("TCM Driver Base Spi"); +MODULE_VERSION("6.1.0.2"); +MODULE_LICENSE("GPL"); -- 2.23.0

1 1

[PATCH kernel-4.19 v1] drivers: TCM: add GMJS(Nationz Tech) TCM driver.
by 李志刚 23 Feb '21

23 Feb '21

1 0

[PATCH OLK-5.10 1/2] arm64: smp: Add support for cpu park
by sangyan＠huawei.com 23 Feb '21

23 Feb '21

From: Sang Yan <sangyan(a)huawei.com> hulk inclusion category: feature bugzilla: 48159 CVE: N/A Introducing a feature of CPU PARK in order to save time of cpus down and up during kexec, which may cost 250ms of per cpu's down and 30ms of up. As a result, for 128 cores, it costs more than 30 seconds to down and up cpus during kexec. Think about 256 cores and more. CPU PARK is a state that cpu power-on and staying in spin loop, polling for exit chances, such as writing exit address. Reserving a block of memory, to fill with cpu park text section, exit address and park-magic-flag of each cpu. In implementation, reserved one page for one cpu core. Cpus going to park state instead of down in machine_shutdown(). Cpus going out of park state in smp_init instead of brought up. One of cpu park sections in pre-reserved memory blocks,: +--------------+ + exit address + +--------------+ + park magic + +--------------+ + park codes + + . + + . + + . + +--------------+ Signed-off-by: Sang Yan <sangyan(a)huawei.com> --- arch/arm64/Kconfig | 12 ++ arch/arm64/include/asm/kexec.h | 6 + arch/arm64/include/asm/smp.h | 15 +++ arch/arm64/kernel/Makefile | 1 + arch/arm64/kernel/cpu-park.S | 59 ++++++++++ arch/arm64/kernel/machine_kexec.c | 2 +- arch/arm64/kernel/process.c | 4 + arch/arm64/kernel/smp.c | 229 ++++++++++++++++++++++++++++++++++++++ arch/arm64/mm/init.c | 55 +++++++++ 9 files changed, 382 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/kernel/cpu-park.S diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b9c5654..0885668 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -345,6 +345,18 @@ config KASAN_SHADOW_OFFSET default 0xeffffff900000000 if ARM64_VA_BITS_36 && KASAN_SW_TAGS default 0xffffffffffffffff +config ARM64_CPU_PARK + bool "Support CPU PARK on kexec" + depends on SMP + depends on KEXEC_CORE + help + This enables support for CPU PARK feature in + order to save time of cpu down to up. + CPU park is a state through kexec, spin loop + instead of cpu die before jumping to new kernel, + jumping out from loop to new kernel entry in + smp_init. + source "arch/arm64/Kconfig.platforms" menu "Kernel Features" diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h index 79909ae..a133889 100644 --- a/arch/arm64/include/asm/kexec.h +++ b/arch/arm64/include/asm/kexec.h @@ -36,6 +36,11 @@ #define CRASH_ADDR_HIGH_MAX MEMBLOCK_ALLOC_ACCESSIBLE +#ifdef CONFIG_ARM64_CPU_PARK +/* CPU park state flag: "park" */ +#define PARK_MAGIC 0x7061726b +#endif + #ifndef __ASSEMBLY__ /** @@ -104,6 +109,7 @@ static inline void crash_post_resume(void) {} #ifdef CONFIG_KEXEC_CORE extern void __init reserve_crashkernel(void); #endif +void machine_kexec_mask_interrupts(void); #ifdef CONFIG_KEXEC_FILE #define ARCH_HAS_KIMAGE_ARCH diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index 2e7f529..8c5d2d6 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -145,6 +145,21 @@ bool cpus_are_stuck_in_kernel(void); extern void crash_smp_send_stop(void); extern bool smp_crash_stop_failed(void); +#ifdef CONFIG_ARM64_CPU_PARK +#define PARK_SECTION_SIZE 1024 +struct cpu_park_info { + /* Physical address of reserved park memory. */ + unsigned long start; + /* park reserve mem len should be PARK_SECTION_SIZE * NR_CPUS */ + unsigned long len; + /* Virtual address of reserved park memory. */ + unsigned long start_v; +}; +extern struct cpu_park_info park_info; +extern void enter_cpu_park(unsigned long text, unsigned long exit); +extern void do_cpu_park(unsigned long exit); +extern int kexec_smp_send_park(void); +#endif #endif /* ifndef __ASSEMBLY__ */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 2621d5c..60478d2 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -54,6 +54,7 @@ obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o \ cpu-reset.o +obj-$(CONFIG_ARM64_CPU_PARK) += cpu-park.o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_ARM64_RELOC_TEST) += arm64-reloc-test.o arm64-reloc-test-y := reloc_test_core.o reloc_test_syms.o diff --git a/arch/arm64/kernel/cpu-park.S b/arch/arm64/kernel/cpu-park.S new file mode 100644 index 0000000..10c685c --- /dev/null +++ b/arch/arm64/kernel/cpu-park.S @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * CPU park routines + * + * Copyright (C) 2020 Huawei Technologies., Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> +#include <asm/kexec.h> +#include <asm/sysreg.h> +#include <asm/virt.h> + +.text +.pushsection .idmap.text, "awx" + +/* cpu park helper in idmap section */ +SYM_CODE_START(enter_cpu_park) + /* Clear sctlr_el1 flags. */ + mrs x12, sctlr_el1 + mov_q x13, SCTLR_ELx_FLAGS + bic x12, x12, x13 + pre_disable_mmu_workaround + msr sctlr_el1, x12 /* disable mmu */ + isb + + mov x18, x0 + mov x0, x1 /* secondary_entry addr */ + br x18 /* call do_cpu_park of each cpu */ +SYM_CODE_END(enter_cpu_park) + +.popsection + +SYM_CODE_START(do_cpu_park) + ldr x18, =PARK_MAGIC /* magic number "park" */ + add x1, x0, #8 + str x18, [x1] /* set on-park flag */ + dc civac, x1 /* flush cache of "park" */ + dsb nsh + isb + +.Lloop: + wfe + isb + ldr x19, [x0] + cmp x19, #0 /* test secondary_entry */ + b.eq .Lloop + + ic iallu /* invalidate the local I-cache */ + dsb nsh + isb + + br x19 /* jump to secondary_entry */ +SYM_CODE_END(do_cpu_park) + diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index a0b144c..f47ce96 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -213,7 +213,7 @@ void machine_kexec(struct kimage *kimage) BUG(); /* Should never get here. */ } -static void machine_kexec_mask_interrupts(void) +void machine_kexec_mask_interrupts(void) { unsigned int i; struct irq_desc *desc; diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 73e3b32..10cffee 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -146,6 +146,10 @@ void arch_cpu_idle_dead(void) */ void machine_shutdown(void) { +#ifdef CONFIG_ARM64_CPU_PARK + if (kexec_smp_send_park() == 0) + return; +#endif smp_shutdown_nonboot_cpus(reboot_cpu); } diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 18e9727..bc475d5 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -32,6 +32,8 @@ #include <linux/irq_work.h> #include <linux/kernel_stat.h> #include <linux/kexec.h> +#include <linux/console.h> + #include <linux/kvm_host.h> #include <asm/alternative.h> @@ -93,6 +95,167 @@ static inline int op_cpu_kill(unsigned int cpu) } #endif +#ifdef CONFIG_ARM64_CPU_PARK +struct cpu_park_section { + unsigned long exit; /* exit address of park look */ + unsigned long magic; /* maigc represent park state */ + char text[0]; /* text section of park */ +}; + +static int mmap_cpu_park_mem(void) +{ + if (!park_info.start) + return -ENOMEM; + + if (park_info.start_v) + return 0; + + park_info.start_v = (unsigned long)__ioremap(park_info.start, + park_info.len, + PAGE_KERNEL_EXEC); + if (!park_info.start_v) { + pr_warn("map park memory failed."); + return -ENOMEM; + } + + return 0; +} + +static inline unsigned long cpu_park_section_v(unsigned int cpu) +{ + return park_info.start_v + PARK_SECTION_SIZE * (cpu - 1); +} + +static inline unsigned long cpu_park_section_p(unsigned int cpu) +{ + return park_info.start + PARK_SECTION_SIZE * (cpu - 1); +} + +/* + * Write the secondary_entry to exit section of park state. + * Then the secondary cpu will jump straight into the kernel + * by the secondary_entry. + */ +static int write_park_exit(unsigned int cpu) +{ + struct cpu_park_section *park_section; + unsigned long *park_exit; + unsigned long *park_text; + + if (mmap_cpu_park_mem() != 0) + return -EPERM; + + park_section = (struct cpu_park_section *)cpu_park_section_v(cpu); + park_exit = &park_section->exit; + park_text = (unsigned long *)park_section->text; + pr_debug("park_text 0x%lx : 0x%lx, do_cpu_park text 0x%lx : 0x%lx", + (unsigned long)park_text, *park_text, + (unsigned long)do_cpu_park, + *(unsigned long *)do_cpu_park); + + /* + * Test first 8 bytes to determine + * whether needs to write cpu park exit. + */ + if (*park_text == *(unsigned long *)do_cpu_park) { + writeq_relaxed(__pa_symbol(secondary_entry), park_exit); + __flush_dcache_area((__force void *)park_exit, + sizeof(unsigned long)); + flush_icache_range((unsigned long)park_exit, + (unsigned long)(park_exit + 1)); + sev(); + dsb(sy); + isb(); + + pr_debug("Write cpu %u secondary entry 0x%lx to 0x%lx.", + cpu, *park_exit, (unsigned long)park_exit); + pr_info("Boot cpu %u from PARK state.", cpu); + return 0; + } + + return -EPERM; +} + +/* Install cpu park sections for the specific cpu. */ +static int install_cpu_park(unsigned int cpu) +{ + struct cpu_park_section *park_section; + unsigned long *park_exit; + unsigned long *park_magic; + unsigned long park_text_len; + + park_section = (struct cpu_park_section *)cpu_park_section_v(cpu); + pr_debug("Install cpu park on cpu %u park exit 0x%lx park text 0x%lx", + cpu, (unsigned long)park_section, + (unsigned long)(park_section->text)); + + park_exit = &park_section->exit; + park_magic = &park_section->magic; + park_text_len = PARK_SECTION_SIZE - sizeof(struct cpu_park_section); + + *park_exit = 0UL; + *park_magic = 0UL; + memcpy((void *)park_section->text, do_cpu_park, park_text_len); + __flush_dcache_area((void *)park_section, PARK_SECTION_SIZE); + + return 0; +} + +static int uninstall_cpu_park(unsigned int cpu) +{ + unsigned long park_section; + + if (mmap_cpu_park_mem() != 0) + return -EPERM; + + park_section = cpu_park_section_v(cpu); + memset((void *)park_section, 0, PARK_SECTION_SIZE); + __flush_dcache_area((void *)park_section, PARK_SECTION_SIZE); + + return 0; +} + +static int cpu_wait_park(unsigned int cpu) +{ + long timeout; + struct cpu_park_section *park_section; + + volatile unsigned long *park_magic; + + park_section = (struct cpu_park_section *)cpu_park_section_v(cpu); + park_magic = &park_section->magic; + + timeout = USEC_PER_SEC; + while (*park_magic != PARK_MAGIC && timeout--) + udelay(1); + + if (timeout > 0) + pr_debug("cpu %u park done.", cpu); + else + pr_err("cpu %u park failed.", cpu); + + return *park_magic == PARK_MAGIC; +} + +static void cpu_park(unsigned int cpu) +{ + unsigned long park_section_p; + unsigned long park_exit_phy; + unsigned long do_park; + typeof(enter_cpu_park) *park; + + park_section_p = cpu_park_section_p(cpu); + park_exit_phy = park_section_p; + pr_debug("Go to park cpu %u exit address 0x%lx", cpu, park_exit_phy); + + do_park = park_section_p + sizeof(struct cpu_park_section); + park = (void *)__pa_symbol(enter_cpu_park); + + cpu_install_idmap(); + park(do_park, park_exit_phy); + unreachable(); +} +#endif /* * Boot a secondary CPU, and assign it the specified idle task. @@ -102,6 +265,10 @@ static int boot_secondary(unsigned int cpu, struct task_struct *idle) { const struct cpu_operations *ops = get_cpu_ops(cpu); +#ifdef CONFIG_ARM64_CPU_PARK + if (write_park_exit(cpu) == 0) + return 0; +#endif if (ops->cpu_boot) return ops->cpu_boot(cpu); @@ -131,6 +298,9 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) return ret; } +#ifdef CONFIG_ARM64_CPU_PARK + uninstall_cpu_park(cpu); +#endif /* * CPU was successfully started, wait for it to come online or * time out. @@ -844,10 +1014,30 @@ void arch_irq_work_raise(void) static void local_cpu_stop(void) { + int cpu; + const struct cpu_operations *ops = NULL; + set_cpu_online(smp_processor_id(), false); local_daif_mask(); sdei_mask_local_cpu(); + +#ifdef CONFIG_ARM64_CPU_PARK + /* + * Go to cpu park state. + * Otherwise go to cpu die. + */ + cpu = smp_processor_id(); + if (kexec_in_progress && park_info.start_v) { + machine_kexec_mask_interrupts(); + cpu_park(cpu); + + ops = get_cpu_ops(cpu); + if (ops && ops->cpu_die) + ops->cpu_die(cpu); + } +#endif + cpu_park_loop(); } @@ -1053,6 +1243,45 @@ void smp_send_stop(void) sdei_mask_local_cpu(); } +#ifdef CONFIG_ARM64_CPU_PARK +int kexec_smp_send_park(void) +{ + unsigned long cpu; + + if (WARN_ON(!kexec_in_progress)) { + pr_crit("%s called not in kexec progress.", __func__); + return -EPERM; + } + + if (mmap_cpu_park_mem() != 0) { + pr_info("no cpuparkmem, goto normal way."); + return -EPERM; + } + + local_irq_disable(); + + if (num_online_cpus() > 1) { + cpumask_t mask; + + cpumask_copy(&mask, cpu_online_mask); + cpumask_clear_cpu(smp_processor_id(), &mask); + + for_each_cpu(cpu, &mask) + install_cpu_park(cpu); + smp_cross_call(&mask, IPI_CPU_STOP); + + /* Wait for other CPUs to park */ + for_each_cpu(cpu, &mask) + cpu_wait_park(cpu); + pr_info("smp park other cpus done\n"); + } + + sdei_mask_local_cpu(); + + return 0; +} +#endif + #ifdef CONFIG_KEXEC_CORE void crash_smp_send_stop(void) { diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 794f992..d01259c 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -236,6 +236,57 @@ static void __init fdt_enforce_memory_region(void) memblock_add(usable_rgns[1].base, usable_rgns[1].size); } +#ifdef CONFIG_ARM64_CPU_PARK +struct cpu_park_info park_info = { + .start = 0, + .len = PARK_SECTION_SIZE * NR_CPUS, + .start_v = 0, +}; + +static int __init parse_park_mem(char *p) +{ + if (!p) + return 0; + + park_info.start = PAGE_ALIGN(memparse(p, NULL)); + if (park_info.start == 0) + pr_info("cpu park mem params[%s]", p); + + return 0; +} +early_param("cpuparkmem", parse_park_mem); + +static int __init reserve_park_mem(void) +{ + if (park_info.start == 0 || park_info.len == 0) + return 0; + + park_info.start = PAGE_ALIGN(park_info.start); + park_info.len = PAGE_ALIGN(park_info.len); + + if (!memblock_is_region_memory(park_info.start, park_info.len)) { + pr_warn("cannot reserve park mem: region is not memory!"); + goto out; + } + + if (memblock_is_region_reserved(park_info.start, park_info.len)) { + pr_warn("cannot reserve park mem: region overlaps reserved memory!"); + goto out; + } + + memblock_remove(park_info.start, park_info.len); + pr_info("cpu park mem reserved: 0x%016lx - 0x%016lx (%ld MB)", + park_info.start, park_info.start + park_info.len, + park_info.len >> 20); + + return 0; +out: + park_info.start = 0; + park_info.len = 0; + return -EINVAL; +} +#endif + void __init arm64_memblock_init(void) { const s64 linear_region_size = BIT(vabits_actual - 1); @@ -357,6 +408,10 @@ void __init arm64_memblock_init(void) reserve_crashkernel(); +#ifdef CONFIG_ARM64_CPU_PARK + reserve_park_mem(); +#endif + reserve_elfcorehdr(); high_memory = __va(memblock_end_of_DRAM() - 1) + 1; -- 2.9.5

2 3

[PATCH OLK-5.10 1/3] kexec: Add quick kexec support for kernel
by sangyan＠huawei.com 22 Feb '21

22 Feb '21

From: Sang Yan <sangyan(a)huawei.com> hulk inclusion category: feature bugzilla: 48159 CVE: N/A In normal kexec, relocating kernel may cost 5 ~ 10 seconds, to copy all segments from vmalloced memory to kernel boot memory, because of disabled mmu. We introduce quick kexec to save time of copying memory as above, just like kdump(kexec on crash), by using reserved memory "Quick Kexec". Constructing quick kimage as the same as crash kernel, then simply copy all segments of kimage to reserved memroy. We also add this support in syscall kexec_load using flags of KEXEC_QUICK. Signed-off-by: Sang Yan <sangyan(a)huawei.com> --- arch/Kconfig | 10 ++++++++++ include/linux/ioport.h | 1 + include/linux/kexec.h | 11 ++++++++++- include/uapi/linux/kexec.h | 1 + kernel/kexec.c | 10 ++++++++++ kernel/kexec_core.c | 42 +++++++++++++++++++++++++++++++++--------- 6 files changed, 65 insertions(+), 10 deletions(-) diff --git a/arch/Kconfig b/arch/Kconfig index 2592b4b..7811eee 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -18,6 +18,16 @@ config KEXEC_CORE select CRASH_CORE bool +config QUICK_KEXEC + bool "Support for quick kexec" + depends on KEXEC_CORE + help + It uses pre-reserved memory to accelerate kexec, just like + crash kexec, loads new kernel and initrd to reserved memory, + and boots new kernel on that memory. It will save the time + of relocating kernel. + + config KEXEC_ELF bool diff --git a/include/linux/ioport.h b/include/linux/ioport.h index 5135d4b..84a716f 100644 --- a/include/linux/ioport.h +++ b/include/linux/ioport.h @@ -139,6 +139,7 @@ enum { IORES_DESC_DEVICE_PRIVATE_MEMORY = 6, IORES_DESC_RESERVED = 7, IORES_DESC_SOFT_RESERVED = 8, + IORES_DESC_QUICK_KEXEC = 9, }; /* diff --git a/include/linux/kexec.h b/include/linux/kexec.h index f301f2f..7fff410 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -269,9 +269,10 @@ struct kimage { unsigned long control_page; /* Flags to indicate special processing */ - unsigned int type : 1; + unsigned int type : 2; #define KEXEC_TYPE_DEFAULT 0 #define KEXEC_TYPE_CRASH 1 +#define KEXEC_TYPE_QUICK 2 unsigned int preserve_context : 1; /* If set, we are using file mode kexec syscall */ unsigned int file_mode:1; @@ -331,6 +332,11 @@ extern int kexec_load_disabled; #define KEXEC_FLAGS (KEXEC_ON_CRASH | KEXEC_PRESERVE_CONTEXT) #endif +#ifdef CONFIG_QUICK_KEXEC +#undef KEXEC_FLAGS +#define KEXEC_FLAGS (KEXEC_ON_CRASH | KEXEC_QUICK) +#endif + /* List of defined/legal kexec file flags */ #define KEXEC_FILE_FLAGS (KEXEC_FILE_UNLOAD | KEXEC_FILE_ON_CRASH | \ KEXEC_FILE_NO_INITRAMFS) @@ -338,6 +344,9 @@ extern int kexec_load_disabled; /* Location of a reserved region to hold the crash kernel. */ extern note_buf_t __percpu *crash_notes; +#ifdef CONFIG_QUICK_KEXEC +extern struct resource quick_kexec_res; +#endif /* flag to track if kexec reboot is in progress */ extern bool kexec_in_progress; diff --git a/include/uapi/linux/kexec.h b/include/uapi/linux/kexec.h index 05669c8..d891d80 100644 --- a/include/uapi/linux/kexec.h +++ b/include/uapi/linux/kexec.h @@ -12,6 +12,7 @@ /* kexec flags for different usage scenarios */ #define KEXEC_ON_CRASH 0x00000001 #define KEXEC_PRESERVE_CONTEXT 0x00000002 +#define KEXEC_QUICK 0x00000004 #define KEXEC_ARCH_MASK 0xffff0000 /* diff --git a/kernel/kexec.c b/kernel/kexec.c index c82c6c0..4acc909 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -44,6 +44,9 @@ static int kimage_alloc_init(struct kimage **rimage, unsigned long entry, int ret; struct kimage *image; bool kexec_on_panic = flags & KEXEC_ON_CRASH; +#ifdef CONFIG_QUICK_KEXEC + bool kexec_on_quick = flags & KEXEC_QUICK; +#endif if (kexec_on_panic) { /* Verify we have a valid entry point */ @@ -69,6 +72,13 @@ static int kimage_alloc_init(struct kimage **rimage, unsigned long entry, image->type = KEXEC_TYPE_CRASH; } +#ifdef CONFIG_QUICK_KEXEC + if (kexec_on_quick) { + image->control_page = quick_kexec_res.start; + image->type = KEXEC_TYPE_QUICK; + } +#endif + ret = sanity_check_segment_list(image); if (ret) goto out_free_image; diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 2ca8875..c7e2aa2 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -53,6 +53,17 @@ note_buf_t __percpu *crash_notes; /* Flag to indicate we are going to kexec a new kernel */ bool kexec_in_progress = false; +/* Resource for quick kexec */ +#ifdef CONFIG_QUICK_KEXEC +struct resource quick_kexec_res = { + .name = "Quick kexec", + .start = 0, + .end = 0, + .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM, + .desc = IORES_DESC_QUICK_KEXEC +}; +#endif + int kexec_should_crash(struct task_struct *p) { /* @@ -396,8 +407,9 @@ static struct page *kimage_alloc_normal_control_pages(struct kimage *image, return pages; } -static struct page *kimage_alloc_crash_control_pages(struct kimage *image, - unsigned int order) +static struct page *kimage_alloc_special_control_pages(struct kimage *image, + unsigned int order, + unsigned long end) { /* Control pages are special, they are the intermediaries * that are needed while we copy the rest of the pages @@ -427,7 +439,7 @@ static struct page *kimage_alloc_crash_control_pages(struct kimage *image, size = (1 << order) << PAGE_SHIFT; hole_start = (image->control_page + (size - 1)) & ~(size - 1); hole_end = hole_start + size - 1; - while (hole_end <= crashk_res.end) { + while (hole_end <= end) { unsigned long i; cond_resched(); @@ -462,7 +474,6 @@ static struct page *kimage_alloc_crash_control_pages(struct kimage *image, return pages; } - struct page *kimage_alloc_control_pages(struct kimage *image, unsigned int order) { @@ -473,8 +484,15 @@ struct page *kimage_alloc_control_pages(struct kimage *image, pages = kimage_alloc_normal_control_pages(image, order); break; case KEXEC_TYPE_CRASH: - pages = kimage_alloc_crash_control_pages(image, order); + pages = kimage_alloc_special_control_pages(image, order, + crashk_res.end); + break; +#ifdef CONFIG_QUICK_KEXEC + case KEXEC_TYPE_QUICK: + pages = kimage_alloc_special_control_pages(image, order, + quick_kexec_res.end); break; +#endif } return pages; @@ -830,11 +848,12 @@ static int kimage_load_normal_segment(struct kimage *image, return result; } -static int kimage_load_crash_segment(struct kimage *image, +static int kimage_load_special_segment(struct kimage *image, struct kexec_segment *segment) { - /* For crash dumps kernels we simply copy the data from - * user space to it's destination. + /* + * For crash dumps kernels and quick kexec kernels + * we simply copy the data from user space to it's destination. * We do things a page at a time for the sake of kmap. */ unsigned long maddr; @@ -908,8 +927,13 @@ int kimage_load_segment(struct kimage *image, result = kimage_load_normal_segment(image, segment); break; case KEXEC_TYPE_CRASH: - result = kimage_load_crash_segment(image, segment); + result = kimage_load_special_segment(image, segment); break; +#ifdef CONFIG_QUICK_KEXEC + case KEXEC_TYPE_QUICK: + result = kimage_load_special_segment(image, segment); + break; +#endif } return result; -- 2.9.5

2 3

[PATCH openEuler-1.0-LTS 000/316] bugfix for 20.03
by Cheng Jian 22 Feb '21

22 Feb '21

bugfix for 20.03 @ 2021/02/22 Aichun Li (7): netpoll: remove dev argument from netpoll_send_skb_on_dev() netpoll: move netpoll_send_skb() out of line netpoll: netpoll_send_skb() returns transmit status netpoll: accept NULL np argument in netpoll_send_skb() bonding: add an option to specify a delay between peer notifications bonding: fix value exported by Netlink for peer_notif_delay bonding: add documentation for peer_notif_delay Akilesh Kailash (1): dm snapshot: flush merged data before committing metadata Al Viro (2): don't dump the threads that had been already exiting when zapped. dump_common_audit_data(): fix racy accesses to ->d_name Aleksandr Nogikh (1): netem: fix zero division in tabledist Alexander Duyck (1): tcp: Set INET_ECN_xmit configuration in tcp_reinit_congestion_control Alexander Lobakin (1): skbuff: back tiny skbs with kmalloc() in __netdev_alloc_skb() too Alexey Dobriyan (2): proc: change ->nlink under proc_subdir_lock proc: fix lookup in /proc/net subdirectories after setns(2) Alexey Kardashevskiy (1): serial_core: Check for port state when tty is in error state Amit Cohen (1): mlxsw: core: Fix use-after-free in mlxsw_emad_trans_finish() Andy Shevchenko (2): device property: Keep secondary firmware node secondary by type device property: Don't clear secondary pointer for shared primary firmware node Antoine Tenart (6): netfilter: bridge: reset skb->pkt_type after NF_INET_POST_ROUTING traversal net: ip6_gre: set dev->hard_header_len when using header_ops net-sysfs: take the rtnl lock when storing xps_cpus net-sysfs: take the rtnl lock when accessing xps_cpus_map and num_tc net-sysfs: take the rtnl lock when storing xps_rxqs net-sysfs: take the rtnl lock when accessing xps_rxqs_map and num_tc Ard Biesheuvel (3): efivarfs: revert "fix memory leak in efivarfs_create()" arm64, mm, efi: Account for GICv3 LPI tables in static memblock reserve table efi/arm: Revert "Defer persistent reservations until after paging_init()" Arjun Roy (1): tcp: Prevent low rmem stalls with SO_RCVLOWAT. Arnaldo Carvalho de Melo (1): perf scripting python: Avoid declaring function pointers with a visibility attribute Axel Lin (1): ASoC: msm8916-wcd-digital: Select REGMAP_MMIO to fix build error Aya Levin (1): net: ipv6: Validate GSO SKB before finish IPv6 processing Bart Van Assche (1): scsi: scsi_transport_spi: Set RQF_PM for domain validation commands Bharat Gooty (1): PCI: iproc: Fix out-of-bound array accesses Bixuan Cui (2): mmap: fix a compiling error for 'MAP_CHECKNODE' powerpc: fix a compiling error for 'access_ok' Bjorn Helgaas (1): PCI: Bounds-check command-line resource alignment requests Björn Töpel (1): ixgbe: avoid premature Rx buffer reuse Boqun Feng (1): fcntl: Fix potential deadlock in send_sig{io, urg}() Boris Protopopov (1): Convert trailing spaces and periods in path components Brian Foster (1): xfs: flush new eof page on truncate to avoid post-eof corruption Calum Mackay (1): lockd: don't use interval-based rebinding over TCP Chen Zhou (1): selinux: Fix error return code in sel_ib_pkey_sid_slow() Chenguangli (1): scsi/hifc:Fix the bug that the system may be oops during unintall hifc module. Cheng Lin (1): nfs_common: need lock during iterate through the list Christoph Hellwig (2): nbd: fix a block_device refcount leak in nbd_release xfs: fix a missing unlock on error in xfs_fs_map_blocks Chunguang Xu (1): ext4: fix a memory leak of ext4_free_data Chunyan Zhang (1): tick/common: Touch watchdog in tick_unfreeze() on all CPUs Colin Ian King (1): PCI: Fix overflow in command-line resource alignment requests Cong Wang (1): erspan: fix version 1 check in gre_parse_header() Damien Le Moal (1): null_blk: Fix zone size initialization Dan Carpenter (1): futex: Don't enable IRQs unconditionally in put_pi_state() Daniel Scally (1): Revert "ACPI / resources: Use AE_CTRL_TERMINATE to terminate resources walks" Darrick J. Wong (12): xfs: fix realtime bitmap/summary file truncation when growing rt volume xfs: don't free rt blocks when we're doing a REMAP bunmapi call xfs: set xefi_discard when creating a deferred agfl free log intent item xfs: fix scrub flagging rtinherit even if there is no rt device xfs: fix flags argument to rmap lookup when converting shared file rmaps xfs: set the unwritten bit in rmap lookup flags in xchk_bmap_get_rmapextents xfs: fix rmap key and record comparison functions xfs: fix brainos in the refcount scrubber's rmap fragment processor vfs: remove lockdep bogosity in __sb_start_write xfs: fix the minrecs logic when dealing with inode root child blocks xfs: strengthen rmap record flags checking xfs: revert "xfs: fix rmap key and record comparison functions" Dave Wysochanski (1): NFS4: Fix use-after-free in trace_event_raw_event_nfs4_set_lock Dexuan Cui (1): ACPI: scan: Harden acpi_device_add() against device ID overflows Dinghao Liu (4): ext4: fix error handling code in add_new_gdb net/mlx5e: Fix memleak in mlx5e_create_l2_table_groups net/mlx5e: Fix two double free cases netfilter: nf_nat: Fix memleak in nf_nat_init Dongdong Wang (1): lwt: Disable BH too in run_lwt_bpf() Dongli Zhang (1): page_frag: Recover from memory pressure Douglas Gilbert (1): sgl_alloc_order: fix memory leak Eddy Wu (1): fork: fix copy_process(CLONE_PARENT) race with the exiting ->real_parent Eran Ben Elisha (1): net/mlx5: Fix wrong address reclaim when command interface is down Eric Auger (1): vfio/pci: Move dummy_resources_list init in vfio_pci_probe() Eric Biggers (1): ext4: fix leaking sysfs kobject after failed mount Eric Dumazet (4): tcp: select sane initial rcvq_space.space for big MSS net: avoid 32 x truesize under-estimation for tiny skbs net_sched: avoid shift-out-of-bounds in tcindex_set_parms() net_sched: reject silly cell_log in qdisc_get_rtab() Fang Lijun (3): arm64/ascend: mm: Add MAP_CHECKNODE flag to check node hugetlb arm64/ascend: mm: Fix arm32 compile warnings arm64/ascend: mm: Fix hugetlb check node error Fangrui Song (1): arm64: Change .weak to SYM_FUNC_START_WEAK_PI for arch/arm64/lib/mem*.S Florian Fainelli (1): net: Have netpoll bring-up DSA management interface Florian Westphal (4): netfilter: nf_tables: avoid false-postive lockdep splat netfilter: xt_RATEEST: reject non-null terminated string from userspace net: ip: always refragment ip defragmented packets net: fix pmtu check in nopmtudisc mode Gabriel Krisman Bertazi (2): blk-cgroup: Fix memleak on error path blk-cgroup: Pre-allocate tree node on blkg_conf_prep George Spelvin (1): random32: make prandom_u32() output unpredictable Gerald Schaefer (1): mm/userfaultfd: do not access vma->vm_mm after calling handle_userfault() Guillaume Nault (4): ipv4: Fix tos mask in inet_rtm_getroute() ipv4: Ignore ECN bits for fib lookups in fib_compute_spec_dst() netfilter: rpfilter: mask ecn bits before fib lookup udp: mask TOS bits in udp_v4_early_demux() Hanjun Guo (1): clocksource/drivers/arch_timer: Fix vdso_fix compile error for arm32 Hannes Reinecke (1): dm: avoid filesystem lookup in dm_get_dev_t() Hans de Goede (1): ACPI: scan: Make acpi_bus_get_device() clear return pointer on error Heiner Kallweit (1): net: bridge: add missing counters to ndo_get_stats64 callback Hoang Le (1): tipc: fix NULL deref in tipc_link_xmit() Huang Shijie (1): lib/genalloc: fix the overflow when size is too big Huang Ying (1): mm: fix a race during THP splitting Hugh Dickins (2): mlock: fix unevictable_pgs event counts on THP mm: fix check_move_unevictable_pages() on THP Hui Wang (1): ACPI: PNP: compare the string length in the matching_id() Hyeongseok Kim (1): dm verity: skip verity work if I/O error when system is shutting down Ido Schimmel (2): mlxsw: core: Fix memory leak on module removal mlxsw: core: Use variable timeout for EMAD retries Ilya Dryomov (1): libceph: clear con->out_msg on Policy::stateful_server faults Jakub Kicinski (1): net: vlan: avoid leaks on register_vlan_dev() failures Jamie Iles (1): bonding: wait for sysfs kobject destruction before freeing struct slave Jan Kara (8): ext4: Detect already used quota file early ext4: fix bogus warning in ext4_update_dx_flag() ext4: Protect superblock modifications with a buffer lock ext4: fix deadlock with fs freezing and EA inodes ext4: don't remount read-only with errors=continue on reboot quota: Don't overflow quota file offsets bfq: Fix computation of shallow depth ext4: fix superblock checksum failure when setting password salt Jann Horn (1): mm, slub: consider rest of partial list if acquire_slab() fails Jason A. Donenfeld (3): netfilter: use actual socket sk rather than skb sk when routing harder net: introduce skb_list_walk_safe for skb segment walking net: skbuff: disambiguate argument and member for skb_list_walk_safe helper Jeff Dike (1): virtio_net: Fix recursive call to cpus_read_lock() Jens Axboe (1): proc: don't allow async path resolution of /proc/self components Jesper Dangaard Brouer (1): netfilter: conntrack: fix reading nf_conntrack_buckets Jessica Yu (1): module: delay kobject uevent until after module init call Jiri Olsa (2): perf python scripting: Fix printable strings in python3 scripts perf tools: Add missing swap for ino_generation Johannes Thumshirn (1): block: factor out requeue handling from dispatch code Jonathan Cameron (1): ACPI: Add out of bounds and numa_off protections to pxm_to_node() Joseph Qi (1): ext4: unlock xattr_sem properly in ext4_inline_data_truncate() Jubin Zhong (1): PCI: Fix pci_slot_release() NULL pointer dereference Kaixu Xia (1): ext4: correctly report "not supported" for {usr, grp}jquota when !CONFIG_QUOTA Keqian Zhu (1): clocksource/drivers/arm_arch_timer: Correct fault programming of CNTKCTL_EL1.EVNTI Kirill Tkhai (1): mm: move nr_deactivate accounting to shrink_active_list() Lang Dai (1): uio: free uio id after uio file node is freed Lecopzer Chen (2): kasan: fix unaligned address is unhandled in kasan_remove_zero_shadow kasan: fix incorrect arguments passing in kasan_add_zero_shadow Lee Duncan (1): scsi: libiscsi: Fix NOP race condition Lee Jones (1): Fonts: Replace discarded const qualifier Leo Yan (1): perf lock: Don't free "lock_seq_stat" if read_count isn't zero Leon Romanovsky (1): net/mlx5: Properly convey driver version to firmware Lijie (1): config: enable CONFIG_NVME_MULTIPATH by default Liu Shixin (3): config: set default value of CONFIG_TEST_FREE_PAGES mm: memcontrol: add struct mem_cgroup_extension mm: fix kabi broken Lorenzo Pieralisi (1): asm-generic/io.h: Fix !CONFIG_GENERIC_IOMAP pci_iounmap() implementation Lu Jialin (1): fs: fix files.usage bug when move tasks Luc Van Oostenryck (1): xsk: Fix xsk_poll()'s return type Luo Meng (2): ext4: fix invalid inode checksum fail_function: Remove a redundant mutex unlock Mao Wenan (1): net: Update window_clamp if SOCK_RCVBUF is set Marc Zyngier (2): arm64: Run ARCH_WORKAROUND_1 enabling code on all CPUs genirq/irqdomain: Don't try to free an interrupt that has no mapping Mark Rutland (3): arm64: syscall: exit userspace before unmasking exceptions arm64: module: rework special section handling arm64: module/ftrace: intialize PLT at load time Martin Wilck (1): scsi: core: Fix VPD LUN ID designator priorities Mateusz Nosek (1): futex: Fix incorrect should_fail_futex() handling Matteo Croce (4): Revert "kernel/reboot.c: convert simple_strtoul to kstrtoint" reboot: fix overflow parsing reboot cpu number ipv6: create multicast route with RTPROT_KERNEL ipv6: set multicast flag on the multicast route Matthew Wilcox (Oracle) (1): mm/page_alloc.c: fix freeing non-compound pages Maurizio Lombardi (2): scsi: target: remove boilerplate code scsi: target: fix hang when multiple threads try to destroy the same iscsi session Maxim Mikityanskiy (1): net/tls: Protect from calling tls_dev_del for TLS RX twice Miaohe Lin (1): mm/hugetlb: fix potential missing huge page size info Michael Schaller (1): efivarfs: Replace invalid slashes with exclamation marks in dentries. Mike Christie (1): scsi: target: iscsi: Fix cmd abort fabric stop race Mike Galbraith (1): futex: Handle transient "ownerless" rtmutex state correctly Miklos Szeredi (1): fuse: fix page dereference after free Mikulas Patocka (3): dm integrity: fix the maximum number of arguments dm integrity: fix flush with external metadata device dm integrity: fix a crash if "recalculate" used without "internal_hash" Ming Lei (2): scsi: core: Don't start concurrent async scan on same host block: fix use-after-free in disk_part_iter_next Minwoo Im (1): nvme: free sq/cq dbbuf pointers when dbbuf set fails Miroslav Benes (1): module: set MODULE_STATE_GOING state when a module fails to load Moshe Shemesh (2): net/mlx4_en: Avoid scheduling restart task if it is already running net/mlx4_en: Handle TX error CQE Naoya Horiguchi (1): mm, hwpoison: double-check page count in __get_any_page() Naveen N. Rao (1): ftrace: Fix updating FTRACE_FL_TRAMP Neal Cardwell (1): tcp: fix cwnd-limited bug for TSO deferral where we send nothing NeilBrown (1): NFS: switch nfsiod to be an UNBOUND workqueue. Nicholas Piggin (1): mm: fix exec activate_mm vs TLB shootdown and lazy tlb switching race Oleg Nesterov (1): ptrace: fix task_join_group_stop() for the case when current is traced Oliver Herms (1): IPv6: Set SIT tunnel hard_header_len to zero Paul Moore (1): selinux: fix inode_doinit_with_dentry() LABEL_INVALID error handling Paulo Alcantara (1): cifs: fix potential use-after-free in cifs_echo_request() Peng Liu (1): sched/deadline: Fix sched_dl_global_validate() Peter Zijlstra (2): serial: pl011: Fix lockdep splat when handling magic-sysrq interrupt perf: Fix get_recursion_context() Petr Malat (1): sctp: Fix COMM_LOST/CANT_STR_ASSOC err reporting on big-endian platforms Qian Cai (1): mm/swapfile: do not sleep with a spin lock held Qiujun Huang (2): ring-buffer: Return 0 on success from ring_buffer_resize() tracing: Fix out of bounds write in get_trace_buf Rafael J. Wysocki (1): driver core: Extend device_is_dependent() Randy Dunlap (1): net: sched: prevent invalid Scell_log shift count Ritika Srivastava (2): block: Return blk_status_t instead of errno codes block: better deal with the delayed not supported case in blk_cloned_rq_check_limits Ronnie Sahlberg (1): cifs: handle -EINTR in cifs_setattr Ryan Sharpelletti (1): tcp: only postpone PROBE_RTT if RTT is < current min_rtt estimate Sami Tolvanen (1): arm64: lse: fix LSE atomics with LLVM's integrated assembler Sean Tranchetti (1): net: ipv6: fib: flush exceptions when purging route Shakeel Butt (2): mm: swap: fix vmstats for huge pages mm: swap: memcg: fix memcg stats for huge pages Shijie Luo (1): mm: mempolicy: fix potential pte_unmap_unlock pte error Shin'ichiro Kawasaki (1): uio: Fix use-after-free in uio_unregister_device() Stefano Brivio (1): netfilter: ipset: Update byte and packet counters regardless of whether they match Steven Rostedt (VMware) (4): ring-buffer: Fix recursion protection transitions between interrupt context ftrace: Fix recursion check for NMI test ftrace: Handle tracing when switching between context tracing: Fix userstacktrace option for instances Subash Abhinov Kasiviswanathan (2): netfilter: x_tables: Switch synchronization to RCU netfilter: x_tables: Update remaining dereference to RCU Sven Eckelmann (2): vxlan: Add needed_headroom for lower device vxlan: Copy needed_tailroom from lowerdev Sylwester Dziedziuch (2): i40e: Fix removing driver while bare-metal VFs pass traffic i40e: Fix Error I40E_AQ_RC_EINVAL when removing VFs Takashi Iwai (1): libata: transport: Use scnprintf() for avoiding potential buffer overflow Tariq Toukan (1): net: Disable NETIF_F_HW_TLS_RX when RXCSUM is disabled Thomas Gleixner (12): sched: Reenable interrupts in do_sched_yield() futex: Move futex exit handling into futex code futex: Replace PF_EXITPIDONE with a state exit/exec: Seperate mm_release() futex: Split futex_mm_release() for exit/exec futex: Set task::futex_state to DEAD right after handling futex exit futex: Mark the begin of futex exit explicitly futex: Sanitize exit state handling futex: Provide state handling for exec() as well futex: Add mutex around futex exit futex: Provide distinct return value when owner is exiting futex: Prevent exit livelock Tianyue Ren (1): selinux: fix error initialization in inode_doinit_with_dentry() Trond Myklebust (5): SUNRPC: xprt_load_transport() needs to support the netid "rdma6" NFSv4: Fix a pNFS layout related use-after-free race when freeing the inode pNFS: Mark layout for return if return-on-close was not sent NFS/pNFS: Fix a leak of the layout 'plh_outstanding' counter NFS: nfs_igrab_and_active must first reference the superblock Tung Nguyen (1): tipc: fix memory leak caused by tipc_buf_append() Tyler Hicks (1): tpm: efi: Don't create binary_bios_measurements file for an empty log Uwe Kleine-König (1): spi: fix resource leak for drivers without .remove callback Vadim Fedorenko (1): net/tls: missing received data after fast remote close Valentin Schneider (1): arm64: topology: Stop using MPIDR for topology information Vamshi K Sthambamkadi (1): efivarfs: fix memory leak in efivarfs_create() Vasily Averin (2): netfilter: ipset: fix shift-out-of-bounds in htable_bits() net: drop bogus skb with CHECKSUM_PARTIAL and offset beyond end of trimmed packet Vincenzo Frascino (1): arm64: lse: Fix LSE atomics with LLVM Vladyslav Tarasiuk (1): net/mlx5: Disable QoS when min_rates on all VFs are zero Wang Hai (4): devlink: Add missing genlmsg_cancel() in devlink_nl_sb_port_pool_fill() inet_diag: Fix error path to cancel the meseage in inet_req_diag_fill() tipc: fix memory leak in tipc_topsrv_start() ipv6: addrlabel: fix possible memory leak in ip6addrlbl_net_init Wang Wensheng (1): sbsa_gwdt: Add WDIOF_PRETIMEOUT flag to watchdog_info at defination Wei Li (1): irqchip/gic-v3: Fix compiling error on ARM32 with GICv3 Wei Yang (1): mm: thp: don't need care deferred split queue in memcg charge move path Weilong Chen (1): hugetlbfs: Add dependency with ascend memory features Wengang Wang (1): ocfs2: initialize ip_next_orphan Will Deacon (3): arm64: psci: Avoid printing in cpu_psci_cpu_die() arm64: pgtable: Fix pte_accessible() arm64: pgtable: Ensure dirty bit is preserved across pte_wrprotect() Willem de Bruijn (1): sock: set sk_err to ee_errno on dequeue from errq Wu Bo (1): scsi: libiscsi: fix task hung when iscsid deamon exited Xie XiuQi (1): cputime: fix undefined reference to get_idle_time when CONFIG_PROC_FS disabled Xin Long (1): sctp: change to hold/put transport for proto_unreach_timer Xiongfeng Wang (1): arm64: fix compile error when CONFIG_HOTPLUG_CPU is disabled Xiubo Li (1): nbd: make the config put is called before the notifying the waiter Xu Qiang (4): NMI: Enable arm-pmu interrupt as NMI in Acensed. irqchip/gic-v3-its: Unconditionally save/restore the ITS state on suspend. irqchip/irq-gic-v3: Add workaround bindings in device tree to init ts core GICR. Document: In the binding document, add enable-init-all-GICR field description. Yang Shi (6): mm: list_lru: set shrinker map bit when child nr_items is not zero mm: thp: extract split_queue_* into a struct mm: move mem_cgroup_uncharge out of __page_cache_release() mm: shrinker: make shrinker not depend on memcg kmem mm: thp: make deferred split shrinker memcg aware mm: vmscan: protect shrinker idr replace with CONFIG_MEMCG Yang Yingliang (5): arm64: arch_timer: only do cntvct workaround on VDSO path on D05 armv7 fix compile error Kconfig: disable KTASK by default futex: sched: fix kabi broken in task_struct futex: sched: fix UAF when free futex_exit_mutex in free_task() Yi-Hung Wei (1): ip_tunnels: Set tunnel option flag when tunnel metadata is present Yicong Yang (1): libfs: fix error cast of negative value in simple_attr_write() Yu Kuai (2): blk-cgroup: prevent rcu_sched detected stalls warnings in blkg_destroy_all() blk-throttle: don't check whether or not lower limit is valid if CONFIG_BLK_DEV_THROTTLING_LOW is off Yufen Yu (2): bdi: fix compiler error in bdi_get_dev_name() scsi: do quiesce for enclosure driver Yunfeng Ye (1): workqueue: Kick a worker based on the actual activation of delayed works Yunjian Wang (2): net: hns: fix return value check in __lb_other_process() vhost_net: fix ubuf refcount incorrectly when sendmsg fails Yunsheng Lin (1): net: sch_generic: fix the missing new qdisc assignment bug Zeng Tao (1): time: Prevent undefined behaviour in timespec64_to_ns() Zhang Changzhong (2): ah6: fix error return code in ah6_input() net: bridge: vlan: fix error return code in __vlan_add() Zhao Heming (2): md/cluster: block reshape with remote resync job md/cluster: fix deadlock when node is doing resync job Zhengyuan Liu (3): arm64/mm: return cpu_all_mask when node is NUMA_NO_NODE hifc: remove unnecessary __init specifier mmap: fix a compiling error for 'MAP_PA32BIT' Zhou Guanghui (2): memcg/ascend: Check sysctl oom config for memcg oom memcg/ascend: enable kmem cgroup by default for ascend Zqiang (1): kthread_worker: prevent queuing delayed work from timer_fn when it is being canceled chenmaodong (1): fix virtio_gpu use-after-free while creating dumb j.nixdorf(a)avm.de (1): net: sunrpc: interpret the return value of kstrtou32 correctly lijinlin (1): ext4: add ext3 report error to userspace by netlink miaoyubo (1): KVM: Enable PUD huge mappings only on 1620 yangerkun (1): ext4: fix bug for rename with RENAME_WHITEOUT zhuoliang zhang (1): net: xfrm: fix a race condition during allocing spi .../interrupt-controller/arm,gic-v3.txt | 4 + Documentation/networking/bonding.txt | 16 +- arch/Kconfig | 7 + arch/alpha/include/uapi/asm/mman.h | 2 + arch/arm/kernel/perf_event_v7.c | 16 +- arch/arm64/Kconfig | 2 + arch/arm64/configs/euleros_defconfig | 1 + arch/arm64/configs/hulk_defconfig | 1 + arch/arm64/configs/openeuler_defconfig | 3 +- arch/arm64/include/asm/atomic_lse.h | 76 ++- arch/arm64/include/asm/cpufeature.h | 4 + arch/arm64/include/asm/lse.h | 6 +- arch/arm64/include/asm/memory.h | 11 + arch/arm64/include/asm/numa.h | 3 + arch/arm64/include/asm/pgtable.h | 34 +- arch/arm64/kernel/cpu_errata.c | 8 + arch/arm64/kernel/cpufeature.c | 2 +- arch/arm64/kernel/ftrace.c | 50 +- arch/arm64/kernel/module.c | 47 +- arch/arm64/kernel/psci.c | 5 +- arch/arm64/kernel/setup.c | 5 +- arch/arm64/kernel/syscall.c | 2 +- arch/arm64/kernel/topology.c | 43 +- arch/arm64/lib/memcpy.S | 3 +- arch/arm64/lib/memmove.S | 3 +- arch/arm64/lib/memset.S | 3 +- arch/arm64/mm/init.c | 17 +- arch/arm64/mm/numa.c | 6 +- arch/mips/include/uapi/asm/mman.h | 2 + arch/parisc/include/uapi/asm/mman.h | 2 + arch/powerpc/include/asm/uaccess.h | 4 +- arch/powerpc/include/uapi/asm/mman.h | 2 + arch/sparc/include/uapi/asm/mman.h | 2 + arch/x86/configs/hulk_defconfig | 1 + arch/x86/configs/openeuler_defconfig | 1 + arch/xtensa/include/uapi/asm/mman.h | 2 + block/bfq-iosched.c | 8 +- block/blk-cgroup.c | 30 +- block/blk-core.c | 39 +- block/blk-mq.c | 29 +- block/blk-throttle.c | 6 + block/genhd.c | 9 +- drivers/acpi/acpi_pnp.c | 3 + drivers/acpi/internal.h | 2 +- drivers/acpi/numa.c | 2 +- drivers/acpi/resource.c | 2 +- drivers/acpi/scan.c | 17 +- drivers/ata/libata-transport.c | 10 +- drivers/base/core.c | 21 +- drivers/block/nbd.c | 3 +- drivers/block/null_blk_zoned.c | 20 +- drivers/char/Kconfig | 2 +- drivers/char/random.c | 1 - drivers/char/tpm/eventlog/efi.c | 5 + drivers/clocksource/arm_arch_timer.c | 36 +- drivers/firmware/efi/efi.c | 4 - drivers/firmware/efi/libstub/arm-stub.c | 3 - drivers/gpu/drm/virtio/virtgpu_gem.c | 4 +- drivers/irqchip/irq-gic-v3-its.c | 30 +- drivers/irqchip/irq-gic-v3.c | 18 +- drivers/md/dm-bufio.c | 6 + drivers/md/dm-integrity.c | 58 ++- drivers/md/dm-snap.c | 24 + drivers/md/dm-table.c | 15 +- drivers/md/dm-verity-target.c | 12 +- drivers/md/md-cluster.c | 67 +-- drivers/md/md.c | 14 +- drivers/mtd/hisilicon/sfc/hrd_sfc_driver.c | 4 +- drivers/net/bonding/bond_main.c | 92 ++-- drivers/net/bonding/bond_netlink.c | 14 + drivers/net/bonding/bond_options.c | 71 ++- drivers/net/bonding/bond_procfs.c | 2 + drivers/net/bonding/bond_sysfs.c | 13 + drivers/net/bonding/bond_sysfs_slave.c | 18 +- .../net/ethernet/hisilicon/hns/hns_ethtool.c | 4 + drivers/net/ethernet/intel/i40e/i40e.h | 4 + drivers/net/ethernet/intel/i40e/i40e_main.c | 32 +- .../ethernet/intel/i40e/i40e_virtchnl_pf.c | 30 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 24 +- .../net/ethernet/mellanox/mlx4/en_netdev.c | 21 +- drivers/net/ethernet/mellanox/mlx4/en_tx.c | 40 +- drivers/net/ethernet/mellanox/mlx4/mlx4_en.h | 12 +- .../net/ethernet/mellanox/mlx5/core/en_fs.c | 3 + .../net/ethernet/mellanox/mlx5/core/eswitch.c | 15 +- .../net/ethernet/mellanox/mlx5/core/main.c | 6 +- .../ethernet/mellanox/mlx5/core/pagealloc.c | 21 +- drivers/net/ethernet/mellanox/mlxsw/core.c | 8 +- drivers/net/geneve.c | 3 +- drivers/net/macvlan.c | 5 +- drivers/net/virtio_net.c | 12 +- drivers/net/vxlan.c | 3 + drivers/nvme/host/pci.c | 15 + drivers/pci/controller/pcie-iproc.c | 10 +- drivers/pci/pci.c | 14 +- drivers/pci/slot.c | 6 +- drivers/scsi/huawei/hifc/unf_common.h | 2 +- drivers/scsi/huawei/hifc/unf_scsi.c | 23 + drivers/scsi/libiscsi.c | 33 +- drivers/scsi/scsi_lib.c | 126 +++-- drivers/scsi/scsi_scan.c | 11 +- drivers/scsi/scsi_transport_spi.c | 27 +- drivers/spi/spi.c | 19 +- drivers/target/iscsi/iscsi_target.c | 96 ++-- drivers/target/iscsi/iscsi_target.h | 1 - drivers/target/iscsi/iscsi_target_configfs.c | 5 +- drivers/target/iscsi/iscsi_target_login.c | 5 +- drivers/tty/serial/amba-pl011.c | 11 +- drivers/tty/serial/serial_core.c | 4 + drivers/uio/uio.c | 12 +- drivers/vfio/pci/vfio_pci.c | 3 +- drivers/vhost/net.c | 6 +- drivers/watchdog/sbsa_gwdt.c | 6 +- fs/cifs/cifs_unicode.c | 8 +- fs/cifs/connect.c | 2 + fs/cifs/inode.c | 13 +- fs/efivarfs/inode.c | 2 + fs/efivarfs/super.c | 3 + fs/exec.c | 17 +- fs/ext4/ext4.h | 6 +- fs/ext4/ext4_jbd2.c | 1 - fs/ext4/file.c | 1 + fs/ext4/inline.c | 1 + fs/ext4/inode.c | 31 +- fs/ext4/ioctl.c | 3 + fs/ext4/mballoc.c | 1 + fs/ext4/namei.c | 23 +- fs/ext4/resize.c | 8 +- fs/ext4/super.c | 32 +- fs/ext4/xattr.c | 1 + fs/fcntl.c | 10 +- fs/filescontrol.c | 73 +-- fs/fuse/dev.c | 28 +- fs/hugetlbfs/inode.c | 2 +- fs/libfs.c | 6 +- fs/lockd/host.c | 20 +- fs/nfs/inode.c | 2 +- fs/nfs/internal.h | 12 +- fs/nfs/nfs4proc.c | 2 +- fs/nfs/nfs4super.c | 2 +- fs/nfs/pnfs.c | 40 +- fs/nfs/pnfs.h | 5 + fs/nfs_common/grace.c | 6 +- fs/ocfs2/super.c | 1 + fs/proc/generic.c | 55 ++- fs/proc/internal.h | 7 + fs/proc/proc_net.c | 16 - fs/proc/self.c | 7 + fs/quota/quota_tree.c | 8 +- fs/super.c | 33 +- fs/xfs/libxfs/xfs_alloc.c | 1 + fs/xfs/libxfs/xfs_bmap.c | 19 +- fs/xfs/libxfs/xfs_bmap.h | 2 +- fs/xfs/libxfs/xfs_rmap.c | 2 +- fs/xfs/scrub/bmap.c | 10 +- fs/xfs/scrub/btree.c | 45 +- fs/xfs/scrub/inode.c | 3 +- fs/xfs/scrub/refcount.c | 8 +- fs/xfs/xfs_iops.c | 10 + fs/xfs/xfs_pnfs.c | 2 +- fs/xfs/xfs_rtalloc.c | 10 +- include/asm-generic/io.h | 39 +- include/linux/backing-dev.h | 1 + include/linux/blkdev.h | 1 + include/linux/compat.h | 2 - include/linux/dm-bufio.h | 1 + include/linux/efi.h | 7 - include/linux/futex.h | 39 +- include/linux/huge_mm.h | 9 + include/linux/hugetlb.h | 10 +- include/linux/if_team.h | 5 +- include/linux/memblock.h | 3 - include/linux/memcontrol.h | 32 +- include/linux/mm.h | 2 + include/linux/mm_types.h | 1 + include/linux/mman.h | 15 + include/linux/mmzone.h | 8 + include/linux/netfilter/x_tables.h | 5 +- include/linux/netfilter_ipv4.h | 2 +- include/linux/netfilter_ipv6.h | 2 +- include/linux/netpoll.h | 10 +- include/linux/prandom.h | 36 +- include/linux/proc_fs.h | 8 +- include/linux/sched.h | 5 +- include/linux/sched/cputime.h | 5 + include/linux/sched/mm.h | 6 +- include/linux/shrinker.h | 7 +- include/linux/skbuff.h | 5 + include/linux/sunrpc/xprt.h | 1 + include/linux/time64.h | 4 + include/net/bond_options.h | 1 + include/net/bonding.h | 14 +- include/net/ip_tunnels.h | 7 +- include/net/red.h | 4 +- include/net/tls.h | 6 + include/scsi/libiscsi.h | 3 + include/target/iscsi/iscsi_target_core.h | 2 +- include/uapi/asm-generic/mman.h | 1 + include/uapi/linux/if_link.h | 1 + init/Kconfig | 2 +- kernel/events/internal.h | 2 +- kernel/exit.c | 35 +- kernel/fail_function.c | 5 +- kernel/fork.c | 61 ++- kernel/futex.c | 291 +++++++++-- kernel/irq/irqdomain.c | 11 +- kernel/kthread.c | 3 +- kernel/module.c | 6 +- kernel/reboot.c | 28 +- kernel/sched/core.c | 6 +- kernel/sched/cputime.c | 6 + kernel/sched/deadline.c | 5 +- kernel/sched/sched.h | 42 +- kernel/signal.c | 19 +- kernel/time/itimer.c | 4 - kernel/time/tick-common.c | 2 + kernel/time/timer.c | 7 - kernel/trace/ftrace.c | 22 +- kernel/trace/ring_buffer.c | 66 ++- kernel/trace/trace.c | 9 +- kernel/trace/trace.h | 32 +- kernel/trace/trace_selftest.c | 9 +- kernel/workqueue.c | 13 +- lib/Kconfig.debug | 9 + lib/Makefile | 1 + lib/fonts/font_10x18.c | 2 +- lib/fonts/font_6x10.c | 2 +- lib/fonts/font_6x11.c | 2 +- lib/fonts/font_7x14.c | 2 +- lib/fonts/font_8x16.c | 2 +- lib/fonts/font_8x8.c | 2 +- lib/fonts/font_acorn_8x8.c | 2 +- lib/fonts/font_mini_4x6.c | 2 +- lib/fonts/font_pearl_8x8.c | 2 +- lib/fonts/font_sun12x22.c | 2 +- lib/fonts/font_sun8x16.c | 2 +- lib/genalloc.c | 25 +- lib/random32.c | 462 +++++++++++------- lib/scatterlist.c | 2 +- lib/test_free_pages.c | 42 ++ mm/huge_memory.c | 167 +++++-- mm/hugetlb.c | 5 +- mm/kasan/kasan_init.c | 23 +- mm/list_lru.c | 10 +- mm/memblock.c | 11 +- mm/memcontrol.c | 34 +- mm/memory-failure.c | 6 + mm/mempolicy.c | 6 +- mm/mlock.c | 25 +- mm/mmap.c | 27 +- mm/page_alloc.c | 9 + mm/slub.c | 2 +- mm/swap.c | 37 +- mm/swapfile.c | 4 +- mm/vmscan.c | 64 +-- net/8021q/vlan.c | 3 +- net/8021q/vlan_dev.c | 5 +- net/bridge/br_device.c | 1 + net/bridge/br_netfilter_hooks.c | 7 +- net/bridge/br_private.h | 5 +- net/bridge/br_vlan.c | 4 +- net/ceph/messenger.c | 5 + net/core/dev.c | 5 + net/core/devlink.c | 6 +- net/core/lwt_bpf.c | 8 +- net/core/net-sysfs.c | 65 ++- net/core/netpoll.c | 51 +- net/core/skbuff.c | 23 +- net/dsa/slave.c | 5 +- net/ipv4/fib_frontend.c | 2 +- net/ipv4/gre_demux.c | 2 +- net/ipv4/inet_diag.c | 4 +- net/ipv4/ip_output.c | 2 +- net/ipv4/ip_tunnel.c | 10 +- net/ipv4/netfilter.c | 12 +- net/ipv4/netfilter/arp_tables.c | 16 +- net/ipv4/netfilter/ip_tables.c | 16 +- net/ipv4/netfilter/ipt_SYNPROXY.c | 2 +- net/ipv4/netfilter/ipt_rpfilter.c | 2 +- net/ipv4/netfilter/iptable_mangle.c | 2 +- net/ipv4/netfilter/nf_nat_l3proto_ipv4.c | 2 +- net/ipv4/netfilter/nf_reject_ipv4.c | 2 +- net/ipv4/netfilter/nft_chain_route_ipv4.c | 2 +- net/ipv4/route.c | 7 +- net/ipv4/syncookies.c | 9 +- net/ipv4/tcp.c | 2 + net/ipv4/tcp_bbr.c | 2 +- net/ipv4/tcp_cong.c | 5 + net/ipv4/tcp_input.c | 6 +- net/ipv4/tcp_output.c | 9 +- net/ipv4/udp.c | 3 +- net/ipv6/addrconf.c | 3 +- net/ipv6/addrlabel.c | 26 +- net/ipv6/ah6.c | 3 +- net/ipv6/ip6_fib.c | 5 +- net/ipv6/ip6_gre.c | 16 +- net/ipv6/ip6_output.c | 40 +- net/ipv6/netfilter.c | 6 +- net/ipv6/netfilter/ip6_tables.c | 16 +- net/ipv6/netfilter/ip6table_mangle.c | 2 +- net/ipv6/netfilter/nf_nat_l3proto_ipv6.c | 2 +- net/ipv6/netfilter/nft_chain_route_ipv6.c | 2 +- net/ipv6/sit.c | 2 - net/ipv6/syncookies.c | 10 +- net/netfilter/ipset/ip_set_core.c | 3 +- net/netfilter/ipset/ip_set_hash_gen.h | 20 +- net/netfilter/ipvs/ip_vs_core.c | 4 +- net/netfilter/nf_conntrack_standalone.c | 3 + net/netfilter/nf_nat_core.c | 1 + net/netfilter/nf_tables_api.c | 3 +- net/netfilter/x_tables.c | 49 +- net/netfilter/xt_RATEEST.c | 3 + net/sched/cls_tcindex.c | 8 +- net/sched/sch_api.c | 3 +- net/sched/sch_choke.c | 2 +- net/sched/sch_generic.c | 3 + net/sched/sch_gred.c | 2 +- net/sched/sch_netem.c | 9 +- net/sched/sch_red.c | 2 +- net/sched/sch_sfq.c | 2 +- net/sctp/input.c | 4 +- net/sctp/sm_sideeffect.c | 8 +- net/sctp/transport.c | 2 +- net/sunrpc/addr.c | 2 +- net/sunrpc/xprt.c | 65 ++- net/sunrpc/xprtrdma/module.c | 1 + net/sunrpc/xprtrdma/transport.c | 1 + net/sunrpc/xprtsock.c | 4 + net/tipc/link.c | 9 +- net/tipc/msg.c | 5 +- net/tipc/topsrv.c | 10 +- net/tls/tls_device.c | 5 +- net/tls/tls_sw.c | 6 + net/xdp/xsk.c | 8 +- net/xfrm/xfrm_state.c | 8 +- security/lsm_audit.c | 7 +- security/selinux/hooks.c | 16 +- security/selinux/ibpkey.c | 4 +- sound/soc/codecs/Kconfig | 1 + tools/include/uapi/linux/if_link.h | 1 + tools/perf/builtin-lock.c | 2 +- tools/perf/util/print_binary.c | 2 +- .../scripting-engines/trace-event-python.c | 7 +- tools/perf/util/session.c | 1 + virt/kvm/arm/mmu.c | 7 + 344 files changed, 3424 insertions(+), 1701 deletions(-) create mode 100644 lib/test_free_pages.c -- 2.25.1

1 316