[PATCH openEuler-21.03 1/2] mm: add pin memory method for checkpoint add restore
by hejingxian＠huawei.com 02 Mar '21
by hejingxian＠huawei.com 02 Mar '21
02 Mar '21
From: Jingxian He <hejingxian(a)huawei.com>
hulk inclusion
category: feature
bugzilla: 48159
CVE: N/A
We can use the checkpoint and restore in userspace(criu) method to
dump and restore tasks when updating the kernel.
Currently, criu needs dump all memory data of tasks to files.
When the memory size is very large(larger than 1G),
the cost time of the dumping data will be very long(more than 1 min).
By pin the memory data of tasks and collect the corresponding
physical pages mapping info in checkpoint process,
we can remap the physical pages to restore tasks after
upgrading the kernel. This pin memory method can
restore the task data within one second.
The pin memory area info is saved in the reserved memblock,
which can keep usable in the kernel update process.
The pin memory driver provides the following ioctl command for criu:
1) SET_PIN_MEM_AREA:
Set pin memory area, which can be remap to the restore task.
2) CLEAR_PIN_MEM_AREA:
Clear the pin memory area info,
which enable user reset the pin data.
3) REMAP_PIN_MEM_AREA:
Remap the pages of the pin memory to the restore task.
Signed-off-by: Jingxian He <hejingxian(a)huawei.com>
Reviewed-by: Wenliang He <hewenliang4(a)huawei.com>
Reviewed-by: Jing Xiangfeng <jingxiangfeng(a)huawei.com>
---
arch/arm64/configs/openeuler_defconfig | 2 +
arch/arm64/kernel/setup.c | 9 +
arch/arm64/mm/init.c | 60 +++
drivers/char/Kconfig | 6 +
drivers/char/Makefile | 1 +
drivers/char/pin_memory.c | 208 ++++++++
include/linux/crash_core.h | 5 +
include/linux/pin_mem.h | 78 +++
kernel/crash_core.c | 11 +
mm/Kconfig | 8 +
mm/Makefile | 1 +
mm/huge_memory.c | 61 +++
mm/memory.c | 59 ++
mm/pin_mem.c | 950 +++++++++++++++++++++++++++++++++
14 files changed, 1459 insertions(+)
create mode 100644 drivers/char/pin_memory.c
create mode 100644 include/linux/pin_mem.h
create mode 100644 mm/pin_mem.c
diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig
index c5271e7..76fda68 100644
--- a/arch/arm64/configs/openeuler_defconfig
+++ b/arch/arm64/configs/openeuler_defconfig
@@ -1036,6 +1036,7 @@ CONFIG_FRAME_VECTOR=y
# CONFIG_GUP_BENCHMARK is not set
# CONFIG_READ_ONLY_THP_FOR_FS is not set
CONFIG_ARCH_HAS_PTE_SPECIAL=y
+CONFIG_PIN_MEMORY=y
# end of Memory Management options
CONFIG_NET=y
@@ -3282,6 +3283,7 @@ CONFIG_TCG_TIS_ST33ZP24_SPI=y
# CONFIG_RANDOM_TRUST_CPU is not set
# CONFIG_RANDOM_TRUST_BOOTLOADER is not set
+CONFIG_PIN_MEMORY_DEV=m
#
# I2C support
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index c1f1fb9..5e282d3 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -50,6 +50,9 @@
#include <asm/efi.h>
#include <asm/xen/hypervisor.h>
#include <asm/mmu_context.h>
+#ifdef CONFIG_PIN_MEMORY
+#include <linux/pin_mem.h>
+#endif
static int num_standard_resources;
static struct resource *standard_resources;
@@ -260,6 +263,12 @@ static void __init request_standard_resources(void)
quick_kexec_res.end <= res->end)
request_resource(res, &quick_kexec_res);
#endif
+#ifdef CONFIG_PIN_MEMORY
+ if (pin_memory_resource.end &&
+ pin_memory_resource.start >= res->start &&
+ pin_memory_resource.end <= res->end)
+ request_resource(res, &pin_memory_resource);
+#endif
}
}
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index f3e5a66..8ab5aac 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -42,6 +42,9 @@
#include <linux/sizes.h>
#include <asm/tlb.h>
#include <asm/alternative.h>
+#ifdef CONFIG_PIN_MEMORY
+#include <linux/pin_mem.h>
+#endif
#define ARM64_ZONE_DMA_BITS 30
@@ -78,6 +81,55 @@ static void __init reserve_crashkernel(void)
*/
#define MAX_USABLE_RANGES 2
+#ifdef CONFIG_PIN_MEMORY
+struct resource pin_memory_resource = {
+ .name = "Pin memory",
+ .start = 0,
+ .end = 0,
+ .flags = IORESOURCE_MEM,
+ .desc = IORES_DESC_RESERVED
+};
+
+static void __init reserve_pin_memory_res(void)
+{
+ unsigned long long mem_start, mem_len;
+ int ret;
+
+ ret = parse_pin_memory(boot_command_line, memblock_phys_mem_size(),
+ &mem_len, &mem_start);
+ if (ret || !mem_len)
+ return;
+
+ mem_len = PAGE_ALIGN(mem_len);
+
+ if (!memblock_is_region_memory(mem_start, mem_len)) {
+ pr_warn("cannot reserve for pin memory: region is not memory!\n");
+ return;
+ }
+
+ if (memblock_is_region_reserved(mem_start, mem_len)) {
+ pr_warn("cannot reserve for pin memory: region overlaps reserved memory!\n");
+ return;
+ }
+
+ if (!IS_ALIGNED(mem_start, SZ_2M)) {
+ pr_warn("cannot reserve for pin memory: base address is not 2MB aligned\n");
+ return;
+ }
+
+ memblock_reserve(mem_start, mem_len);
+ pr_debug("pin memory resource reserved: 0x%016llx - 0x%016llx (%lld MB)\n",
+ mem_start, mem_start + mem_len, mem_len >> 20);
+
+ pin_memory_resource.start = mem_start;
+ pin_memory_resource.end = mem_start + mem_len - 1;
+}
+#else
+static void __init reserve_pin_memory_res(void)
+{
+}
+#endif /* CONFIG_PIN_MEMORY */
+
#ifdef CONFIG_CRASH_DUMP
static int __init early_init_dt_scan_elfcorehdr(unsigned long node,
const char *uname, int depth, void *data)
@@ -455,6 +507,8 @@ void __init arm64_memblock_init(void)
reserve_park_mem();
#endif
+ reserve_pin_memory_res();
+
reserve_elfcorehdr();
high_memory = __va(memblock_end_of_DRAM() - 1) + 1;
@@ -583,6 +637,12 @@ void __init mem_init(void)
/* this will put all unused low memory onto the freelists */
memblock_free_all();
+#ifdef CONFIG_PIN_MEMORY
+ /* pre alloc the pages for pin memory */
+ init_reserve_page_map((unsigned long)pin_memory_resource.start,
+ (unsigned long)(pin_memory_resource.end - pin_memory_resource.start + 1));
+#endif
+
mem_init_print_info(NULL);
/*
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
index d229a2d..fbb94b8 100644
--- a/drivers/char/Kconfig
+++ b/drivers/char/Kconfig
@@ -496,3 +496,9 @@ config RANDOM_TRUST_BOOTLOADER
booloader is trustworthy so it will be added to the kernel's entropy
pool. Otherwise, say N here so it will be regarded as device input that
only mixes the entropy pool.
+
+config PIN_MEMORY_DEV
+ bool "/dev/pinmem character device"
+ default m
+ help
+ pin memory driver
diff --git a/drivers/char/Makefile b/drivers/char/Makefile
index ffce287..71d76fd 100644
--- a/drivers/char/Makefile
+++ b/drivers/char/Makefile
@@ -47,3 +47,4 @@ obj-$(CONFIG_PS3_FLASH) += ps3flash.o
obj-$(CONFIG_XILLYBUS) += xillybus/
obj-$(CONFIG_POWERNV_OP_PANEL) += powernv-op-panel.o
obj-$(CONFIG_ADI) += adi.o
+obj-$(CONFIG_PIN_MEMORY_DEV) += pin_memory.o
diff --git a/drivers/char/pin_memory.c b/drivers/char/pin_memory.c
new file mode 100644
index 0000000..f46e056
--- /dev/null
+++ b/drivers/char/pin_memory.c
@@ -0,0 +1,208 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright @ Huawei Technologies Co., Ltd. 2020-2020. ALL rights reserved.
+ * Description: Euler pin memory driver
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/kprobes.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/init.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/mm_types.h>
+#include <linux/processor.h>
+#include <uapi/asm-generic/ioctl.h>
+#include <uapi/asm-generic/mman-common.h>
+#include <uapi/asm/setup.h>
+#include <linux/pin_mem.h>
+#include <linux/sched/mm.h>
+
+#define MAX_PIN_MEM_AREA_NUM 16
+struct _pin_mem_area {
+ unsigned long virt_start;
+ unsigned long virt_end;
+};
+
+struct pin_mem_area_set {
+ unsigned int pid;
+ unsigned int area_num;
+ struct _pin_mem_area mem_area[MAX_PIN_MEM_AREA_NUM];
+};
+
+#define PIN_MEM_MAGIC 0x59
+#define _SET_PIN_MEM_AREA 1
+#define _CLEAR_PIN_MEM_AREA 2
+#define _REMAP_PIN_MEM_AREA 3
+#define _FINISH_PIN_MEM_DUMP 4
+#define _PIN_MEM_IOC_MAX_NR 4
+#define SET_PIN_MEM_AREA _IOW(PIN_MEM_MAGIC, _SET_PIN_MEM_AREA, struct pin_mem_area_set)
+#define CLEAR_PIN_MEM_AREA _IOW(PIN_MEM_MAGIC, _CLEAR_PIN_MEM_AREA, int)
+#define REMAP_PIN_MEM_AREA _IOW(PIN_MEM_MAGIC, _REMAP_PIN_MEM_AREA, int)
+#define FINISH_PIN_MEM_DUMP _IOW(PIN_MEM_MAGIC, _FINISH_PIN_MEM_DUMP, int)
+static int set_pin_mem(struct pin_mem_area_set *pmas)
+{
+ int i;
+ int ret = 0;
+ struct _pin_mem_area *pma;
+ struct mm_struct *mm;
+ struct task_struct *task;
+ struct pid *pid_s;
+
+ pid_s = find_get_pid(pmas->pid);
+ if (!pid_s) {
+ pr_warn("Get pid struct fail:%d.\n", pmas->pid);
+ return -EFAULT;
+ }
+ rcu_read_lock();
+ task = pid_task(pid_s, PIDTYPE_PID);
+ if (!task) {
+ pr_warn("Get task struct fail:%d.\n", pmas->pid);
+ goto fail;
+ }
+ mm = get_task_mm(task);
+ for (i = 0; i < pmas->area_num; i++) {
+ pma = &(pmas->mem_area[i]);
+ ret = pin_mem_area(task, mm, pma->virt_start, pma->virt_end);
+ if (ret) {
+ mmput(mm);
+ goto fail;
+ }
+ }
+ mmput(mm);
+ rcu_read_unlock();
+ put_pid(pid_s);
+ return ret;
+
+fail:
+ rcu_read_unlock();
+ put_pid(pid_s);
+ return -EFAULT;
+}
+
+static int set_pin_mem_area(unsigned long arg)
+{
+ struct pin_mem_area_set pmas;
+ void __user *buf = (void __user *)arg;
+
+ if (!access_ok(buf, sizeof(pmas)))
+ return -EFAULT;
+ if (copy_from_user(&pmas, buf, sizeof(pmas)))
+ return -EINVAL;
+ if (pmas.area_num > MAX_PIN_MEM_AREA_NUM) {
+ pr_warn("Input area_num is too large.\n");
+ return -EINVAL;
+ }
+
+ return set_pin_mem(&pmas);
+}
+
+static int pin_mem_remap(unsigned long arg)
+{
+ int pid;
+ struct task_struct *task;
+ struct mm_struct *mm;
+ vm_fault_t ret;
+ void __user *buf = (void __user *)arg;
+ struct pid *pid_s;
+
+ if (!access_ok(buf, sizeof(int)))
+ return -EINVAL;
+ if (copy_from_user(&pid, buf, sizeof(int)))
+ return -EINVAL;
+
+ pid_s = find_get_pid(pid);
+ if (!pid_s) {
+ pr_warn("Get pid struct fail:%d.\n", pid);
+ return -EINVAL;
+ }
+ rcu_read_lock();
+ task = pid_task(pid_s, PIDTYPE_PID);
+ if (!task) {
+ pr_warn("Get task struct fail:%d.\n", pid);
+ goto fault;
+ }
+ mm = get_task_mm(task);
+ ret = do_mem_remap(pid, mm);
+ if (ret) {
+ pr_warn("Handle pin memory remap fail.\n");
+ mmput(mm);
+ goto fault;
+ }
+ mmput(mm);
+ rcu_read_unlock();
+ put_pid(pid_s);
+ return 0;
+
+fault:
+ rcu_read_unlock();
+ put_pid(pid_s);
+ return -EFAULT;
+}
+
+static long pin_memory_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+ long ret = 0;
+
+ if (_IOC_TYPE(cmd) != PIN_MEM_MAGIC)
+ return -EINVAL;
+ if (_IOC_NR(cmd) > _PIN_MEM_IOC_MAX_NR)
+ return -EINVAL;
+
+ switch (cmd) {
+ case SET_PIN_MEM_AREA:
+ ret = set_pin_mem_area(arg);
+ break;
+ case CLEAR_PIN_MEM_AREA:
+ clear_pin_memory_record();
+ break;
+ case REMAP_PIN_MEM_AREA:
+ ret = pin_mem_remap(arg);
+ break;
+ case FINISH_PIN_MEM_DUMP:
+ ret = finish_pin_mem_dump();
+ break;
+ default:
+ return -EINVAL;
+ }
+ return ret;
+}
+
+static const struct file_operations pin_memory_fops = {
+ .owner = THIS_MODULE,
+ .unlocked_ioctl = pin_memory_ioctl,
+ .compat_ioctl = pin_memory_ioctl,
+};
+
+static struct miscdevice pin_memory_miscdev = {
+ .minor = MISC_DYNAMIC_MINOR,
+ .name = "pinmem",
+ .fops = &pin_memory_fops,
+};
+
+static int pin_memory_init(void)
+{
+ int err = misc_register(&pin_memory_miscdev);
+
+ if (!err)
+ pr_info("pin_memory init\n");
+ else
+ pr_warn("pin_memory init failed!\n");
+ return err;
+}
+
+static void pin_memory_exit(void)
+{
+ misc_deregister(&pin_memory_miscdev);
+ pr_info("pin_memory ko exists!\n");
+}
+
+module_init(pin_memory_init);
+module_exit(pin_memory_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Euler");
+MODULE_DESCRIPTION("pin memory");
diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h
index fc0ef33..30f0df3 100644
--- a/include/linux/crash_core.h
+++ b/include/linux/crash_core.h
@@ -87,4 +87,9 @@ int parse_crashkernel_high(char *cmdline, unsigned long long system_ram,
int parse_crashkernel_low(char *cmdline, unsigned long long system_ram,
unsigned long long *crash_size, unsigned long long *crash_base);
+#ifdef CONFIG_PIN_MEMORY
+int __init parse_pin_memory(char *cmdline, unsigned long long system_ram,
+ unsigned long long *pin_size, unsigned long long *pin_base);
+#endif
+
#endif /* LINUX_CRASH_CORE_H */
diff --git a/include/linux/pin_mem.h b/include/linux/pin_mem.h
new file mode 100644
index 0000000..bc8b03e
--- /dev/null
+++ b/include/linux/pin_mem.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
+ * Provide the pin memory method for check point and restore task.
+ */
+#ifndef _LINUX_PIN_MEMORY_H
+#define _LINUX_PIN_MEMORY_H
+
+#ifdef CONFIG_PIN_MEMORY
+#include <linux/errno.h>
+#include <linux/mm_types.h>
+#include <linux/err.h>
+#ifdef CONFIG_ARM64
+#include <linux/ioport.h>
+#endif
+
+#define PAGE_BUDDY_MAPCOUNT_VALUE (~PG_buddy)
+
+#define COLLECT_PAGES_FINISH 0
+#define COLLECT_PAGES_NEED_CONTINUE 1
+#define COLLECT_PAGES_FAIL -1
+
+#define COMPOUND_PAD_MASK 0xffffffff
+#define COMPOUND_PAD_START 0x88
+#define COMPOUND_PAD_DELTA 0x40
+#define LIST_POISON4 0xdead000000000400
+#define PAGE_FLAGS_CHECK_RESERVED (1UL << PG_reserved)
+#define SHA256_DIGEST_SIZE 32
+#define next_pme(pme) ((unsigned long *)(pme + 1) + pme->nr_pages)
+#define PIN_MEM_DUMP_MAGIC 0xfeab000000001acd
+struct page_map_entry {
+ unsigned long virt_addr;
+ unsigned int nr_pages;
+ unsigned int is_huge_page;
+ unsigned long redirect_start;
+ unsigned long phy_addr_array[0];
+};
+
+struct page_map_info {
+ int pid;
+ int pid_reserved;
+ unsigned int entry_num;
+ int disable_free_page;
+ struct page_map_entry *pme;
+};
+
+struct pin_mem_dump_info {
+ char sha_digest[SHA256_DIGEST_SIZE];
+ unsigned long magic;
+ unsigned int pin_pid_num;
+ struct page_map_info pmi_array[0];
+};
+
+struct redirect_info {
+ unsigned int redirect_pages;
+ unsigned int redirect_index[0];
+};
+
+extern struct page_map_info *get_page_map_info(int pid);
+extern struct page_map_info *create_page_map_info(int pid);
+extern vm_fault_t do_mem_remap(int pid, struct mm_struct *mm);
+extern vm_fault_t do_anon_page_remap(struct vm_area_struct *vma, unsigned long address,
+ pmd_t *pmd, struct page *page);
+extern void clear_pin_memory_record(void);
+extern int pin_mem_area(struct task_struct *task, struct mm_struct *mm,
+ unsigned long start_addr, unsigned long end_addr);
+extern vm_fault_t do_anon_huge_page_remap(struct vm_area_struct *vma, unsigned long address,
+ pmd_t *pmd, struct page *page);
+extern int finish_pin_mem_dump(void);
+
+/* reserve space for pin memory*/
+#ifdef CONFIG_ARM64
+extern struct resource pin_memory_resource;
+#endif
+extern void init_reserve_page_map(unsigned long map_addr, unsigned long map_size);
+
+#endif /* CONFIG_PIN_MEMORY */
+#endif /* _LINUX_PIN_MEMORY_H */
diff --git a/kernel/crash_core.c b/kernel/crash_core.c
index bfed474..2407de3 100644
--- a/kernel/crash_core.c
+++ b/kernel/crash_core.c
@@ -450,6 +450,17 @@ void __init reserve_crashkernel(void)
}
#endif /* CONFIG_ARCH_WANT_RESERVE_CRASH_KERNEL */
+#ifdef CONFIG_PIN_MEMORY
+int __init parse_pin_memory(char *cmdline,
+ unsigned long long system_ram,
+ unsigned long long *pin_size,
+ unsigned long long *pin_base)
+{
+ return __parse_crashkernel(cmdline, system_ram, pin_size, pin_base,
+ "pinmemory=", NULL);
+}
+#endif
+
Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type,
void *data, size_t data_len)
{
diff --git a/mm/Kconfig b/mm/Kconfig
index 390165f..930dc13 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -859,4 +859,12 @@ config ARCH_HAS_HUGEPD
config MAPPING_DIRTY_HELPERS
bool
+config PIN_MEMORY
+ bool "Support for pin memory"
+ depends on CHECKPOINT_RESTORE
+ help
+ Say y here to enable the pin memory feature for checkpoint
+ and restore. We can pin the memory data of tasks and collect
+ the corresponding physical pages mapping info in checkpoint,
+ and remap the physical pages to restore tasks in restore.
endmenu
diff --git a/mm/Makefile b/mm/Makefile
index d73aed0..4963827 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -120,3 +120,4 @@ obj-$(CONFIG_MEMFD_CREATE) += memfd.o
obj-$(CONFIG_MAPPING_DIRTY_HELPERS) += mapping_dirty_helpers.o
obj-$(CONFIG_PTDUMP_CORE) += ptdump.o
obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o
+obj-$(CONFIG_PIN_MEMORY) += pin_mem.o
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 0bc4a2c..8a11d30 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2996,3 +2996,64 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
update_mmu_cache_pmd(vma, address, pvmw->pmd);
}
#endif
+
+#ifdef CONFIG_PIN_MEMORY
+vm_fault_t do_anon_huge_page_remap(struct vm_area_struct *vma, unsigned long address,
+ pmd_t *pmd, struct page *page)
+{
+ gfp_t gfp;
+ pgtable_t pgtable;
+ spinlock_t *ptl;
+ pmd_t entry;
+ vm_fault_t ret = 0;
+
+ if (unlikely(anon_vma_prepare(vma)))
+ return VM_FAULT_OOM;
+ if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
+ return VM_FAULT_OOM;
+ gfp = alloc_hugepage_direct_gfpmask(vma);
+ prep_transhuge_page(page);
+ if (mem_cgroup_charge(page, vma->vm_mm, gfp)) {
+ put_page(page);
+ count_vm_event(THP_FAULT_FALLBACK);
+ count_vm_event(THP_FAULT_FALLBACK_CHARGE);
+ return VM_FAULT_FALLBACK;
+ }
+ cgroup_throttle_swaprate(page, gfp);
+
+ pgtable = pte_alloc_one(vma->vm_mm);
+ if (unlikely(!pgtable)) {
+ ret = VM_FAULT_OOM;
+ goto release;
+ }
+ __SetPageUptodate(page);
+ ptl = pmd_lock(vma->vm_mm, pmd);
+ if (unlikely(!pmd_none(*pmd))) {
+ goto unlock_release;
+ } else {
+ ret = check_stable_address_space(vma->vm_mm);
+ if (ret)
+ goto unlock_release;
+ entry = mk_huge_pmd(page, vma->vm_page_prot);
+ entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
+ page_add_new_anon_rmap(page, vma, address, true);
+ lru_cache_add_inactive_or_unevictable(page, vma);
+ pgtable_trans_huge_deposit(vma->vm_mm, pmd, pgtable);
+ set_pmd_at(vma->vm_mm, address, pmd, entry);
+ add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
+ mm_inc_nr_ptes(vma->vm_mm);
+ spin_unlock(ptl);
+ count_vm_event(THP_FAULT_ALLOC);
+ count_memcg_event_mm(vma->vm_mm, THP_FAULT_ALLOC);
+ }
+
+ return 0;
+unlock_release:
+ spin_unlock(ptl);
+release:
+ if (pgtable)
+ pte_free(vma->vm_mm, pgtable);
+ put_page(page);
+ return ret;
+}
+#endif
diff --git a/mm/memory.c b/mm/memory.c
index 50632c4..7b7f1a7 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -5248,3 +5248,62 @@ void ptlock_free(struct page *page)
kmem_cache_free(page_ptl_cachep, page->ptl);
}
#endif
+
+#ifdef CONFIG_PIN_MEMORY
+vm_fault_t do_anon_page_remap(struct vm_area_struct *vma, unsigned long address,
+ pmd_t *pmd, struct page *page)
+{
+ pte_t entry;
+ spinlock_t *ptl;
+ pte_t *pte;
+ vm_fault_t ret = 0;
+
+ if (pte_alloc(vma->vm_mm, pmd))
+ return VM_FAULT_OOM;
+
+ /* See the comment in pte_alloc_one_map() */
+ if (unlikely(pmd_trans_unstable(pmd)))
+ return 0;
+
+ /* Allocate our own private page. */
+ if (unlikely(anon_vma_prepare(vma)))
+ goto oom;
+
+ if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL))
+ goto oom_free_page;
+ cgroup_throttle_swaprate(page, GFP_KERNEL);
+
+ __SetPageUptodate(page);
+
+ entry = mk_pte(page, vma->vm_page_prot);
+ if (vma->vm_flags & VM_WRITE)
+ entry = pte_mkwrite(pte_mkdirty(entry));
+ pte = pte_offset_map_lock(vma->vm_mm, pmd, address,
+ &ptl);
+ if (!pte_none(*pte)) {
+ ret = VM_FAULT_FALLBACK;
+ goto release;
+ }
+
+ ret = check_stable_address_space(vma->vm_mm);
+ if (ret)
+ goto release;
+ inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
+ page_add_new_anon_rmap(page, vma, address, false);
+ lru_cache_add_inactive_or_unevictable(page, vma);
+
+ set_pte_at(vma->vm_mm, address, pte, entry);
+ /* No need to invalidate - it was non-present before */
+ update_mmu_cache(vma, address, pte);
+unlock:
+ pte_unmap_unlock(pte, ptl);
+ return ret;
+release:
+ put_page(page);
+ goto unlock;
+oom_free_page:
+ put_page(page);
+oom:
+ return VM_FAULT_OOM;
+}
+#endif
diff --git a/mm/pin_mem.c b/mm/pin_mem.c
new file mode 100644
index 0000000..0a143b6
--- /dev/null
+++ b/mm/pin_mem.c
@@ -0,0 +1,950 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
+ * Provide the pin memory method for check point and restore task.
+ */
+#ifdef CONFIG_PIN_MEMORY
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/time.h>
+#include <linux/sched/cputime.h>
+#include <linux/tick.h>
+#include <linux/mm.h>
+#include <linux/pin_mem.h>
+#include <linux/idr.h>
+#include <linux/page-isolation.h>
+#include <linux/sched/mm.h>
+#include <linux/ctype.h>
+#include <linux/highmem.h>
+#include <crypto/sha.h>
+
+#define MAX_PIN_PID_NUM 128
+static DEFINE_SPINLOCK(page_map_entry_lock);
+
+struct pin_mem_dump_info *pin_mem_dump_start;
+unsigned int pin_pid_num;
+static unsigned int *pin_pid_num_addr;
+static unsigned long __page_map_entry_start;
+static unsigned long page_map_entry_end;
+static struct page_map_info *user_space_reserve_start;
+static struct page_map_entry *page_map_entry_start;
+unsigned int max_pin_pid_num __read_mostly;
+unsigned long redirect_space_size;
+unsigned long redirect_space_start;
+#define DEFAULT_REDIRECT_SPACE_SIZE 0x100000
+
+static int __init setup_max_pin_pid_num(char *str)
+{
+ int ret = 0;
+
+ if (!str)
+ goto out;
+
+ ret = kstrtouint(str, 10, &max_pin_pid_num);
+out:
+ if (ret) {
+ pr_warn("Unable to parse max pin pid num.\n");
+ } else {
+ if (max_pin_pid_num > MAX_PIN_PID_NUM) {
+ max_pin_pid_num = 0;
+ pr_warn("Input max_pin_pid_num is too large.\n");
+ }
+ }
+ return ret;
+}
+early_param("max_pin_pid_num", setup_max_pin_pid_num);
+
+static int __init setup_redirect_space_size(char *str)
+{
+ if (!str)
+ goto out;
+
+ redirect_space_size = memparse(str, NULL);
+out:
+ if (!redirect_space_size) {
+ pr_warn("Unable to parse redirect space size, use the default value.\n");
+ redirect_space_size = DEFAULT_REDIRECT_SPACE_SIZE;
+ }
+ return 0;
+}
+early_param("redirect_space_size", setup_redirect_space_size);
+
+struct page_map_info *create_page_map_info(int pid)
+{
+ struct page_map_info *new;
+
+ if (!user_space_reserve_start)
+ return NULL;
+
+ if (pin_pid_num >= max_pin_pid_num) {
+ pr_warn("Pin pid num too large than max_pin_pid_num, fail create: %d!", pid);
+ return NULL;
+ }
+ new = (struct page_map_info *)(user_space_reserve_start + pin_pid_num);
+ new->pid = pid;
+ new->pme = NULL;
+ new->entry_num = 0;
+ new->pid_reserved = false;
+ new->disable_free_page = false;
+ (*pin_pid_num_addr)++;
+ pin_pid_num++;
+ return new;
+}
+EXPORT_SYMBOL_GPL(create_page_map_info);
+
+struct page_map_info *get_page_map_info(int pid)
+{
+ int i;
+
+ if (!user_space_reserve_start)
+ return NULL;
+
+ for (i = 0; i < pin_pid_num; i++) {
+ if (user_space_reserve_start[i].pid == pid)
+ return &(user_space_reserve_start[i]);
+ }
+ return NULL;
+}
+EXPORT_SYMBOL_GPL(get_page_map_info);
+
+static struct page *find_head_page(struct page *page)
+{
+ struct page *p = page;
+
+ while (!PageBuddy(p)) {
+ if (PageLRU(p))
+ return NULL;
+ p--;
+ }
+ return p;
+}
+
+static void spilt_page_area_left(struct zone *zone, struct free_area *area, struct page *page,
+ unsigned long size, int order)
+{
+ unsigned long cur_size = 1 << order;
+ unsigned long total_size = 0;
+
+ while (size && cur_size > size) {
+ cur_size >>= 1;
+ order--;
+ area--;
+ if (cur_size <= size) {
+ list_add(&page[total_size].lru, &area->free_list[MIGRATE_MOVABLE]);
+ atomic_set(&(page[total_size]._mapcount), PAGE_BUDDY_MAPCOUNT_VALUE);
+ set_page_private(&page[total_size], order);
+ set_pageblock_migratetype(&page[total_size], MIGRATE_MOVABLE);
+ area->nr_free++;
+ total_size += cur_size;
+ size -= cur_size;
+ }
+ }
+}
+
+static void spilt_page_area_right(struct zone *zone, struct free_area *area, struct page *page,
+ unsigned long size, int order)
+{
+ unsigned long cur_size = 1 << order;
+ struct page *right_page, *head_page;
+
+ right_page = page + size;
+ while (size && cur_size > size) {
+ cur_size >>= 1;
+ order--;
+ area--;
+ if (cur_size <= size) {
+ head_page = right_page - cur_size;
+ list_add(&head_page->lru, &area->free_list[MIGRATE_MOVABLE]);
+ atomic_set(&(head_page->_mapcount), PAGE_BUDDY_MAPCOUNT_VALUE);
+ set_page_private(head_page, order);
+ set_pageblock_migratetype(head_page, MIGRATE_MOVABLE);
+ area->nr_free++;
+ size -= cur_size;
+ right_page = head_page;
+ }
+ }
+}
+
+void reserve_page_from_buddy(unsigned long nr_pages, struct page *page)
+{
+ unsigned int current_order;
+ struct page *page_end;
+ struct free_area *area;
+ struct zone *zone;
+ struct page *head_page;
+
+ head_page = find_head_page(page);
+ if (!head_page) {
+ pr_warn("Find page head fail.");
+ return;
+ }
+ current_order = head_page->private;
+ page_end = head_page + (1 << current_order);
+ zone = page_zone(head_page);
+ area = &(zone->free_area[current_order]);
+ list_del(&head_page->lru);
+ atomic_set(&head_page->_mapcount, -1);
+ set_page_private(head_page, 0);
+ area->nr_free--;
+ if (head_page != page)
+ spilt_page_area_left(zone, area, head_page,
+ (unsigned long)(page - head_page), current_order);
+ page = page + nr_pages;
+ if (page < page_end) {
+ spilt_page_area_right(zone, area, page,
+ (unsigned long)(page_end - page), current_order);
+ } else if (page > page_end) {
+ pr_warn("Find page end smaller than page.");
+ }
+}
+
+static inline void reserve_user_normal_pages(struct page *page)
+{
+ atomic_inc(&page->_refcount);
+ reserve_page_from_buddy(1, page);
+}
+
+static void init_huge_pmd_pages(struct page *head_page)
+{
+ int i = 0;
+ struct page *page = head_page;
+
+ __set_bit(PG_head, &page->flags);
+ __set_bit(PG_active, &page->flags);
+ atomic_set(&page->_refcount, 1);
+ page++;
+ i++;
+ page->compound_head = (unsigned long)head_page + 1;
+ page->compound_dtor = HUGETLB_PAGE_DTOR + 1;
+ page->compound_order = HPAGE_PMD_ORDER;
+ page++;
+ i++;
+ page->compound_head = (unsigned long)head_page + 1;
+ i++;
+ INIT_LIST_HEAD(&(page->deferred_list));
+ for (; i < HPAGE_PMD_NR; i++) {
+ page = head_page + i;
+ page->compound_head = (unsigned long)head_page + 1;
+ }
+}
+
+static inline void reserve_user_huge_pmd_pages(struct page *page)
+{
+ atomic_inc(&page->_refcount);
+ reserve_page_from_buddy((1 << HPAGE_PMD_ORDER), page);
+ init_huge_pmd_pages(page);
+}
+
+int reserve_user_map_pages_fail;
+
+void free_user_map_pages(unsigned int pid_index, unsigned int entry_index, unsigned int page_index)
+{
+ unsigned int i, j, index, order;
+ struct page_map_info *pmi;
+ struct page_map_entry *pme;
+ struct page *page;
+ unsigned long phy_addr;
+
+ for (index = 0; index < pid_index; index++) {
+ pmi = &(user_space_reserve_start[index]);
+ pme = pmi->pme;
+ for (i = 0; i < pmi->entry_num; i++) {
+ for (j = 0; j < pme->nr_pages; j++) {
+ order = pme->is_huge_page ? HPAGE_PMD_ORDER : 0;
+ phy_addr = pme->phy_addr_array[j];
+ if (phy_addr) {
+ page = phys_to_page(phy_addr);
+ if (!(page->flags & PAGE_FLAGS_CHECK_RESERVED)) {
+ __free_pages(page, order);
+ pme->phy_addr_array[j] = 0;
+ }
+ }
+ }
+ pme = (struct page_map_entry *)next_pme(pme);
+ }
+ }
+ pmi = &(user_space_reserve_start[index]);
+ pme = pmi->pme;
+ for (i = 0; i < entry_index; i++) {
+ for (j = 0; j < pme->nr_pages; j++) {
+ order = pme->is_huge_page ? HPAGE_PMD_ORDER : 0;
+ phy_addr = pme->phy_addr_array[j];
+ if (phy_addr) {
+ page = phys_to_page(phy_addr);
+ if (!(page->flags & PAGE_FLAGS_CHECK_RESERVED)) {
+ __free_pages(page, order);
+ pme->phy_addr_array[j] = 0;
+ }
+ }
+ }
+ pme = (struct page_map_entry *)next_pme(pme);
+ }
+ for (j = 0; j < page_index; j++) {
+ order = pme->is_huge_page ? HPAGE_PMD_ORDER : 0;
+ phy_addr = pme->phy_addr_array[j];
+ if (phy_addr) {
+ page = phys_to_page(phy_addr);
+ if (!(page->flags & PAGE_FLAGS_CHECK_RESERVED)) {
+ __free_pages(page, order);
+ pme->phy_addr_array[j] = 0;
+ }
+ }
+ }
+}
+
+bool check_redirect_end_valid(struct redirect_info *redirect_start,
+ unsigned long max_redirect_page_num)
+{
+ unsigned long redirect_end;
+
+ redirect_end = ((unsigned long)(redirect_start + 1) +
+ max_redirect_page_num * sizeof(unsigned int));
+ if (redirect_end > redirect_space_start + redirect_space_size)
+ return false;
+ return false;
+}
+
+static void reserve_user_space_map_pages(void)
+{
+ struct page_map_info *pmi;
+ struct page_map_entry *pme;
+ unsigned int i, j, index;
+ struct page *page;
+ unsigned long flags;
+ unsigned long phy_addr;
+ unsigned long redirect_pages = 0;
+ struct redirect_info *redirect_start = (struct redirect_info *)redirect_space_start;
+
+ if (!user_space_reserve_start || !redirect_start)
+ return;
+ spin_lock_irqsave(&page_map_entry_lock, flags);
+ for (index = 0; index < pin_pid_num; index++) {
+ pmi = &(user_space_reserve_start[index]);
+ pme = pmi->pme;
+ for (i = 0; i < pmi->entry_num; i++) {
+ redirect_pages = 0;
+ if (!check_redirect_end_valid(redirect_start, pme->nr_pages))
+ redirect_start = NULL;
+ for (j = 0; j < pme->nr_pages; j++) {
+ phy_addr = pme->phy_addr_array[j];
+ if (!phy_addr)
+ continue;
+ page = phys_to_page(phy_addr);
+ if (atomic_read(&page->_refcount)) {
+ if ((page->flags & PAGE_FLAGS_CHECK_RESERVED)
+ && !pme->redirect_start)
+ pme->redirect_start =
+ (unsigned long)redirect_start;
+ if (redirect_start &&
+ (page->flags & PAGE_FLAGS_CHECK_RESERVED)) {
+ redirect_start->redirect_index[redirect_pages] = j;
+ redirect_pages++;
+ continue;
+ } else {
+ reserve_user_map_pages_fail = 1;
+ pr_warn("Page %pK refcount %d large than zero, no need reserve.\n",
+ page, atomic_read(&page->_refcount));
+ goto free_pages;
+ }
+ }
+ if (!pme->is_huge_page)
+ reserve_user_normal_pages(page);
+ else
+ reserve_user_huge_pmd_pages(page);
+ }
+ pme = (struct page_map_entry *)next_pme(pme);
+ if (redirect_pages && redirect_start) {
+ redirect_start->redirect_pages = redirect_pages;
+ redirect_start = (struct redirect_info *)(
+ (unsigned long)(redirect_start + 1) +
+ redirect_start->redirect_pages * sizeof(unsigned int));
+ }
+ }
+ }
+ spin_unlock(&page_map_entry_lock);
+ return;
+free_pages:
+ free_user_map_pages(index, i, j);
+ spin_unlock(&page_map_entry_lock);
+}
+
+
+int calculate_pin_mem_digest(struct pin_mem_dump_info *pmdi, char *digest)
+{
+ int i;
+ struct sha256_state sctx;
+
+ if (!digest)
+ digest = pmdi->sha_digest;
+ sha256_init(&sctx);
+ sha256_update(&sctx, (unsigned char *)(&(pmdi->magic)),
+ sizeof(struct pin_mem_dump_info) - SHA256_DIGEST_SIZE);
+ for (i = 0; i < pmdi->pin_pid_num; i++) {
+ sha256_update(&sctx, (unsigned char *)(&(pmdi->pmi_array[i])),
+ sizeof(struct page_map_info));
+ }
+ sha256_final(&sctx, digest);
+ return 0;
+}
+
+static int check_sha_digest(struct pin_mem_dump_info *pmdi)
+{
+ int ret = 0;
+ char digest[SHA256_DIGEST_SIZE] = {0};
+
+ ret = calculate_pin_mem_digest(pmdi, digest);
+ if (ret) {
+ pr_warn("calculate pin mem digest fail:%d\n", ret);
+ return ret;
+ }
+ if (memcmp(pmdi->sha_digest, digest, SHA256_DIGEST_SIZE)) {
+ pr_warn("pin mem dump info sha256 digest match error!\n");
+ return -EFAULT;
+ }
+ return ret;
+}
+
+/*
+ * The whole page map entry collect process must be Sequentially.
+ * The user_space_reserve_start points to the first page map info for
+ * the first dump task. And the page_map_entry_start points to
+ * the first page map entry of the first dump vma.
+ */
+static void init_page_map_info(struct pin_mem_dump_info *pmdi, unsigned long map_len)
+{
+ if (pin_mem_dump_start || !max_pin_pid_num) {
+ pr_warn("pin page map already init or max_pin_pid_num not set.\n");
+ return;
+ }
+ if (map_len < sizeof(struct pin_mem_dump_info) +
+ max_pin_pid_num * sizeof(struct page_map_info) + redirect_space_size) {
+ pr_warn("pin memory reserved memblock too small.\n");
+ return;
+ }
+ if ((pmdi->magic != PIN_MEM_DUMP_MAGIC) || (pmdi->pin_pid_num > max_pin_pid_num) ||
+ check_sha_digest(pmdi))
+ memset(pmdi, 0, sizeof(struct pin_mem_dump_info));
+ pin_mem_dump_start = pmdi;
+ pin_pid_num = pmdi->pin_pid_num;
+ pr_info("pin_pid_num: %d\n", pin_pid_num);
+ pin_pid_num_addr = &(pmdi->pin_pid_num);
+ user_space_reserve_start =
+ (struct page_map_info *)pmdi->pmi_array;
+ page_map_entry_start =
+ (struct page_map_entry *)(user_space_reserve_start + max_pin_pid_num);
+ page_map_entry_end = (unsigned long)pmdi + map_len - redirect_space_size;
+ redirect_space_start = page_map_entry_end;
+ if (pin_pid_num > 0)
+ reserve_user_space_map_pages();
+}
+
+int finish_pin_mem_dump(void)
+{
+ int ret;
+
+ pin_mem_dump_start->magic = PIN_MEM_DUMP_MAGIC;
+ memset(pin_mem_dump_start->sha_digest, 0, SHA256_DIGEST_SIZE);
+ ret = calculate_pin_mem_digest(pin_mem_dump_start, NULL);
+ if (ret) {
+ pr_warn("calculate pin mem digest fail:%d\n", ret);
+ return ret;
+ }
+ return ret;
+}
+
+int collect_pmd_huge_pages(struct task_struct *task,
+ unsigned long start_addr, unsigned long end_addr, struct page_map_entry *pme)
+{
+ long res;
+ int index = 0;
+ unsigned long start = start_addr;
+ struct page *temp_page;
+
+ while (start < end_addr) {
+ temp_page = NULL;
+ res = get_user_pages_remote(task->mm, start, 1,
+ FOLL_TOUCH | FOLL_GET, &temp_page, NULL, NULL);
+ if (!res) {
+ pr_warn("Get huge page for addr(%lx) fail.", start);
+ return COLLECT_PAGES_FAIL;
+ }
+ if (PageHead(temp_page)) {
+ start += HPAGE_PMD_SIZE;
+ pme->phy_addr_array[index] = page_to_phys(temp_page);
+ index++;
+ } else {
+ pme->nr_pages = index;
+ atomic_dec(&((temp_page)->_refcount));
+ return COLLECT_PAGES_NEED_CONTINUE;
+ }
+ }
+ pme->nr_pages = index;
+ return COLLECT_PAGES_FINISH;
+}
+
+int collect_normal_pages(struct task_struct *task,
+ unsigned long start_addr, unsigned long end_addr, struct page_map_entry *pme)
+{
+ int res;
+ unsigned long next;
+ unsigned long i, nr_pages;
+ struct page *tmp_page;
+ unsigned long *phy_addr_array = pme->phy_addr_array;
+ struct page **page_array = (struct page **)pme->phy_addr_array;
+
+ next = (start_addr & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE;
+ next = (next > end_addr) ? end_addr : next;
+ pme->nr_pages = 0;
+ while (start_addr < next) {
+ nr_pages = (PAGE_ALIGN(next) - start_addr) / PAGE_SIZE;
+ res = get_user_pages_remote(task->mm, start_addr, 1,
+ FOLL_TOUCH | FOLL_GET, &tmp_page, NULL, NULL);
+ if (!res) {
+ pr_warn("Get user page of %lx fail.\n", start_addr);
+ return COLLECT_PAGES_FAIL;
+ }
+ if (PageHead(tmp_page)) {
+ atomic_dec(&(tmp_page->_refcount));
+ return COLLECT_PAGES_NEED_CONTINUE;
+ }
+ atomic_dec(&(tmp_page->_refcount));
+ if (PageTail(tmp_page)) {
+ start_addr = next;
+ pme->virt_addr = start_addr;
+ next = (next + HPAGE_PMD_SIZE) > end_addr ?
+ end_addr : (next + HPAGE_PMD_SIZE);
+ continue;
+ }
+ res = get_user_pages_remote(task->mm, start_addr, nr_pages,
+ FOLL_TOUCH | FOLL_GET, page_array, NULL, NULL);
+ if (!res) {
+ pr_warn("Get user pages of %lx fail.\n", start_addr);
+ return COLLECT_PAGES_FAIL;
+ }
+ for (i = 0; i < nr_pages; i++)
+ phy_addr_array[i] = page_to_phys(page_array[i]);
+ pme->nr_pages += nr_pages;
+ page_array += nr_pages;
+ phy_addr_array += nr_pages;
+ start_addr = next;
+ next = (next + HPAGE_PMD_SIZE) > end_addr ? end_addr : (next + HPAGE_PMD_SIZE);
+ }
+ return COLLECT_PAGES_FINISH;
+}
+
+/* Users make sure that the pin memory belongs to anonymous vma. */
+int pin_mem_area(struct task_struct *task, struct mm_struct *mm,
+ unsigned long start_addr, unsigned long end_addr)
+{
+ int pid, ret;
+ int is_huge_page = false;
+ unsigned int page_size;
+ unsigned long nr_pages, flags;
+ struct page_map_entry *pme;
+ struct page_map_info *pmi;
+ struct vm_area_struct *vma;
+ unsigned long i;
+ struct page *tmp_page;
+
+ if (!page_map_entry_start
+ || !task || !mm
+ || start_addr >= end_addr)
+ return -EFAULT;
+
+ pid = task->pid;
+ spin_lock_irqsave(&page_map_entry_lock, flags);
+ nr_pages = ((end_addr - start_addr) / PAGE_SIZE);
+ if ((unsigned long)page_map_entry_start + nr_pages * sizeof(struct page *) >=
+ page_map_entry_end) {
+ pr_warn("Page map entry use up!\n");
+ ret = -EFAULT;
+ goto finish;
+ }
+ vma = find_extend_vma(mm, start_addr);
+ if (!vma) {
+ pr_warn("Find no match vma!\n");
+ ret = -EFAULT;
+ goto finish;
+ }
+ if (start_addr == (start_addr & HPAGE_PMD_MASK) &&
+ transparent_hugepage_enabled(vma)) {
+ page_size = HPAGE_PMD_SIZE;
+ is_huge_page = true;
+ } else {
+ page_size = PAGE_SIZE;
+ }
+ pme = page_map_entry_start;
+ pme->virt_addr = start_addr;
+ pme->redirect_start = 0;
+ pme->is_huge_page = is_huge_page;
+ memset(pme->phy_addr_array, 0, nr_pages * sizeof(unsigned long));
+ down_write(&mm->mmap_lock);
+ if (!is_huge_page) {
+ ret = collect_normal_pages(task, start_addr, end_addr, pme);
+ if (ret != COLLECT_PAGES_FAIL && !pme->nr_pages) {
+ if (ret == COLLECT_PAGES_FINISH) {
+ ret = 0;
+ up_write(&mm->mmap_lock);
+ goto finish;
+ }
+ pme->is_huge_page = true;
+ page_size = HPAGE_PMD_SIZE;
+ ret = collect_pmd_huge_pages(task, pme->virt_addr, end_addr, pme);
+ }
+ } else {
+ ret = collect_pmd_huge_pages(task, start_addr, end_addr, pme);
+ if (ret != COLLECT_PAGES_FAIL && !pme->nr_pages) {
+ if (ret == COLLECT_PAGES_FINISH) {
+ ret = 0;
+ up_write(&mm->mmap_lock);
+ goto finish;
+ }
+ pme->is_huge_page = false;
+ page_size = PAGE_SIZE;
+ ret = collect_normal_pages(task, pme->virt_addr, end_addr, pme);
+ }
+ }
+ up_write(&mm->mmap_lock);
+ if (ret == COLLECT_PAGES_FAIL) {
+ ret = -EFAULT;
+ goto finish;
+ }
+
+ /* check for zero pages */
+ for (i = 0; i < pme->nr_pages; i++) {
+ tmp_page = phys_to_page(pme->phy_addr_array[i]);
+ if (!pme->is_huge_page) {
+ if (page_to_pfn(tmp_page) == my_zero_pfn(pme->virt_addr + i * PAGE_SIZE))
+ pme->phy_addr_array[i] = 0;
+ } else if (is_huge_zero_page(tmp_page))
+ pme->phy_addr_array[i] = 0;
+ }
+
+ page_map_entry_start = (struct page_map_entry *)(next_pme(pme));
+ pmi = get_page_map_info(pid);
+ if (!pmi)
+ pmi = create_page_map_info(pid);
+ if (!pmi) {
+ pr_warn("Create page map info fail for pid: %d!\n", pid);
+ ret = -EFAULT;
+ goto finish;
+ }
+ if (!pmi->pme)
+ pmi->pme = pme;
+ pmi->entry_num++;
+ spin_unlock_irqrestore(&page_map_entry_lock, flags);
+ if (ret == COLLECT_PAGES_NEED_CONTINUE)
+ ret = pin_mem_area(task, mm, pme->virt_addr + pme->nr_pages * page_size, end_addr);
+ return ret;
+finish:
+ spin_unlock_irqrestore(&page_map_entry_lock, flags);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(pin_mem_area);
+
+vm_fault_t remap_normal_pages(struct mm_struct *mm, struct vm_area_struct *vma,
+ struct page_map_entry *pme)
+{
+ int ret;
+ unsigned int j, i;
+ pgd_t *pgd;
+ p4d_t *p4d;
+ pmd_t *pmd;
+ pud_t *pud;
+ struct page *page, *new;
+ unsigned long address;
+ unsigned long phy_addr;
+ unsigned int redirect_pages = 0;
+ struct redirect_info *redirect_start;
+
+ redirect_start = (struct redirect_info *)pme->redirect_start;
+ for (j = 0; j < pme->nr_pages; j++) {
+ address = pme->virt_addr + j * PAGE_SIZE;
+ phy_addr = pme->phy_addr_array[j];
+ if (!phy_addr)
+ continue;
+ page = phys_to_page(phy_addr);
+ if (page_to_pfn(page) == my_zero_pfn(address)) {
+ pme->phy_addr_array[j] = 0;
+ continue;
+ }
+ pme->phy_addr_array[j] = 0;
+ if (redirect_start && (redirect_pages < redirect_start->redirect_pages) &&
+ (j == redirect_start->redirect_index[redirect_pages])) {
+ new = alloc_zeroed_user_highpage_movable(vma, address);
+ if (!new) {
+ pr_warn("Redirect alloc page fail\n");
+ continue;
+ }
+ copy_page(page_to_virt(new), phys_to_virt(phy_addr));
+ page = new;
+ redirect_pages++;
+ }
+ page->mapping = NULL;
+ pgd = pgd_offset(mm, address);
+ p4d = p4d_alloc(mm, pgd, address);
+ if (!p4d) {
+ ret = VM_FAULT_OOM;
+ goto free;
+ }
+ pud = pud_alloc(mm, p4d, address);
+ if (!pud) {
+ ret = VM_FAULT_OOM;
+ goto free;
+ }
+ pmd = pmd_alloc(mm, pud, address);
+ if (!pmd) {
+ ret = VM_FAULT_OOM;
+ goto free;
+ }
+ ret = do_anon_page_remap(vma, address, pmd, page);
+ if (ret)
+ goto free;
+ }
+ return 0;
+free:
+ for (i = j; i < pme->nr_pages; i++) {
+ phy_addr = pme->phy_addr_array[i];
+ if (phy_addr) {
+ __free_page(phys_to_page(phy_addr));
+ pme->phy_addr_array[i] = 0;
+ }
+ }
+ return ret;
+}
+
+static inline gfp_t get_hugepage_gfpmask(struct vm_area_struct *vma)
+{
+ const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE);
+
+ if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
+ return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY);
+ if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
+ return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM;
+ if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
+ return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM :
+ __GFP_KSWAPD_RECLAIM);
+ if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
+ return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM :
+ 0);
+ return GFP_TRANSHUGE_LIGHT;
+}
+
+vm_fault_t remap_huge_pmd_pages(struct mm_struct *mm, struct vm_area_struct *vma,
+ struct page_map_entry *pme)
+{
+ int ret;
+ unsigned int j, i;
+ pgd_t *pgd;
+ p4d_t *p4d;
+ pmd_t *pmd;
+ pud_t *pud;
+ gfp_t gfp;
+ struct page *page, *new;
+ unsigned long address;
+ unsigned long phy_addr;
+ unsigned int redirect_pages = 0;
+ struct redirect_info *redirect_start;
+
+ redirect_start = (struct redirect_info *)pme->redirect_start;
+ for (j = 0; j < pme->nr_pages; j++) {
+ address = pme->virt_addr + j * HPAGE_PMD_SIZE;
+ phy_addr = pme->phy_addr_array[j];
+ if (!phy_addr)
+ continue;
+ page = phys_to_page(phy_addr);
+ if (is_huge_zero_page(page)) {
+ pme->phy_addr_array[j] = 0;
+ continue;
+ }
+ pme->phy_addr_array[j] = 0;
+ if (redirect_start && (redirect_pages < redirect_start->redirect_pages) &&
+ (j == redirect_start->redirect_index[redirect_pages])) {
+ gfp = get_hugepage_gfpmask(vma);
+ new = alloc_hugepage_vma(gfp, vma, address, HPAGE_PMD_ORDER);
+ if (!new) {
+ pr_warn("Redirect alloc huge page fail\n");
+ continue;
+ }
+ memcpy(page_to_virt(new), phys_to_virt(phy_addr), HPAGE_PMD_SIZE);
+ page = new;
+ redirect_pages++;
+ }
+ pgd = pgd_offset(mm, address);
+ p4d = p4d_alloc(mm, pgd, address);
+ if (!p4d) {
+ ret = VM_FAULT_OOM;
+ goto free;
+ }
+ pud = pud_alloc(mm, p4d, address);
+ if (!pud) {
+ ret = VM_FAULT_OOM;
+ goto free;
+ }
+ pmd = pmd_alloc(mm, pud, address);
+ if (!pmd) {
+ ret = VM_FAULT_OOM;
+ goto free;
+ }
+ ret = do_anon_huge_page_remap(vma, address, pmd, page);
+ if (ret)
+ goto free;
+ }
+ return 0;
+free:
+ for (i = j; i < pme->nr_pages; i++) {
+ phy_addr = pme->phy_addr_array[i];
+ if (phy_addr) {
+ page = phys_to_page(phy_addr);
+ if (!(page->flags & PAGE_FLAGS_CHECK_RESERVED)) {
+ __free_pages(page, HPAGE_PMD_ORDER);
+ pme->phy_addr_array[i] = 0;
+ }
+ }
+ }
+ return ret;
+}
+
+static void free_unmap_pages(struct page_map_info *pmi,
+ struct page_map_entry *pme,
+ unsigned int index)
+{
+ unsigned int i, j;
+ unsigned long phy_addr;
+ unsigned int order;
+ struct page *page;
+
+ pme = (struct page_map_entry *)(next_pme(pme));
+ for (i = index; i < pmi->entry_num; i++) {
+ for (j = 0; j < pme->nr_pages; j++) {
+ phy_addr = pme->phy_addr_array[i];
+ if (phy_addr) {
+ page = phys_to_page(phy_addr);
+ order = pme->is_huge_page ? HPAGE_PMD_ORDER : 0;
+ if (!(page->flags & PAGE_FLAGS_CHECK_RESERVED)) {
+ __free_pages(page, order);
+ pme->phy_addr_array[i] = 0;
+ }
+ }
+ }
+ pme = (struct page_map_entry *)(next_pme(pme));
+ }
+}
+
+vm_fault_t do_mem_remap(int pid, struct mm_struct *mm)
+{
+ unsigned int i = 0;
+ vm_fault_t ret = 0;
+ struct vm_area_struct *vma;
+ struct page_map_info *pmi;
+ struct page_map_entry *pme;
+ unsigned long flags;
+
+ if (reserve_user_map_pages_fail)
+ return -EFAULT;
+ pmi = get_page_map_info(pid);
+ if (!pmi)
+ return -EFAULT;
+
+ spin_lock_irqsave(&page_map_entry_lock, flags);
+ pmi->disable_free_page = true;
+ spin_unlock(&page_map_entry_lock);
+ down_write(&mm->mmap_lock);
+ pme = pmi->pme;
+ vma = mm->mmap;
+ while ((i < pmi->entry_num) && (vma != NULL)) {
+ if (pme->virt_addr >= vma->vm_start && pme->virt_addr < vma->vm_end) {
+ i++;
+ if (!vma_is_anonymous(vma)) {
+ pme = (struct page_map_entry *)(next_pme(pme));
+ continue;
+ }
+ if (!pme->is_huge_page) {
+ ret = remap_normal_pages(mm, vma, pme);
+ if (ret < 0)
+ goto free;
+ } else {
+ ret = remap_huge_pmd_pages(mm, vma, pme);
+ if (ret < 0)
+ goto free;
+ }
+ pme = (struct page_map_entry *)(next_pme(pme));
+ } else {
+ vma = vma->vm_next;
+ }
+ }
+ up_write(&mm->mmap_lock);
+ return 0;
+free:
+ free_unmap_pages(pmi, pme, i);
+ up_write(&mm->mmap_lock);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(do_mem_remap);
+
+#if defined(CONFIG_ARM64)
+void init_reserve_page_map(unsigned long map_addr, unsigned long map_size)
+{
+ void *addr;
+
+ if (!map_addr || !map_size)
+ return;
+ addr = phys_to_virt(map_addr);
+ init_page_map_info((struct pin_mem_dump_info *)addr, map_size);
+}
+#else
+void init_reserve_page_map(unsigned long map_addr, unsigned long map_size)
+{
+}
+#endif
+
+static void free_all_reserved_pages(void)
+{
+ unsigned int i, j, index, order;
+ struct page_map_info *pmi;
+ struct page_map_entry *pme;
+ struct page *page;
+ unsigned long phy_addr;
+
+ if (!user_space_reserve_start || reserve_user_map_pages_fail)
+ return;
+
+ for (index = 0; index < pin_pid_num; index++) {
+ pmi = &(user_space_reserve_start[index]);
+ if (pmi->disable_free_page)
+ continue;
+ pme = pmi->pme;
+ for (i = 0; i < pmi->entry_num; i++) {
+ for (j = 0; j < pme->nr_pages; j++) {
+ order = pme->is_huge_page ? HPAGE_PMD_ORDER : 0;
+ phy_addr = pme->phy_addr_array[j];
+ if (phy_addr) {
+ page = phys_to_page(phy_addr);
+ if (!(page->flags & PAGE_FLAGS_CHECK_RESERVED)) {
+ __free_pages(page, order);
+ pme->phy_addr_array[j] = 0;
+ }
+ }
+ }
+ pme = (struct page_map_entry *)next_pme(pme);
+ }
+ }
+}
+
+/* Clear all pin memory record. */
+void clear_pin_memory_record(void)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&page_map_entry_lock, flags);
+ free_all_reserved_pages();
+ if (pin_pid_num_addr) {
+ *pin_pid_num_addr = 0;
+ pin_pid_num = 0;
+ page_map_entry_start = (struct page_map_entry *)__page_map_entry_start;
+ }
+ spin_unlock(&page_map_entry_lock);
+}
+EXPORT_SYMBOL_GPL(clear_pin_memory_record);
+
+#endif /* CONFIG_PIN_MEMORY */
--
2.9.5
1
1
[PATCH OLK-5.10 v1] arm64: Declare var of local_cpu_stop only on PARK
by sangyan＠huawei.com 01 Mar '21
by sangyan＠huawei.com 01 Mar '21
01 Mar '21
From: Sang Yan <sangyan(a)huawei.com>
hulk inclusion
category: feature
bugzilla: 48159
CVE: N/A
Fix compile warning: unused variable 'ops' 'cpu'
while CONFIG_ARM64_CPU_PARK=n.
Put declaration of 'ops' and 'cpu' under
CONFIG_ARM64_CPU_PARK in local_cpu_stop.
Signed-off-by: Sang Yan <sangyan(a)huawei.com>
---
arch/arm64/kernel/smp.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 644bbd7..d7b750a 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -1024,8 +1024,10 @@ void arch_irq_work_raise(void)
static void local_cpu_stop(void)
{
+#ifdef CONFIG_ARM64_CPU_PARK
int cpu;
const struct cpu_operations *ops = NULL;
+#endif
set_cpu_online(smp_processor_id(), false);
--
2.9.5
1
0
26 Feb '21
From: Sang Yan <sangyan(a)huawei.com>
hulk inclusion
category: feature
bugzilla: 48159
CVE: N/A
Introducing a feature of CPU PARK in order to save time
of cpus down and up during kexec, which may cost 250ms of
per cpu's down and 30ms of up.
As a result, for 128 cores, it costs more than 30 seconds
to down and up cpus during kexec. Think about 256 cores and more.
CPU PARK is a state that cpu power-on and staying in spin loop, polling
for exit chances, such as writing exit address.
Reserving a block of memory, to fill with cpu park text section,
exit address and park-magic-flag of each cpu. In implementation,
reserved one page for one cpu core.
Cpus going to park state instead of down in machine_shutdown().
Cpus going out of park state in smp_init instead of brought up.
One of cpu park sections in pre-reserved memory blocks,:
+--------------+
+ exit address +
+--------------+
+ park magic +
+--------------+
+ park codes +
+ . +
+ . +
+ . +
+--------------+
Signed-off-by: Sang Yan <sangyan(a)huawei.com>
Reviewed-by: Jing Xiangfeng <jingxiangfeng(a)huawei.com>
---
arch/arm64/Kconfig | 12 ++
arch/arm64/include/asm/kexec.h | 6 +
arch/arm64/include/asm/smp.h | 15 +++
arch/arm64/kernel/Makefile | 1 +
arch/arm64/kernel/cpu-park.S | 59 ++++++++++
arch/arm64/kernel/machine_kexec.c | 2 +-
arch/arm64/kernel/process.c | 4 +
arch/arm64/kernel/smp.c | 230 ++++++++++++++++++++++++++++++++++++++
arch/arm64/mm/init.c | 55 +++++++++
9 files changed, 383 insertions(+), 1 deletion(-)
create mode 100644 arch/arm64/kernel/cpu-park.S
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b9c5654..0885668 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -345,6 +345,18 @@ config KASAN_SHADOW_OFFSET
default 0xeffffff900000000 if ARM64_VA_BITS_36 && KASAN_SW_TAGS
default 0xffffffffffffffff
+config ARM64_CPU_PARK
+ bool "Support CPU PARK on kexec"
+ depends on SMP
+ depends on KEXEC_CORE
+ help
+ This enables support for CPU PARK feature in
+ order to save time of cpu down to up.
+ CPU park is a state through kexec, spin loop
+ instead of cpu die before jumping to new kernel,
+ jumping out from loop to new kernel entry in
+ smp_init.
+
source "arch/arm64/Kconfig.platforms"
menu "Kernel Features"
diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h
index 79909ae..a133889 100644
--- a/arch/arm64/include/asm/kexec.h
+++ b/arch/arm64/include/asm/kexec.h
@@ -36,6 +36,11 @@
#define CRASH_ADDR_HIGH_MAX MEMBLOCK_ALLOC_ACCESSIBLE
+#ifdef CONFIG_ARM64_CPU_PARK
+/* CPU park state flag: "park" */
+#define PARK_MAGIC 0x7061726b
+#endif
+
#ifndef __ASSEMBLY__
/**
@@ -104,6 +109,7 @@ static inline void crash_post_resume(void) {}
#ifdef CONFIG_KEXEC_CORE
extern void __init reserve_crashkernel(void);
#endif
+void machine_kexec_mask_interrupts(void);
#ifdef CONFIG_KEXEC_FILE
#define ARCH_HAS_KIMAGE_ARCH
diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h
index 2e7f529..8c5d2d6 100644
--- a/arch/arm64/include/asm/smp.h
+++ b/arch/arm64/include/asm/smp.h
@@ -145,6 +145,21 @@ bool cpus_are_stuck_in_kernel(void);
extern void crash_smp_send_stop(void);
extern bool smp_crash_stop_failed(void);
+#ifdef CONFIG_ARM64_CPU_PARK
+#define PARK_SECTION_SIZE 1024
+struct cpu_park_info {
+ /* Physical address of reserved park memory. */
+ unsigned long start;
+ /* park reserve mem len should be PARK_SECTION_SIZE * NR_CPUS */
+ unsigned long len;
+ /* Virtual address of reserved park memory. */
+ unsigned long start_v;
+};
+extern struct cpu_park_info park_info;
+extern void enter_cpu_park(unsigned long text, unsigned long exit);
+extern void do_cpu_park(unsigned long exit);
+extern int kexec_smp_send_park(void);
+#endif
#endif /* ifndef __ASSEMBLY__ */
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 2621d5c..60478d2 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -54,6 +54,7 @@ obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o
obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o \
cpu-reset.o
+obj-$(CONFIG_ARM64_CPU_PARK) += cpu-park.o
obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o
obj-$(CONFIG_ARM64_RELOC_TEST) += arm64-reloc-test.o
arm64-reloc-test-y := reloc_test_core.o reloc_test_syms.o
diff --git a/arch/arm64/kernel/cpu-park.S b/arch/arm64/kernel/cpu-park.S
new file mode 100644
index 0000000..10c685c
--- /dev/null
+++ b/arch/arm64/kernel/cpu-park.S
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * CPU park routines
+ *
+ * Copyright (C) 2020 Huawei Technologies., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+#include <asm/kexec.h>
+#include <asm/sysreg.h>
+#include <asm/virt.h>
+
+.text
+.pushsection .idmap.text, "awx"
+
+/* cpu park helper in idmap section */
+SYM_CODE_START(enter_cpu_park)
+ /* Clear sctlr_el1 flags. */
+ mrs x12, sctlr_el1
+ mov_q x13, SCTLR_ELx_FLAGS
+ bic x12, x12, x13
+ pre_disable_mmu_workaround
+ msr sctlr_el1, x12 /* disable mmu */
+ isb
+
+ mov x18, x0
+ mov x0, x1 /* secondary_entry addr */
+ br x18 /* call do_cpu_park of each cpu */
+SYM_CODE_END(enter_cpu_park)
+
+.popsection
+
+SYM_CODE_START(do_cpu_park)
+ ldr x18, =PARK_MAGIC /* magic number "park" */
+ add x1, x0, #8
+ str x18, [x1] /* set on-park flag */
+ dc civac, x1 /* flush cache of "park" */
+ dsb nsh
+ isb
+
+.Lloop:
+ wfe
+ isb
+ ldr x19, [x0]
+ cmp x19, #0 /* test secondary_entry */
+ b.eq .Lloop
+
+ ic iallu /* invalidate the local I-cache */
+ dsb nsh
+ isb
+
+ br x19 /* jump to secondary_entry */
+SYM_CODE_END(do_cpu_park)
+
diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c
index a0b144c..f47ce96 100644
--- a/arch/arm64/kernel/machine_kexec.c
+++ b/arch/arm64/kernel/machine_kexec.c
@@ -213,7 +213,7 @@ void machine_kexec(struct kimage *kimage)
BUG(); /* Should never get here. */
}
-static void machine_kexec_mask_interrupts(void)
+void machine_kexec_mask_interrupts(void)
{
unsigned int i;
struct irq_desc *desc;
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 73e3b32..10cffee 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -146,6 +146,10 @@ void arch_cpu_idle_dead(void)
*/
void machine_shutdown(void)
{
+#ifdef CONFIG_ARM64_CPU_PARK
+ if (kexec_smp_send_park() == 0)
+ return;
+#endif
smp_shutdown_nonboot_cpus(reboot_cpu);
}
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 18e9727..dea67d0 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -32,6 +32,7 @@
#include <linux/irq_work.h>
#include <linux/kernel_stat.h>
#include <linux/kexec.h>
+
#include <linux/kvm_host.h>
#include <asm/alternative.h>
@@ -93,6 +94,167 @@ static inline int op_cpu_kill(unsigned int cpu)
}
#endif
+#ifdef CONFIG_ARM64_CPU_PARK
+struct cpu_park_section {
+ unsigned long exit; /* exit address of park look */
+ unsigned long magic; /* maigc represent park state */
+ char text[0]; /* text section of park */
+};
+
+static int mmap_cpu_park_mem(void)
+{
+ if (!park_info.start)
+ return -ENOMEM;
+
+ if (park_info.start_v)
+ return 0;
+
+ park_info.start_v = (unsigned long)__ioremap(park_info.start,
+ park_info.len,
+ PAGE_KERNEL_EXEC);
+ if (!park_info.start_v) {
+ pr_warn("map park memory failed.");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static inline unsigned long cpu_park_section_v(unsigned int cpu)
+{
+ return park_info.start_v + PARK_SECTION_SIZE * (cpu - 1);
+}
+
+static inline unsigned long cpu_park_section_p(unsigned int cpu)
+{
+ return park_info.start + PARK_SECTION_SIZE * (cpu - 1);
+}
+
+/*
+ * Write the secondary_entry to exit section of park state.
+ * Then the secondary cpu will jump straight into the kernel
+ * by the secondary_entry.
+ */
+static int write_park_exit(unsigned int cpu)
+{
+ struct cpu_park_section *park_section;
+ unsigned long *park_exit;
+ unsigned long *park_text;
+
+ if (mmap_cpu_park_mem() != 0)
+ return -EPERM;
+
+ park_section = (struct cpu_park_section *)cpu_park_section_v(cpu);
+ park_exit = &park_section->exit;
+ park_text = (unsigned long *)park_section->text;
+ pr_debug("park_text 0x%lx : 0x%lx, do_cpu_park text 0x%lx : 0x%lx",
+ (unsigned long)park_text, *park_text,
+ (unsigned long)do_cpu_park,
+ *(unsigned long *)do_cpu_park);
+
+ /*
+ * Test first 8 bytes to determine
+ * whether needs to write cpu park exit.
+ */
+ if (*park_text == *(unsigned long *)do_cpu_park) {
+ writeq_relaxed(__pa_symbol(secondary_entry), park_exit);
+ __flush_dcache_area((__force void *)park_exit,
+ sizeof(unsigned long));
+ flush_icache_range((unsigned long)park_exit,
+ (unsigned long)(park_exit + 1));
+ sev();
+ dsb(sy);
+ isb();
+
+ pr_debug("Write cpu %u secondary entry 0x%lx to 0x%lx.",
+ cpu, *park_exit, (unsigned long)park_exit);
+ pr_info("Boot cpu %u from PARK state.", cpu);
+ return 0;
+ }
+
+ return -EPERM;
+}
+
+/* Install cpu park sections for the specific cpu. */
+static int install_cpu_park(unsigned int cpu)
+{
+ struct cpu_park_section *park_section;
+ unsigned long *park_exit;
+ unsigned long *park_magic;
+ unsigned long park_text_len;
+
+ park_section = (struct cpu_park_section *)cpu_park_section_v(cpu);
+ pr_debug("Install cpu park on cpu %u park exit 0x%lx park text 0x%lx",
+ cpu, (unsigned long)park_section,
+ (unsigned long)(park_section->text));
+
+ park_exit = &park_section->exit;
+ park_magic = &park_section->magic;
+ park_text_len = PARK_SECTION_SIZE - sizeof(struct cpu_park_section);
+
+ *park_exit = 0UL;
+ *park_magic = 0UL;
+ memcpy((void *)park_section->text, do_cpu_park, park_text_len);
+ __flush_dcache_area((void *)park_section, PARK_SECTION_SIZE);
+
+ return 0;
+}
+
+static int uninstall_cpu_park(unsigned int cpu)
+{
+ unsigned long park_section;
+
+ if (mmap_cpu_park_mem() != 0)
+ return -EPERM;
+
+ park_section = cpu_park_section_v(cpu);
+ memset((void *)park_section, 0, PARK_SECTION_SIZE);
+ __flush_dcache_area((void *)park_section, PARK_SECTION_SIZE);
+
+ return 0;
+}
+
+static int cpu_wait_park(unsigned int cpu)
+{
+ long timeout;
+ struct cpu_park_section *park_section;
+
+ volatile unsigned long *park_magic;
+
+ park_section = (struct cpu_park_section *)cpu_park_section_v(cpu);
+ park_magic = &park_section->magic;
+
+ timeout = USEC_PER_SEC;
+ while (*park_magic != PARK_MAGIC && timeout--)
+ udelay(1);
+
+ if (timeout > 0)
+ pr_debug("cpu %u park done.", cpu);
+ else
+ pr_err("cpu %u park failed.", cpu);
+
+ return *park_magic == PARK_MAGIC;
+}
+
+static void cpu_park(unsigned int cpu)
+{
+ unsigned long park_section_p;
+ unsigned long park_exit_phy;
+ unsigned long do_park;
+ typeof(enter_cpu_park) *park;
+
+ park_section_p = cpu_park_section_p(cpu);
+ park_exit_phy = park_section_p;
+ pr_debug("Go to park cpu %u exit address 0x%lx", cpu, park_exit_phy);
+
+ do_park = park_section_p + sizeof(struct cpu_park_section);
+ park = (void *)__pa_symbol(enter_cpu_park);
+
+ cpu_install_idmap();
+ park(do_park, park_exit_phy);
+ unreachable();
+}
+#endif
/*
* Boot a secondary CPU, and assign it the specified idle task.
@@ -102,6 +264,10 @@ static int boot_secondary(unsigned int cpu, struct task_struct *idle)
{
const struct cpu_operations *ops = get_cpu_ops(cpu);
+#ifdef CONFIG_ARM64_CPU_PARK
+ if (write_park_exit(cpu) == 0)
+ return 0;
+#endif
if (ops->cpu_boot)
return ops->cpu_boot(cpu);
@@ -131,6 +297,9 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
return ret;
}
+#ifdef CONFIG_ARM64_CPU_PARK
+ uninstall_cpu_park(cpu);
+#endif
/*
* CPU was successfully started, wait for it to come online or
* time out.
@@ -844,10 +1013,32 @@ void arch_irq_work_raise(void)
static void local_cpu_stop(void)
{
+#ifdef CONFIG_ARM64_CPU_PARK
+ int cpu;
+ const struct cpu_operations *ops = NULL;
+#endif
+
set_cpu_online(smp_processor_id(), false);
local_daif_mask();
sdei_mask_local_cpu();
+
+#ifdef CONFIG_ARM64_CPU_PARK
+ /*
+ * Go to cpu park state.
+ * Otherwise go to cpu die.
+ */
+ cpu = smp_processor_id();
+ if (kexec_in_progress && park_info.start_v) {
+ machine_kexec_mask_interrupts();
+ cpu_park(cpu);
+
+ ops = get_cpu_ops(cpu);
+ if (ops && ops->cpu_die)
+ ops->cpu_die(cpu);
+ }
+#endif
+
cpu_park_loop();
}
@@ -1053,6 +1244,45 @@ void smp_send_stop(void)
sdei_mask_local_cpu();
}
+#ifdef CONFIG_ARM64_CPU_PARK
+int kexec_smp_send_park(void)
+{
+ unsigned long cpu;
+
+ if (WARN_ON(!kexec_in_progress)) {
+ pr_crit("%s called not in kexec progress.", __func__);
+ return -EPERM;
+ }
+
+ if (mmap_cpu_park_mem() != 0) {
+ pr_info("no cpuparkmem, goto normal way.");
+ return -EPERM;
+ }
+
+ local_irq_disable();
+
+ if (num_online_cpus() > 1) {
+ cpumask_t mask;
+
+ cpumask_copy(&mask, cpu_online_mask);
+ cpumask_clear_cpu(smp_processor_id(), &mask);
+
+ for_each_cpu(cpu, &mask)
+ install_cpu_park(cpu);
+ smp_cross_call(&mask, IPI_CPU_STOP);
+
+ /* Wait for other CPUs to park */
+ for_each_cpu(cpu, &mask)
+ cpu_wait_park(cpu);
+ pr_info("smp park other cpus done\n");
+ }
+
+ sdei_mask_local_cpu();
+
+ return 0;
+}
+#endif
+
#ifdef CONFIG_KEXEC_CORE
void crash_smp_send_stop(void)
{
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 794f992..d01259c 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -236,6 +236,57 @@ static void __init fdt_enforce_memory_region(void)
memblock_add(usable_rgns[1].base, usable_rgns[1].size);
}
+#ifdef CONFIG_ARM64_CPU_PARK
+struct cpu_park_info park_info = {
+ .start = 0,
+ .len = PARK_SECTION_SIZE * NR_CPUS,
+ .start_v = 0,
+};
+
+static int __init parse_park_mem(char *p)
+{
+ if (!p)
+ return 0;
+
+ park_info.start = PAGE_ALIGN(memparse(p, NULL));
+ if (park_info.start == 0)
+ pr_info("cpu park mem params[%s]", p);
+
+ return 0;
+}
+early_param("cpuparkmem", parse_park_mem);
+
+static int __init reserve_park_mem(void)
+{
+ if (park_info.start == 0 || park_info.len == 0)
+ return 0;
+
+ park_info.start = PAGE_ALIGN(park_info.start);
+ park_info.len = PAGE_ALIGN(park_info.len);
+
+ if (!memblock_is_region_memory(park_info.start, park_info.len)) {
+ pr_warn("cannot reserve park mem: region is not memory!");
+ goto out;
+ }
+
+ if (memblock_is_region_reserved(park_info.start, park_info.len)) {
+ pr_warn("cannot reserve park mem: region overlaps reserved memory!");
+ goto out;
+ }
+
+ memblock_remove(park_info.start, park_info.len);
+ pr_info("cpu park mem reserved: 0x%016lx - 0x%016lx (%ld MB)",
+ park_info.start, park_info.start + park_info.len,
+ park_info.len >> 20);
+
+ return 0;
+out:
+ park_info.start = 0;
+ park_info.len = 0;
+ return -EINVAL;
+}
+#endif
+
void __init arm64_memblock_init(void)
{
const s64 linear_region_size = BIT(vabits_actual - 1);
@@ -357,6 +408,10 @@ void __init arm64_memblock_init(void)
reserve_crashkernel();
+#ifdef CONFIG_ARM64_CPU_PARK
+ reserve_park_mem();
+#endif
+
reserve_elfcorehdr();
high_memory = __va(memblock_end_of_DRAM() - 1) + 1;
--
2.9.5
1
1
MPAM bugfix @ 20210224
James Morse (10):
arm64/mpam: Add mpam driver discovery phase and kbuild boiler plate
cacheinfo: Provide a helper to find a cacheinfo leaf
arm64/mpam: Probe supported partid/pmg ranges from devices
arm64/mpam: Supplement MPAM MSC register layout definitions
arm64/mpam: Probe the features resctrl supports
arm64/mpam: Reset controls when CPUs come online
arm64/mpam: Summarize feature support during mpam_enable()
arm64/mpam: resctrl: Re-synchronise resctrl's view of online CPUs
drivers: base: cacheinfo: Add helper to search cacheinfo by of_node
arm64/mpam: Enabling registering and logging error interrupts
Wang ShaoBo (55):
arm64/mpam: Preparing for MPAM refactoring
arm64/mpam: Add helper for getting mpam sysprops
arm64/mpam: Allocate mpam component configuration arrays
arm64/mpam: Pick MPAM resources and events for resctrl_res exported
arm64/mpam: Init resctrl resources' info from resctrl_res selected
arm64/mpam: resctrl: Handle cpuhp and resctrl_dom allocation
arm64/mpam: Implement helpers for handling configuration and
monitoring
arm64/mpam: Migrate old MSCs' discovery process to new branch
arm64/mpam: Add helper for getting MSCs' configuration
arm64/mpam: Probe partid,pmg and feature capabilities' ranges from
classes
arm64/mpam: resctrl: Rebuild configuration and monitoring pipeline
arm64/mpam: resctrl: Append schemata CDP definitions
arm64/mpam: resctrl: Supplement cdpl2,cdpl3 for mount options
arm64/mpam: resctrl: Add helpers for init and destroy schemata list
arm64/mpam: resctrl: Use resctrl_group_init_alloc() to init schema
list
arm64/mpam: resctrl: Write and read schemata by schema_list
arm64/mpam: Support cdp in mpam_sched_in()
arm64/mpam: resctrl: Update resources reset process
arm64/mpam: resctrl: Update closid alloc and free process with bitmap
arm64/mpam: resctrl: Move ctrlmon sysfile write/read function to
mpam_ctrlmon.c
arm64/mpam: Support cdp on allocating monitors
arm64/mpam: resctrl: Support cdp on monitoring data
arm64/mpam: Clean up header files and rearrange declarations
arm64/mpam: resctrl: Remove ctrlmon sysfile
arm64/mpam: resctrl: Remove unnecessary CONFIG_ARM64
arm64/mpam: Implement intpartid narrowing process
arm64/mpam: Using software-defined id for rdtgroup instead of 32-bit
integer
arm64/mpam: resctrl: collect child mon group's monitor data
arm64/mpam: resctrl: Support cpus' monitoring for mon group
arm64/mpam: resctrl: Support priority and hardlimit(Memory bandwidth)
configuration
arm64/mpam: Store intpri and dspri for mpam device reset
arm64/mpam: Squash default priority from mpam device to class
arm64/mpam: Restore extend ctrls' max width for checking schemata
input
arm64/mpam: Re-plan intpartid narrowing process
arm64/mpam: Add hook-events id for ctrl features
arm64/mpam: Integrate monitor data for Memory Bandwidth if cdp enabled
arm64/mpam: Fix MPAM_ESR intPARTID_range error
arm64/mpam: Separate internal and downstream priority event
arm64/mpam: Remap reqpartid,pmg to rmid and intpartid to closid
arm64/mpam: Add wait queue for monitor alloc and free
arm64/mpam: Add resctrl_ctrl_feature structure to manage ctrl features
arm64/mpam: resctrl: Export resource's properties to info directory
arm64/mpam: Split header files into suitable location
arm64/mpam: resctrl: Add rmid file in resctrl sysfs
arm64/mpam: Filter schema control type with ctrl features
arm64/mpam: Simplify mpamid cdp mapping process
arm64/mpam: Set per-cpu's closid to none zero for cdp
ACPI/MPAM: Use acpi_map_pxm_to_node() to get node id for memory node
arm64/mpam: Supplement additional useful ctrl features for mount
options
arm64/mpam: resctrl: Add proper error handling to resctrl_mount()
arm64/mpam: resctrl: Use resctrl_group_init_alloc() for default group
arm64/mpam: resctrl: Allow setting register MPAMCFG_MBW_MIN to 0
arm64/mpam: resctrl: Refresh cpu mask for handling cpuhp
arm64/mpam: Sort domains when cpu online
arm64/mpam: Fix compile warning
arch/arm64/include/asm/mpam.h | 324 +---
arch/arm64/include/asm/mpam_resource.h | 129 --
arch/arm64/include/asm/mpam_sched.h | 8 -
arch/arm64/include/asm/resctrl.h | 514 +++++-
arch/arm64/kernel/Makefile | 2 +-
arch/arm64/kernel/mpam.c | 1499 ----------------
arch/arm64/kernel/mpam/Makefile | 3 +
arch/arm64/kernel/mpam/mpam_ctrlmon.c | 961 ++++++++++
arch/arm64/kernel/mpam/mpam_device.c | 1706 ++++++++++++++++++
arch/arm64/kernel/mpam/mpam_device.h | 140 ++
arch/arm64/kernel/mpam/mpam_internal.h | 345 ++++
arch/arm64/kernel/mpam/mpam_mon.c | 334 ++++
arch/arm64/kernel/mpam/mpam_resctrl.c | 2240 ++++++++++++++++++++++++
arch/arm64/kernel/mpam/mpam_resource.h | 228 +++
arch/arm64/kernel/mpam/mpam_setup.c | 608 +++++++
arch/arm64/kernel/mpam_ctrlmon.c | 623 -------
arch/arm64/kernel/mpam_mon.c | 124 --
drivers/acpi/arm64/mpam.c | 87 +-
drivers/base/cacheinfo.c | 38 +
fs/resctrlfs.c | 396 +++--
include/linux/arm_mpam.h | 118 ++
include/linux/cacheinfo.h | 36 +
include/linux/resctrlfs.h | 30 -
23 files changed, 7521 insertions(+), 2972 deletions(-)
delete mode 100644 arch/arm64/include/asm/mpam_resource.h
delete mode 100644 arch/arm64/kernel/mpam.c
create mode 100644 arch/arm64/kernel/mpam/Makefile
create mode 100644 arch/arm64/kernel/mpam/mpam_ctrlmon.c
create mode 100644 arch/arm64/kernel/mpam/mpam_device.c
create mode 100644 arch/arm64/kernel/mpam/mpam_device.h
create mode 100644 arch/arm64/kernel/mpam/mpam_internal.h
create mode 100644 arch/arm64/kernel/mpam/mpam_mon.c
create mode 100644 arch/arm64/kernel/mpam/mpam_resctrl.c
create mode 100644 arch/arm64/kernel/mpam/mpam_resource.h
create mode 100644 arch/arm64/kernel/mpam/mpam_setup.c
delete mode 100644 arch/arm64/kernel/mpam_ctrlmon.c
delete mode 100644 arch/arm64/kernel/mpam_mon.c
create mode 100644 include/linux/arm_mpam.h
--
2.25.1
1
65
23 Feb '21
From: Li ZhiGang <lizhigang(a)kylinos.cn>
Nationz Tech TCM are used for trusted computing, the chip attached via SPI or LPC.
We have a brief verify/test with this driver on KunPeng920 + openEuler system, with externally compiled module.
Signed-off-by: Li ZhiGang <lizhigang(a)kylinos.cn>
---
drivers/staging/Kconfig | 2 +
drivers/staging/Makefile | 2 +
drivers/staging/gmjstcm/Kconfig | 21 +
drivers/staging/gmjstcm/Makefile | 5 +
drivers/staging/gmjstcm/tcm.c | 949 ++++++++++++++++++++++++++
drivers/staging/gmjstcm/tcm.h | 123 ++++
drivers/staging/gmjstcm/tcm_tis_spi.c | 868 +++++++++++++++++++++++
7 files changed, 1970 insertions(+)
create mode 100644 drivers/staging/gmjstcm/Kconfig
create mode 100644 drivers/staging/gmjstcm/Makefile
create mode 100644 drivers/staging/gmjstcm/tcm.c
create mode 100644 drivers/staging/gmjstcm/tcm.h
create mode 100644 drivers/staging/gmjstcm/tcm_tis_spi.c
diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig
index 1abf76be2aa8..d51fa4f4e7ca 100644
--- a/drivers/staging/Kconfig
+++ b/drivers/staging/Kconfig
@@ -126,4 +126,6 @@ source "drivers/staging/axis-fifo/Kconfig"
source "drivers/staging/erofs/Kconfig"
+source "drivers/staging/gmjstcm/Kconfig"
+
endif # STAGING
diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile
index ab0cbe8815b1..6d41915dad5b 100644
--- a/drivers/staging/Makefile
+++ b/drivers/staging/Makefile
@@ -53,3 +53,5 @@ obj-$(CONFIG_SOC_MT7621) += mt7621-dts/
obj-$(CONFIG_STAGING_GASKET_FRAMEWORK) += gasket/
obj-$(CONFIG_XIL_AXIS_FIFO) += axis-fifo/
obj-$(CONFIG_EROFS_FS) += erofs/
+obj-$(CONFIG_GMJS_TCM) += gmjstcm/
+
diff --git a/drivers/staging/gmjstcm/Kconfig b/drivers/staging/gmjstcm/Kconfig
new file mode 100644
index 000000000000..5b5397ae1832
--- /dev/null
+++ b/drivers/staging/gmjstcm/Kconfig
@@ -0,0 +1,21 @@
+menu "GMJS TCM support"
+
+config GMJS_TCM
+ bool
+
+config GMJS_TCM_CORE
+ tristate "GMJS TCM core support"
+ depends on ARM64 || MIPS
+ default m
+ select GMJS_TCM
+ help
+ GMJS TCM core support.
+
+config GMJS_TCM_SPI
+ tristate "GMJS TCM support on SPI interface"
+ depends on GMJS_TCM_CORE && SPI_MASTER
+ default m
+ help
+ GMJS TCM support on SPI interface.
+
+endmenu
diff --git a/drivers/staging/gmjstcm/Makefile b/drivers/staging/gmjstcm/Makefile
new file mode 100644
index 000000000000..601c78e44793
--- /dev/null
+++ b/drivers/staging/gmjstcm/Makefile
@@ -0,0 +1,5 @@
+
+obj-$(CONFIG_GMJS_TCM_CORE) += tcm_core.o
+tcm_core-objs := tcm.o
+obj-$(CONFIG_GMJS_TCM_SPI) += tcm_tis_spi.o
+
diff --git a/drivers/staging/gmjstcm/tcm.c b/drivers/staging/gmjstcm/tcm.c
new file mode 100644
index 000000000000..5c41bfa8b423
--- /dev/null
+++ b/drivers/staging/gmjstcm/tcm.c
@@ -0,0 +1,949 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2009 Nationz Technologies Inc.
+ *
+ * Description: Exprot symbol for tcm_tis module
+ *
+ * Major Function: public write read register function etc.
+ *
+ */
+
+#include <linux/sched.h>
+#include <linux/poll.h>
+#include <linux/spinlock.h>
+#include <linux/timer.h>
+#include "tcm.h"
+
+/*
+ * const var
+ */
+enum tcm_const {
+ TCM_MINOR = 224, /* officially assigned */
+ TCM_BUFSIZE = 2048, /* Buffer Size */
+ TCM_NUM_DEVICES = 256, /* Max supporting tcm device number */
+};
+
+/*
+ * CMD duration
+ */
+enum tcm_duration {
+ TCM_SHORT = 0,
+ TCM_MEDIUM = 1,
+ TCM_LONG = 2,
+ TCM_UNDEFINED,
+};
+
+/* Max Total of Command Number */
+#define TCM_MAX_ORDINAL 88 /*243*/
+
+static LIST_HEAD(tcm_chip_list);
+static DEFINE_SPINLOCK(driver_lock); /* spin lock */
+static DECLARE_BITMAP(dev_mask, TCM_NUM_DEVICES);
+
+typedef struct tagTCM_Command {
+ u8 ordinal;
+ u8 DURATION;
+} TCM_Command;
+
+static const TCM_Command TCM_Command_List[TCM_MAX_ORDINAL + 1] = {
+ {/*TCM_ORD_ActivateIdentity, */122, 1},
+ {/*TCM_ORD_CertifyKey, */50, 1},
+ {/*TCM_ORD_CertifyKeyM, */51, 1},
+ {/*TCM_ORD_ChangeAuth, */12, 1},
+ {/*TCM_ORD_ChangeAuthOwner, */16, 0},
+ {/*TCM_ORD_ContinueSelfTeSt, */83, 2},
+ {/*TCM_ORD_CreateCounter, */220, 0},
+ {/*TCM_ORD_CreateWrapKey, */31, 2},
+ {/*TCM_ORD_DiSableForceClear, */94, 0},
+ {/*TCM_ORD_DiSableOwnerClear, */92, 0},
+ {/*TCM_ORD_EStabliShTranSport, */230, 0},
+ {/*TCM_ORD_ExecuteTranSport, */231, 2},
+ {/*TCM_ORD_Extend, */20, 0},
+ {/*TCM_ORD_FieldUpgrade, */170, 2},
+ {/*TCM_ORD_FluShSpecific, */186, 0},
+ {/*TCM_ORD_ForceClear, */93, 0},
+ {/*TCM_ORD_GetAuditDigeSt, */133, 0},
+ {/*TCM_ORD_GetAuditDigeStSigned, */134, 1},
+ {/*TCM_ORD_GetCapability, */101, 0},
+ {/*TCM_ORD_GetPubKey, */33, 0},
+ {/*TCM_ORD_GetRandoM, */70, 0},
+ {/*TCM_ORD_GetTeStReSult, */84, 0},
+ {/*TCM_ORD_GetTickS, */241, 0},
+ {/*TCM_ORD_IncreMentCounter, */221, 0},
+ {/*TCM_ORD_LoadContext, */185, 1},
+ {/*TCM_ORD_MakeIdentity, */121, 2},
+ {/*TCM_ORD_NV_DefineSpace, */204, 0},
+ {/*TCM_ORD_NV_ReadValue, */207, 0},
+ {/*TCM_ORD_NV_ReadValueAuth, */208, 0},
+ {/*TCM_ORD_NV_WriteValue, */205, 0},
+ {/*TCM_ORD_NV_WriteValueAuth, */206, 0},
+ {/*TCM_ORD_OwnerClear, */91, 0},
+ {/*TCM_ORD_OwnerReadInternalPub, */129, 0},
+ {/*TCM_ORD_OwnerSetDiSable, */110, 0},
+ {/*TCM_ORD_PCR_ReSet, */200, 0},
+ {/*TCM_ORD_PcrRead, */21, 0},
+ {/*TCM_ORD_PhySicalDiSable, */112, 0},
+ {/*TCM_ORD_PhySicalEnable, */111, 0},
+ {/*TCM_ORD_PhySicalSetDeactivated, */114, 0},
+ {/*TCM_ORD_Quote, */22, 1},
+ {/*TCM_ORD_QuoteM, */62, 1},
+ {/*TCM_ORD_ReadCounter, */222, 0},
+ {/*TCM_ORD_ReadPubek, */124, 0},
+ {/*TCM_ORD_ReleaSeCounter, */223, 0},
+ {/*TCM_ORD_ReleaSeCounterOwner, */224, 0},
+ {/*TCM_ORD_ReleaSeTranSportSigned, */232, 1},
+ {/*TCM_ORD_ReSetLockValue, */64, 0},
+ {/*TCM_ORD_RevokeTruSt, */128, 0},
+ {/*TCM_ORD_SaveContext, */184, 1},
+ {/*TCM_ORD_SaveState, */152, 1},
+ {/*TCM_ORD_Seal, */23, 1},
+ {/*TCM_ORD_Sealx, */61, 1},
+ {/*TCM_ORD_SelfTeStFull, */80, 2},
+ {/*TCM_ORD_SetCapability, */63, 0},
+ {/*TCM_ORD_SetOperatorAuth, */116, 0},
+ {/*TCM_ORD_SetOrdinalAuditStatuS, */141, 0},
+ {/*TCM_ORD_SetOwnerInStall, */113, 0},
+ {/*TCM_ORD_SetTeMpDeactivated, */115, 0},
+ {/*TCM_ORD_Sign, */60, 1},
+ {/*TCM_ORD_Startup, */153, 0},
+ {/*TCM_ORD_TakeOwnerShip, */13, 1},
+ {/*TCM_ORD_TickStaMpBlob, */242, 1},
+ {/*TCM_ORD_UnSeal, */24, 1},
+ {/*TSC_ORD_PhySicalPreSence, */10, 0},
+ {/*TSC_ORD_ReSetEStabliShMentBit, */11, 0},
+ {/*TCM_ORD_WrapKey, */189, 2},
+ {/*TCM_ORD_APcreate, */191, 0},
+ {/*TCM_ORD_APTerMinate, */192, 0},
+ {/*TCM_ORD_CreateMigratedBlob, */193, 1},
+ {/*TCM_ORD_ConvertMigratedBlob, */194, 1},
+ {/*TCM_ORD_AuthorizeMigrationKey, */195, 0},
+ {/*TCM_ORD_SMS4Encrypt, */197, 1},
+ {/*TCM_ORD_SMS4Decrypt, */198, 1},
+ {/*TCM_ORD_ReadEKCert, */199, 1},
+ {/*TCM_ORD_WriteEKCert, */233, 1},
+ {/*TCM_ORD_SCHStart, */234, 0},
+ {/*TCM_ORD_SCHUpdata, */235, 0},
+ {/*TCM_ORD_SCHCoMplete, */236, 0},
+ {/*TCM_ORD_SCHCoMpleteExtend, */237, 0},
+ {/*TCM_ORD_ECCDecrypt, */238, 1},
+ {/*TCM_ORD_LoadKey, */239, 1},
+ {/*TCM_ORD_CreateEndorSeMentKeyPair, */120, 2},
+ {/*TCM_ORD_CreateRevocableEK, */127, 2},
+ {/*TCM_ORD_ReleaSeECCExchangeSeSSion, */174, 1},
+ {/*TCM_ORD_CreateECCExchangeSeSSion, */175, 1},
+ {/*TCM_ORD_GetKeyECCExchangeSeSSion, */176, 1},
+ {/*TCM_ORD_ActivatePEK, */217, 1},
+ {/*TCM_ORD_ActivatePEKCert, */218, 1},
+ {0, 0}
+};
+
+static void user_reader_timeout(struct timer_list *t)
+{
+ struct tcm_chip *chip = from_timer(chip, t, user_read_timer);
+
+ schedule_work(&chip->work);
+}
+
+static void timeout_work(struct work_struct *work)
+{
+ struct tcm_chip *chip = container_of(work, struct tcm_chip, work);
+
+ mutex_lock(&chip->buffer_mutex);
+ atomic_set(&chip->data_pending, 0);
+ memset(chip->data_buffer, 0, TCM_BUFSIZE);
+ mutex_unlock(&chip->buffer_mutex);
+}
+
+unsigned long tcm_calc_ordinal_duration(struct tcm_chip *chip,
+ u32 ordinal)
+{
+ int duration_idx = TCM_UNDEFINED;
+ int duration = 0;
+ int i = 0;
+
+ for (i = 0; i < TCM_MAX_ORDINAL; i++) {
+ if (ordinal == TCM_Command_List[i].ordinal) {
+ duration_idx = TCM_Command_List[i].DURATION;
+ break;
+ }
+ }
+
+ if (duration_idx != TCM_UNDEFINED)
+ duration = chip->vendor.duration[duration_idx];
+ if (duration <= 0)
+ return 2 * 60 * HZ;
+ else
+ return duration;
+}
+EXPORT_SYMBOL_GPL(tcm_calc_ordinal_duration);
+
+/*
+ * Internal kernel interface to transmit TCM commands
+ * buff format: TAG(2 bytes) + Total Size(4 bytes ) +
+ * Command Ordinal(4 bytes ) + ......
+ */
+static ssize_t tcm_transmit(struct tcm_chip *chip, const char *buf,
+ size_t bufsiz)
+{
+ ssize_t rc = 0;
+ u32 count = 0, ordinal = 0;
+ unsigned long stop = 0;
+
+ count = be32_to_cpu(*((__be32 *)(buf + 2))); /* buff size */
+ ordinal = be32_to_cpu(*((__be32 *)(buf + 6))); /* command ordinal */
+
+ if (count == 0)
+ return -ENODATA;
+ if (count > bufsiz) { /* buff size err ,invalid buff stream */
+ dev_err(chip->dev, "invalid count value %x, %zx\n",
+ count, bufsiz);
+ return -E2BIG;
+ }
+
+ mutex_lock(&chip->tcm_mutex); /* enter mutex */
+
+ rc = chip->vendor.send(chip, (u8 *)buf, count);
+ if (rc < 0) {
+ dev_err(chip->dev, "%s: tcm_send: error %zd\n",
+ __func__, rc);
+ goto out;
+ }
+
+ if (chip->vendor.irq)
+ goto out_recv;
+
+ stop = jiffies + tcm_calc_ordinal_duration(chip,
+ ordinal); /* cmd duration */
+ do {
+ u8 status = chip->vendor.status(chip);
+
+ if ((status & chip->vendor.req_complete_mask) ==
+ chip->vendor.req_complete_val)
+ goto out_recv;
+
+ if ((status == chip->vendor.req_canceled)) {
+ dev_err(chip->dev, "Operation Canceled\n");
+ rc = -ECANCELED;
+ goto out;
+ }
+
+ msleep(TCM_TIMEOUT); /* CHECK */
+ rmb();
+ } while (time_before(jiffies, stop));
+ /* time out */
+ chip->vendor.cancel(chip);
+ dev_err(chip->dev, "Operation Timed out\n");
+ rc = -ETIME;
+ goto out;
+
+out_recv:
+ rc = chip->vendor.recv(chip, (u8 *)buf, bufsiz);
+ if (rc < 0)
+ dev_err(chip->dev, "%s: tcm_recv: error %zd\n",
+ __func__, rc);
+out:
+ mutex_unlock(&chip->tcm_mutex);
+ return rc;
+}
+
+#define TCM_DIGEST_SIZE 32
+#define TCM_ERROR_SIZE 10
+#define TCM_RET_CODE_IDX 6
+#define TCM_GET_CAP_RET_SIZE_IDX 10
+#define TCM_GET_CAP_RET_UINT32_1_IDX 14
+#define TCM_GET_CAP_RET_UINT32_2_IDX 18
+#define TCM_GET_CAP_RET_UINT32_3_IDX 22
+#define TCM_GET_CAP_RET_UINT32_4_IDX 26
+#define TCM_GET_CAP_PERM_DISABLE_IDX 16
+#define TCM_GET_CAP_PERM_INACTIVE_IDX 18
+#define TCM_GET_CAP_RET_BOOL_1_IDX 14
+#define TCM_GET_CAP_TEMP_INACTIVE_IDX 16
+
+#define TCM_CAP_IDX 13
+#define TCM_CAP_SUBCAP_IDX 21
+
+enum tcm_capabilities {
+ TCM_CAP_FLAG = 4,
+ TCM_CAP_PROP = 5,
+};
+
+enum tcm_sub_capabilities {
+ TCM_CAP_PROP_PCR = 0x1, /* tcm 0x101 */
+ TCM_CAP_PROP_MANUFACTURER = 0x3, /* tcm 0x103 */
+ TCM_CAP_FLAG_PERM = 0x8, /* tcm 0x108 */
+ TCM_CAP_FLAG_VOL = 0x9, /* tcm 0x109 */
+ TCM_CAP_PROP_OWNER = 0x11, /* tcm 0x101 */
+ TCM_CAP_PROP_TIS_TIMEOUT = 0x15, /* tcm 0x115 */
+ TCM_CAP_PROP_TIS_DURATION = 0x20, /* tcm 0x120 */
+};
+
+/*
+ * This is a semi generic GetCapability command for use
+ * with the capability type TCM_CAP_PROP or TCM_CAP_FLAG
+ * and their associated sub_capabilities.
+ */
+
+static const u8 tcm_cap[] = {
+ 0, 193, /* TCM_TAG_RQU_COMMAND 0xc1*/
+ 0, 0, 0, 22, /* length */
+ 0, 0, 128, 101, /* TCM_ORD_GetCapability */
+ 0, 0, 0, 0, /* TCM_CAP_<TYPE> */
+ 0, 0, 0, 4, /* TCM_CAP_SUB_<TYPE> size */
+ 0, 0, 1, 0 /* TCM_CAP_SUB_<TYPE> */
+};
+
+static ssize_t transmit_cmd(struct tcm_chip *chip, u8 *data, int len,
+ char *desc)
+{
+ int err = 0;
+
+ len = tcm_transmit(chip, data, len);
+ if (len < 0)
+ return len;
+ if (len == TCM_ERROR_SIZE) {
+ err = be32_to_cpu(*((__be32 *)(data + TCM_RET_CODE_IDX)));
+ dev_dbg(chip->dev, "A TCM error (%d) occurred %s\n", err, desc);
+ return err;
+ }
+ return 0;
+}
+
+/*
+ * Get default timeouts value form tcm by GetCapability with TCM_CAP_PROP_TIS_TIMEOUT prop
+ */
+void tcm_get_timeouts(struct tcm_chip *chip)
+{
+ u8 data[max_t(int, ARRAY_SIZE(tcm_cap), 30)];
+ ssize_t rc = 0;
+ u32 timeout = 0;
+
+ memcpy(data, tcm_cap, sizeof(tcm_cap));
+ data[TCM_CAP_IDX] = TCM_CAP_PROP;
+ data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_PROP_TIS_TIMEOUT;
+
+ rc = transmit_cmd(chip, data, sizeof(data),
+ "attempting to determine the timeouts");
+ if (rc)
+ goto duration;
+
+ if (be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_SIZE_IDX))) !=
+ 4 * sizeof(u32))
+ goto duration;
+
+ /* Don't overwrite default if value is 0 */
+ timeout = be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_UINT32_1_IDX)));
+ if (timeout)
+ chip->vendor.timeout_a = msecs_to_jiffies(timeout);
+ timeout = be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_UINT32_2_IDX)));
+ if (timeout)
+ chip->vendor.timeout_b = msecs_to_jiffies(timeout);
+ timeout = be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_UINT32_3_IDX)));
+ if (timeout)
+ chip->vendor.timeout_c = msecs_to_jiffies(timeout);
+ timeout = be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_UINT32_4_IDX)));
+ if (timeout)
+ chip->vendor.timeout_d = msecs_to_jiffies(timeout);
+
+duration:
+ memcpy(data, tcm_cap, sizeof(tcm_cap));
+ data[TCM_CAP_IDX] = TCM_CAP_PROP;
+ data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_PROP_TIS_DURATION;
+
+ rc = transmit_cmd(chip, data, sizeof(data),
+ "attempting to determine the durations");
+ if (rc)
+ return;
+
+ if (be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_SIZE_IDX))) !=
+ 3 * sizeof(u32))
+ return;
+
+ chip->vendor.duration[TCM_SHORT] =
+ msecs_to_jiffies(be32_to_cpu(*((__be32 *)(data +
+ TCM_GET_CAP_RET_UINT32_1_IDX))));
+ chip->vendor.duration[TCM_MEDIUM] =
+ msecs_to_jiffies(be32_to_cpu(*((__be32 *)(data +
+ TCM_GET_CAP_RET_UINT32_2_IDX))));
+ chip->vendor.duration[TCM_LONG] =
+ msecs_to_jiffies(be32_to_cpu(*((__be32 *)(data +
+ TCM_GET_CAP_RET_UINT32_3_IDX))));
+}
+EXPORT_SYMBOL_GPL(tcm_get_timeouts);
+
+ssize_t tcm_show_enabled(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ u8 data[max_t(int, ARRAY_SIZE(tcm_cap), 35)];
+ ssize_t rc = 0;
+ struct tcm_chip *chip = dev_get_drvdata(dev);
+
+ if (chip == NULL)
+ return -ENODEV;
+
+ memcpy(data, tcm_cap, sizeof(tcm_cap));
+ data[TCM_CAP_IDX] = TCM_CAP_FLAG;
+ data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_FLAG_PERM;
+
+ rc = transmit_cmd(chip, data, sizeof(data),
+ "attemtping to determine the permanent state");
+ if (rc)
+ return 0;
+ if (data[TCM_GET_CAP_PERM_DISABLE_IDX])
+ return sprintf(buf, "disable\n");
+ else
+ return sprintf(buf, "enable\n");
+}
+EXPORT_SYMBOL_GPL(tcm_show_enabled);
+
+ssize_t tcm_show_active(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ u8 data[max_t(int, ARRAY_SIZE(tcm_cap), 35)];
+ ssize_t rc = 0;
+ struct tcm_chip *chip = dev_get_drvdata(dev);
+
+ if (chip == NULL)
+ return -ENODEV;
+
+ memcpy(data, tcm_cap, sizeof(tcm_cap));
+ data[TCM_CAP_IDX] = TCM_CAP_FLAG;
+ data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_FLAG_PERM;
+
+ rc = transmit_cmd(chip, data, sizeof(data),
+ "attemtping to determine the permanent state");
+ if (rc)
+ return 0;
+ if (data[TCM_GET_CAP_PERM_INACTIVE_IDX])
+ return sprintf(buf, "deactivated\n");
+ else
+ return sprintf(buf, "activated\n");
+}
+EXPORT_SYMBOL_GPL(tcm_show_active);
+
+ssize_t tcm_show_owned(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ u8 data[sizeof(tcm_cap)];
+ ssize_t rc = 0;
+ struct tcm_chip *chip = dev_get_drvdata(dev);
+
+ if (chip == NULL)
+ return -ENODEV;
+
+ memcpy(data, tcm_cap, sizeof(tcm_cap));
+ data[TCM_CAP_IDX] = TCM_CAP_PROP;
+ data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_PROP_OWNER;
+
+ rc = transmit_cmd(chip, data, sizeof(data),
+ "attempting to determine the owner state");
+ if (rc)
+ return 0;
+ if (data[TCM_GET_CAP_RET_BOOL_1_IDX])
+ return sprintf(buf, "Owner installed\n");
+ else
+ return sprintf(buf, "Owner have not installed\n");
+}
+EXPORT_SYMBOL_GPL(tcm_show_owned);
+
+ssize_t tcm_show_temp_deactivated(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ u8 data[sizeof(tcm_cap)];
+ ssize_t rc = 0;
+ struct tcm_chip *chip = dev_get_drvdata(dev);
+
+ if (chip == NULL)
+ return -ENODEV;
+
+ memcpy(data, tcm_cap, sizeof(tcm_cap));
+ data[TCM_CAP_IDX] = TCM_CAP_FLAG;
+ data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_FLAG_VOL;
+
+ rc = transmit_cmd(chip, data, sizeof(data),
+ "attempting to determine the temporary state");
+ if (rc)
+ return 0;
+ if (data[TCM_GET_CAP_TEMP_INACTIVE_IDX])
+ return sprintf(buf, "Temp deactivated\n");
+ else
+ return sprintf(buf, "activated\n");
+}
+EXPORT_SYMBOL_GPL(tcm_show_temp_deactivated);
+
+static const u8 pcrread[] = {
+ 0, 193, /* TCM_TAG_RQU_COMMAND */
+ 0, 0, 0, 14, /* length */
+ 0, 0, 128, 21, /* TCM_ORD_PcrRead */
+ 0, 0, 0, 0 /* PCR index */
+};
+
+ssize_t tcm_show_pcrs(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ u8 data[1024];
+ ssize_t rc = 0;
+ int i = 0, j = 0, num_pcrs = 0;
+ __be32 index = 0;
+ char *str = buf;
+ struct tcm_chip *chip = dev_get_drvdata(dev);
+
+ if (chip == NULL)
+ return -ENODEV;
+
+ memcpy(data, tcm_cap, sizeof(tcm_cap));
+ data[TCM_CAP_IDX] = TCM_CAP_PROP;
+ data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_PROP_PCR;
+
+ rc = transmit_cmd(chip, data, sizeof(data),
+ "attempting to determine the number of PCRS");
+ if (rc)
+ return 0;
+
+ num_pcrs = be32_to_cpu(*((__be32 *)(data + 14)));
+ for (i = 0; i < num_pcrs; i++) {
+ memcpy(data, pcrread, sizeof(pcrread));
+ index = cpu_to_be32(i);
+ memcpy(data + 10, &index, 4);
+ rc = transmit_cmd(chip, data, sizeof(data),
+ "attempting to read a PCR");
+ if (rc)
+ goto out;
+ str += sprintf(str, "PCR-%02d: ", i);
+ for (j = 0; j < TCM_DIGEST_SIZE; j++)
+ str += sprintf(str, "%02X ", *(data + 10 + j));
+ str += sprintf(str, "\n");
+ memset(data, 0, 1024);
+ }
+out:
+ return str - buf;
+}
+EXPORT_SYMBOL_GPL(tcm_show_pcrs);
+
+#define READ_PUBEK_RESULT_SIZE 128
+static const u8 readpubek[] = {
+ 0, 193, /* TCM_TAG_RQU_COMMAND */
+ 0, 0, 0, 42, /* length */
+ 0, 0, 128, 124, /* TCM_ORD_ReadPubek */
+ 0, 0, 0, 0, 0, 0, 0, 0, /* NONCE */
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0
+};
+
+ssize_t tcm_show_pubek(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ u8 data[READ_PUBEK_RESULT_SIZE] = {0};
+ ssize_t err = 0;
+ int i = 0, rc = 0;
+ char *str = buf;
+ struct tcm_chip *chip = dev_get_drvdata(dev);
+
+ if (chip == NULL)
+ return -ENODEV;
+
+ memcpy(data, readpubek, sizeof(readpubek));
+
+ err = transmit_cmd(chip, data, sizeof(data),
+ "attempting to read the PUBEK");
+ if (err)
+ goto out;
+
+ str += sprintf(str, "PUBEK:");
+ for (i = 0 ; i < 65 ; i++) {
+ if ((i) % 16 == 0)
+ str += sprintf(str, "\n");
+ str += sprintf(str, "%02X ", data[i+10]);
+ }
+
+ str += sprintf(str, "\n");
+out:
+ rc = str - buf;
+ return rc;
+}
+EXPORT_SYMBOL_GPL(tcm_show_pubek);
+
+#define CAP_VERSION_1_1 6
+#define CAP_VERSION_1_2 0x1A
+#define CAP_VERSION_IDX 13
+static const u8 cap_version[] = {
+ 0, 193, /* TCM_TAG_RQU_COMMAND */
+ 0, 0, 0, 18, /* length */
+ 0, 0, 128, 101, /* TCM_ORD_GetCapability */
+ 0, 0, 0, 0,
+ 0, 0, 0, 0
+};
+
+ssize_t tcm_show_caps(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ u8 data[max_t(int, max(ARRAY_SIZE(tcm_cap), ARRAY_SIZE(cap_version)), 30)];
+ ssize_t rc = 0;
+ char *str = buf;
+ struct tcm_chip *chip = dev_get_drvdata(dev);
+
+ if (chip == NULL)
+ return -ENODEV;
+
+ memcpy(data, tcm_cap, sizeof(tcm_cap));
+ data[TCM_CAP_IDX] = TCM_CAP_PROP;
+ data[TCM_CAP_SUBCAP_IDX] = TCM_CAP_PROP_MANUFACTURER;
+
+ rc = transmit_cmd(chip, data, sizeof(data),
+ "attempting to determine the manufacturer");
+ if (rc)
+ return 0;
+
+ str += sprintf(str, "Manufacturer: 0x%x\n",
+ be32_to_cpu(*((__be32 *)(data + TCM_GET_CAP_RET_UINT32_1_IDX))));
+
+ memcpy(data, cap_version, sizeof(cap_version));
+ data[CAP_VERSION_IDX] = CAP_VERSION_1_1;
+ rc = transmit_cmd(chip, data, sizeof(data),
+ "attempting to determine the 1.1 version");
+ if (rc)
+ goto out;
+
+ str += sprintf(str, "Firmware version: %02X.%02X.%02X.%02X\n",
+ (int)data[14], (int)data[15], (int)data[16],
+ (int)data[17]);
+
+out:
+ return str - buf;
+}
+EXPORT_SYMBOL_GPL(tcm_show_caps);
+
+ssize_t tcm_store_cancel(struct device *dev, struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ struct tcm_chip *chip = dev_get_drvdata(dev);
+
+ if (chip == NULL)
+ return 0;
+
+ chip->vendor.cancel(chip);
+ return count;
+}
+EXPORT_SYMBOL_GPL(tcm_store_cancel);
+
+/*
+ * Device file system interface to the TCM
+ * when App call file open in usr space ,this func will respone
+ */
+int tcm_open(struct inode *inode, struct file *file)
+{
+ int rc = 0, minor = iminor(inode);
+ struct tcm_chip *chip = NULL, *pos = NULL;
+
+ spin_lock(&driver_lock);
+
+ list_for_each_entry(pos, &tcm_chip_list, list) {
+ if (pos->vendor.miscdev.minor == minor) {
+ chip = pos;
+ break;
+ }
+ }
+
+ if (chip == NULL) {
+ rc = -ENODEV;
+ goto err_out;
+ }
+
+ if (chip->num_opens) {
+ dev_dbg(chip->dev, "Another process owns this TCM\n");
+ rc = -EBUSY;
+ goto err_out;
+ }
+
+ chip->num_opens++;
+ get_device(chip->dev);
+
+ spin_unlock(&driver_lock);
+
+ chip->data_buffer = kmalloc(TCM_BUFSIZE * sizeof(u8), GFP_KERNEL);
+ if (chip->data_buffer == NULL) {
+ chip->num_opens--;
+ put_device(chip->dev);
+ return -ENOMEM;
+ }
+
+ atomic_set(&chip->data_pending, 0);
+
+ file->private_data = chip;
+ return 0;
+
+err_out:
+ spin_unlock(&driver_lock);
+ return rc;
+}
+EXPORT_SYMBOL_GPL(tcm_open);
+
+int tcm_release(struct inode *inode, struct file *file)
+{
+ struct tcm_chip *chip = file->private_data;
+
+ spin_lock(&driver_lock);
+ file->private_data = NULL;
+ chip->num_opens--;
+ del_singleshot_timer_sync(&chip->user_read_timer);
+ flush_work(&chip->work);
+ atomic_set(&chip->data_pending, 0);
+ put_device(chip->dev);
+ kfree(chip->data_buffer);
+ spin_unlock(&driver_lock);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(tcm_release);
+
+ssize_t tcm_write(struct file *file, const char __user *buf,
+ size_t size, loff_t *off)
+{
+ struct tcm_chip *chip = file->private_data;
+ int in_size = size, out_size;
+
+ /*
+ * cannot perform a write until the read has cleared
+ * either via tcm_read or a user_read_timer timeout
+ */
+ while (atomic_read(&chip->data_pending) != 0)
+ msleep(TCM_TIMEOUT);
+
+ mutex_lock(&chip->buffer_mutex);
+
+ if (in_size > TCM_BUFSIZE)
+ in_size = TCM_BUFSIZE;
+
+ if (copy_from_user(chip->data_buffer, (void __user *)buf, in_size)) {
+ mutex_unlock(&chip->buffer_mutex);
+ return -EFAULT;
+ }
+
+ /* atomic tcm command send and result receive */
+ out_size = tcm_transmit(chip, chip->data_buffer, TCM_BUFSIZE);
+
+ if (out_size >= 0) {
+ atomic_set(&chip->data_pending, out_size);
+ mutex_unlock(&chip->buffer_mutex);
+
+ /* Set a timeout by which the reader must come claim the result */
+ mod_timer(&chip->user_read_timer, jiffies + (60 * HZ));
+ } else
+ mutex_unlock(&chip->buffer_mutex);
+
+ return in_size;
+}
+EXPORT_SYMBOL_GPL(tcm_write);
+
+ssize_t tcm_read(struct file *file, char __user *buf,
+ size_t size, loff_t *off)
+{
+ struct tcm_chip *chip = file->private_data;
+ int ret_size = 0;
+
+ del_singleshot_timer_sync(&chip->user_read_timer);
+ flush_work(&chip->work);
+ ret_size = atomic_read(&chip->data_pending);
+ atomic_set(&chip->data_pending, 0);
+ if (ret_size > 0) { /* relay data */
+ if (size < ret_size)
+ ret_size = size;
+
+ mutex_lock(&chip->buffer_mutex);
+ if (copy_to_user(buf, chip->data_buffer, ret_size))
+ ret_size = -EFAULT;
+ mutex_unlock(&chip->buffer_mutex);
+ }
+
+ return ret_size;
+}
+EXPORT_SYMBOL_GPL(tcm_read);
+
+void tcm_remove_hardware(struct device *dev)
+{
+ struct tcm_chip *chip = dev_get_drvdata(dev);
+
+ if (chip == NULL) {
+ dev_err(dev, "No device data found\n");
+ return;
+ }
+
+ spin_lock(&driver_lock);
+ list_del(&chip->list);
+ spin_unlock(&driver_lock);
+
+ dev_set_drvdata(dev, NULL);
+ misc_deregister(&chip->vendor.miscdev);
+ kfree(chip->vendor.miscdev.name);
+
+ sysfs_remove_group(&dev->kobj, chip->vendor.attr_group);
+ /* tcm_bios_log_teardown(chip->bios_dir); */
+
+ clear_bit(chip->dev_num, dev_mask);
+ kfree(chip);
+ put_device(dev);
+}
+EXPORT_SYMBOL_GPL(tcm_remove_hardware);
+
+static u8 savestate[] = {
+ 0, 193, /* TCM_TAG_RQU_COMMAND */
+ 0, 0, 0, 10, /* blob length (in bytes) */
+ 0, 0, 128, 152 /* TCM_ORD_SaveState */
+};
+
+/*
+ * We are about to suspend. Save the TCM state
+ * so that it can be restored.
+ */
+int tcm_pm_suspend(struct device *dev, pm_message_t pm_state)
+{
+ struct tcm_chip *chip = dev_get_drvdata(dev);
+
+ if (chip == NULL)
+ return -ENODEV;
+
+ tcm_transmit(chip, savestate, sizeof(savestate));
+ return 0;
+}
+EXPORT_SYMBOL_GPL(tcm_pm_suspend);
+
+int tcm_pm_suspend_p(struct device *dev)
+{
+ struct tcm_chip *chip = dev_get_drvdata(dev);
+
+ if (chip == NULL)
+ return -ENODEV;
+
+ tcm_transmit(chip, savestate, sizeof(savestate));
+ return 0;
+}
+EXPORT_SYMBOL_GPL(tcm_pm_suspend_p);
+
+void tcm_startup(struct tcm_chip *chip)
+{
+ u8 start_up[] = {
+ 0, 193, /* TCM_TAG_RQU_COMMAND */
+ 0, 0, 0, 12, /* blob length (in bytes) */
+ 0, 0, 128, 153, /* TCM_ORD_SaveState */
+ 0, 1
+ };
+ if (chip == NULL)
+ return;
+ tcm_transmit(chip, start_up, sizeof(start_up));
+}
+EXPORT_SYMBOL_GPL(tcm_startup);
+
+/*
+ * Resume from a power safe. The BIOS already restored
+ * the TCM state.
+ */
+int tcm_pm_resume(struct device *dev)
+{
+ u8 start_up[] = {
+ 0, 193, /* TCM_TAG_RQU_COMMAND */
+ 0, 0, 0, 12, /* blob length (in bytes) */
+ 0, 0, 128, 153, /* TCM_ORD_SaveState */
+ 0, 1
+ };
+ struct tcm_chip *chip = dev_get_drvdata(dev);
+ /* dev_info(chip->dev ,"--call tcm_pm_resume\n"); */
+ if (chip == NULL)
+ return -ENODEV;
+
+ tcm_transmit(chip, start_up, sizeof(start_up));
+ return 0;
+}
+EXPORT_SYMBOL_GPL(tcm_pm_resume);
+
+/*
+ * Called from tcm_<specific>.c probe function only for devices
+ * the driver has determined it should claim. Prior to calling
+ * this function the specific probe function has called pci_enable_device
+ * upon errant exit from this function specific probe function should call
+ * pci_disable_device
+ */
+struct tcm_chip *tcm_register_hardware(struct device *dev,
+ const struct tcm_vendor_specific *entry)
+{
+ int rc;
+#define DEVNAME_SIZE 7
+
+ char *devname = NULL;
+ struct tcm_chip *chip = NULL;
+
+ /* Driver specific per-device data */
+ chip = kzalloc(sizeof(*chip), GFP_KERNEL);
+ if (chip == NULL) {
+ dev_err(dev, "chip kzalloc err\n");
+ return NULL;
+ }
+
+ mutex_init(&chip->buffer_mutex);
+ mutex_init(&chip->tcm_mutex);
+ INIT_LIST_HEAD(&chip->list);
+
+ INIT_WORK(&chip->work, timeout_work);
+ timer_setup(&chip->user_read_timer, user_reader_timeout, 0);
+
+ memcpy(&chip->vendor, entry, sizeof(struct tcm_vendor_specific));
+
+ chip->dev_num = find_first_zero_bit(dev_mask, TCM_NUM_DEVICES);
+
+ if (chip->dev_num >= TCM_NUM_DEVICES) {
+ dev_err(dev, "No available tcm device numbers\n");
+ kfree(chip);
+ return NULL;
+ } else if (chip->dev_num == 0)
+ chip->vendor.miscdev.minor = TCM_MINOR;
+ else
+ chip->vendor.miscdev.minor = MISC_DYNAMIC_MINOR;
+
+ set_bit(chip->dev_num, dev_mask);
+
+ devname = kmalloc(DEVNAME_SIZE, GFP_KERNEL);
+ scnprintf(devname, DEVNAME_SIZE, "%s%d", "tcm", chip->dev_num);
+ chip->vendor.miscdev.name = devname;
+
+ /* chip->vendor.miscdev.dev = dev; */
+
+ chip->dev = get_device(dev);
+
+ if (misc_register(&chip->vendor.miscdev)) {
+ dev_err(chip->dev,
+ "unable to misc_register %s, minor %d\n",
+ chip->vendor.miscdev.name,
+ chip->vendor.miscdev.minor);
+ put_device(dev);
+ clear_bit(chip->dev_num, dev_mask);
+ kfree(chip);
+ kfree(devname);
+ return NULL;
+ }
+
+ spin_lock(&driver_lock);
+ dev_set_drvdata(dev, chip);
+ list_add(&chip->list, &tcm_chip_list);
+ spin_unlock(&driver_lock);
+
+ rc = sysfs_create_group(&dev->kobj, chip->vendor.attr_group);
+ /* chip->bios_dir = tcm_bios_log_setup(devname); */
+
+ return chip;
+}
+EXPORT_SYMBOL_GPL(tcm_register_hardware);
+
+static int __init tcm_init_module(void)
+{
+ return 0;
+}
+
+static void __exit tcm_exit_module(void)
+{
+}
+
+module_init(tcm_init_module);
+module_exit(tcm_exit_module);
+
+MODULE_AUTHOR("Nationz Technologies Inc.");
+MODULE_DESCRIPTION("TCM Driver");
+MODULE_VERSION("1.1.1.0");
+MODULE_LICENSE("GPL");
diff --git a/drivers/staging/gmjstcm/tcm.h b/drivers/staging/gmjstcm/tcm.h
new file mode 100644
index 000000000000..b8cafe78d590
--- /dev/null
+++ b/drivers/staging/gmjstcm/tcm.h
@@ -0,0 +1,123 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2009 Nationz Technologies Inc.
+ *
+ */
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/fs.h>
+#include <linux/miscdevice.h>
+#include <linux/platform_device.h>
+#include <linux/io.h>
+
+struct device;
+struct tcm_chip;
+
+enum tcm_timeout {
+ TCM_TIMEOUT = 5,
+};
+
+/* TCM addresses */
+enum tcm_addr {
+ TCM_SUPERIO_ADDR = 0x2E,
+ TCM_ADDR = 0x4E,
+};
+
+extern ssize_t tcm_show_pubek(struct device *, struct device_attribute *attr,
+ char *);
+extern ssize_t tcm_show_pcrs(struct device *, struct device_attribute *attr,
+ char *);
+extern ssize_t tcm_show_caps(struct device *, struct device_attribute *attr,
+ char *);
+extern ssize_t tcm_store_cancel(struct device *, struct device_attribute *attr,
+ const char *, size_t);
+extern ssize_t tcm_show_enabled(struct device *, struct device_attribute *attr,
+ char *);
+extern ssize_t tcm_show_active(struct device *, struct device_attribute *attr,
+ char *);
+extern ssize_t tcm_show_owned(struct device *, struct device_attribute *attr,
+ char *);
+extern ssize_t tcm_show_temp_deactivated(struct device *,
+ struct device_attribute *attr, char *);
+
+struct tcm_vendor_specific {
+ const u8 req_complete_mask;
+ const u8 req_complete_val;
+ const u8 req_canceled;
+ void __iomem *iobase; /* ioremapped address */
+ void __iomem *iolbc;
+ unsigned long base; /* TCM base address */
+
+ int irq;
+
+ int region_size;
+ int have_region;
+
+ int (*recv)(struct tcm_chip *, u8 *, size_t);
+ int (*send)(struct tcm_chip *, u8 *, size_t);
+ void (*cancel)(struct tcm_chip *);
+ u8 (*status)(struct tcm_chip *);
+ struct miscdevice miscdev;
+ struct attribute_group *attr_group;
+ struct list_head list;
+ int locality;
+ unsigned long timeout_a, timeout_b, timeout_c, timeout_d; /* jiffies */
+ unsigned long duration[3]; /* jiffies */
+
+ wait_queue_head_t read_queue;
+ wait_queue_head_t int_queue;
+};
+
+struct tcm_chip {
+ struct device *dev; /* Device stuff */
+
+ int dev_num; /* /dev/tcm# */
+ int num_opens; /* only one allowed */
+ int time_expired;
+
+ /* Data passed to and from the tcm via the read/write calls */
+ u8 *data_buffer;
+ atomic_t data_pending;
+ struct mutex buffer_mutex;
+
+ struct timer_list user_read_timer; /* user needs to claim result */
+ struct work_struct work;
+ struct mutex tcm_mutex; /* tcm is processing */
+
+ struct tcm_vendor_specific vendor;
+
+ struct dentry **bios_dir;
+
+ struct list_head list;
+};
+
+#define to_tcm_chip(n) container_of(n, struct tcm_chip, vendor)
+
+static inline int tcm_read_index(int base, int index)
+{
+ outb(index, base);
+ return inb(base+1) & 0xFF;
+}
+
+static inline void tcm_write_index(int base, int index, int value)
+{
+ outb(index, base);
+ outb(value & 0xFF, base+1);
+}
+extern void tcm_startup(struct tcm_chip *);
+extern void tcm_get_timeouts(struct tcm_chip *);
+extern unsigned long tcm_calc_ordinal_duration(struct tcm_chip *, u32);
+extern struct tcm_chip *tcm_register_hardware(struct device *,
+ const struct tcm_vendor_specific *);
+extern int tcm_open(struct inode *, struct file *);
+extern int tcm_release(struct inode *, struct file *);
+extern ssize_t tcm_write(struct file *, const char __user *, size_t,
+ loff_t *);
+extern ssize_t tcm_read(struct file *, char __user *, size_t, loff_t *);
+extern void tcm_remove_hardware(struct device *);
+extern int tcm_pm_suspend(struct device *, pm_message_t);
+extern int tcm_pm_suspend_p(struct device *);
+extern int tcm_pm_resume(struct device *);
+
diff --git a/drivers/staging/gmjstcm/tcm_tis_spi.c b/drivers/staging/gmjstcm/tcm_tis_spi.c
new file mode 100644
index 000000000000..db30a5b4c47d
--- /dev/null
+++ b/drivers/staging/gmjstcm/tcm_tis_spi.c
@@ -0,0 +1,868 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2020 Kylin Tech. Co., Ltd.
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/interrupt.h>
+#include <linux/wait.h>
+#include <linux/acpi.h>
+#include <linux/spi/spi.h>
+
+#include "tcm.h"
+
+#if !defined(CONFIG_KYLINOS_SERVER) && !defined(CONFIG_KYLINOS_DESKTOP)
+static int is_ft_all(void) {
+ return 0;
+}
+#endif
+
+#define TCM_HEADER_SIZE 10
+
+static bool tcm_debug;
+module_param_named(debug, tcm_debug, bool, 0600);
+MODULE_PARM_DESC(debug, "Turn TCM debugging mode on and off");
+
+#define tcm_dbg(fmt, args...) \
+{ \
+ if (tcm_debug) \
+ pr_err(fmt, ## args); \
+}
+
+enum tis_access {
+ TCM_ACCESS_VALID = 0x80,
+ TCM_ACCESS_ACTIVE_LOCALITY = 0x20,
+ TCM_ACCESS_REQUEST_PENDING = 0x04,
+ TCM_ACCESS_REQUEST_USE = 0x02,
+};
+
+enum tis_status {
+ TCM_STS_VALID = 0x80,
+ TCM_STS_COMMAND_READY = 0x40,
+ TCM_STS_GO = 0x20,
+ TCM_STS_DATA_AVAIL = 0x10,
+ TCM_STS_DATA_EXPECT = 0x08,
+};
+
+enum tis_int_flags {
+ TCM_GLOBAL_INT_ENABLE = 0x80000000,
+ TCM_INTF_BURST_COUNT_STATIC = 0x100,
+ TCM_INTF_CMD_READY_INT = 0x080,
+ TCM_INTF_INT_EDGE_FALLING = 0x040,
+ TCM_INTF_INT_EDGE_RISING = 0x020,
+ TCM_INTF_INT_LEVEL_LOW = 0x010,
+ TCM_INTF_INT_LEVEL_HIGH = 0x008,
+ TCM_INTF_LOCALITY_CHANGE_INT = 0x004,
+ TCM_INTF_STS_VALID_INT = 0x002,
+ TCM_INTF_DATA_AVAIL_INT = 0x001,
+};
+
+enum tis_defaults {
+ TIS_SHORT_TIMEOUT = 750, /* ms */
+ TIS_LONG_TIMEOUT = 2000, /* 2 sec */
+};
+
+#define TCM_ACCESS(l) (0x0000 | ((l) << 12))
+#define TCM_INT_ENABLE(l) (0x0008 | ((l) << 12)) /* interperet */
+#define TCM_INT_VECTOR(l) (0x000C | ((l) << 12))
+#define TCM_INT_STATUS(l) (0x0010 | ((l) << 12))
+#define TCM_INTF_CAPS(l) (0x0014 | ((l) << 12))
+#define TCM_STS(l) (0x0018 | ((l) << 12))
+#define TCM_DATA_FIFO(l) (0x0024 | ((l) << 12))
+
+#define TCM_DID_VID(l) (0x0F00 | ((l) << 12))
+#define TCM_RID(l) (0x0F04 | ((l) << 12))
+
+#define TIS_MEM_BASE_huawei 0x3fed40000LL
+
+#define MAX_SPI_FRAMESIZE 64
+
+//
+#define _CPU_FT2000A4
+#define REUSE_CONF_REG_BASE 0x28180208
+#define REUSE_GPIO1_A5_BASE 0x28005000
+
+static void *__iomem reuse_conf_reg;
+static void *__iomem gpio1_a5;
+
+//
+static LIST_HEAD(tis_chips);
+static DEFINE_SPINLOCK(tis_lock);
+
+struct chip_data {
+ u8 cs;
+ u8 tmode;
+ u8 type;
+ u8 poll_mode;
+ u16 clk_div;
+ u32 speed_hz;
+ void (*cs_control)(u32 command);
+};
+
+struct tcm_tis_spi_phy {
+ struct spi_device *spi_device;
+ struct completion ready;
+ u8 *iobuf;
+};
+
+int tcm_tis_spi_transfer(struct device *dev, u32 addr, u16 len,
+ u8 *in, const u8 *out)
+{
+ struct tcm_tis_spi_phy *phy = dev_get_drvdata(dev);
+ int ret = 0;
+ struct spi_message m;
+ struct spi_transfer spi_xfer;
+ u8 transfer_len;
+
+ tcm_dbg("TCM-dbg: %s, addr: 0x%x, len: %x, %s\n",
+ __func__, addr, len, (in) ? "in" : "out");
+
+ spi_bus_lock(phy->spi_device->master);
+
+ /* set gpio1_a5 to LOW */
+ if (is_ft_all() && (phy->spi_device->chip_select == 0)) {
+ iowrite32(0x0, gpio1_a5);
+ }
+
+ while (len) {
+ transfer_len = min_t(u16, len, MAX_SPI_FRAMESIZE);
+
+ phy->iobuf[0] = (in ? 0x80 : 0) | (transfer_len - 1);
+ phy->iobuf[1] = 0xd4;
+ phy->iobuf[2] = addr >> 8;
+ phy->iobuf[3] = addr;
+
+ memset(&spi_xfer, 0, sizeof(spi_xfer));
+ spi_xfer.tx_buf = phy->iobuf;
+ spi_xfer.rx_buf = phy->iobuf;
+ spi_xfer.len = 4;
+ spi_xfer.cs_change = 1;
+
+ spi_message_init(&m);
+ spi_message_add_tail(&spi_xfer, &m);
+ ret = spi_sync_locked(phy->spi_device, &m);
+ if (ret < 0)
+ goto exit;
+
+ spi_xfer.cs_change = 0;
+ spi_xfer.len = transfer_len;
+ spi_xfer.delay_usecs = 5;
+
+ if (in) {
+ spi_xfer.tx_buf = NULL;
+ } else if (out) {
+ spi_xfer.rx_buf = NULL;
+ memcpy(phy->iobuf, out, transfer_len);
+ out += transfer_len;
+ }
+
+ spi_message_init(&m);
+ spi_message_add_tail(&spi_xfer, &m);
+ reinit_completion(&phy->ready);
+ ret = spi_sync_locked(phy->spi_device, &m);
+ if (ret < 0)
+ goto exit;
+
+ if (in) {
+ memcpy(in, phy->iobuf, transfer_len);
+ in += transfer_len;
+ }
+
+ len -= transfer_len;
+ }
+
+exit:
+ /* set gpio1_a5 to HIGH */
+ if (is_ft_all() && (phy->spi_device->chip_select == 0)) {
+ iowrite32(0x20, gpio1_a5);
+ }
+
+ spi_bus_unlock(phy->spi_device->master);
+ tcm_dbg("TCM-dbg: ret: %d\n", ret);
+ return ret;
+}
+
+static int tcm_tis_read8(struct device *dev,
+ u32 addr, u16 len, u8 *result)
+{
+ return tcm_tis_spi_transfer(dev, addr, len, result, NULL);
+}
+
+static int tcm_tis_write8(struct device *dev,
+ u32 addr, u16 len, u8 *value)
+{
+ return tcm_tis_spi_transfer(dev, addr, len, NULL, value);
+}
+
+static int tcm_tis_readb(struct device *dev, u32 addr, u8 *value)
+{
+ return tcm_tis_read8(dev, addr, sizeof(u8), value);
+}
+
+static int tcm_tis_writeb(struct device *dev, u32 addr, u8 value)
+{
+ return tcm_tis_write8(dev, addr, sizeof(u8), &value);
+}
+
+static int tcm_tis_readl(struct device *dev, u32 addr, u32 *result)
+{
+ int rc;
+ __le32 result_le;
+
+ rc = tcm_tis_read8(dev, addr, sizeof(u32), (u8 *)&result_le);
+ tcm_dbg("TCM-dbg: result_le: 0x%x\n", result_le);
+ if (!rc)
+ *result = le32_to_cpu(result_le);
+
+ return rc;
+}
+
+static int tcm_tis_writel(struct device *dev, u32 addr, u32 value)
+{
+ int rc;
+ __le32 value_le;
+
+ value_le = cpu_to_le32(value);
+ rc = tcm_tis_write8(dev, addr, sizeof(u32), (u8 *)&value_le);
+
+ return rc;
+}
+
+static int request_locality(struct tcm_chip *chip, int l);
+static void release_locality(struct tcm_chip *chip, int l, int force);
+static void cleanup_tis(void)
+{
+ int ret;
+ u32 inten;
+ struct tcm_vendor_specific *i, *j;
+ struct tcm_chip *chip;
+
+ spin_lock(&tis_lock);
+ list_for_each_entry_safe(i, j, &tis_chips, list) {
+ chip = to_tcm_chip(i);
+ ret = tcm_tis_readl(chip->dev,
+ TCM_INT_ENABLE(chip->vendor.locality), &inten);
+ if (ret < 0)
+ return;
+
+ tcm_tis_writel(chip->dev, TCM_INT_ENABLE(chip->vendor.locality),
+ ~TCM_GLOBAL_INT_ENABLE & inten);
+ release_locality(chip, chip->vendor.locality, 1);
+ }
+ spin_unlock(&tis_lock);
+}
+
+static void tcm_tis_init(struct tcm_chip *chip)
+{
+ int ret;
+ u8 rid;
+ u32 vendor, intfcaps;
+
+ ret = tcm_tis_readl(chip->dev, TCM_DID_VID(0), &vendor);
+
+ if ((vendor & 0xffff) != 0x19f5 && (vendor & 0xffff) != 0x1B4E)
+ pr_info("there is no Nationz TCM on you computer\n");
+
+ ret = tcm_tis_readb(chip->dev, TCM_RID(0), &rid);
+ if (ret < 0)
+ return;
+ pr_info("kylin: 2019-09-21 1.2 TCM (device-id 0x%X, rev-id %d)\n",
+ vendor >> 16, rid);
+
+ /* Figure out the capabilities */
+ ret = tcm_tis_readl(chip->dev,
+ TCM_INTF_CAPS(chip->vendor.locality), &intfcaps);
+ if (ret < 0)
+ return;
+
+ if (request_locality(chip, 0) != 0)
+ pr_err("tcm request_locality err\n");
+
+ atomic_set(&chip->data_pending, 0);
+}
+
+static void tcm_handle_err(struct tcm_chip *chip)
+{
+ cleanup_tis();
+ tcm_tis_init(chip);
+}
+
+static bool check_locality(struct tcm_chip *chip, int l)
+{
+ int ret;
+ u8 access;
+
+ ret = tcm_tis_readb(chip->dev, TCM_ACCESS(l), &access);
+ tcm_dbg("TCM-dbg: access: 0x%x\n", access);
+ if (ret < 0)
+ return false;
+
+ if ((access & (TCM_ACCESS_ACTIVE_LOCALITY | TCM_ACCESS_VALID)) ==
+ (TCM_ACCESS_ACTIVE_LOCALITY | TCM_ACCESS_VALID)) {
+ chip->vendor.locality = l;
+ return true;
+ }
+
+ return false;
+}
+
+static int request_locality(struct tcm_chip *chip, int l)
+{
+ unsigned long stop;
+
+ if (check_locality(chip, l))
+ return l;
+
+ tcm_tis_writeb(chip->dev, TCM_ACCESS(l), TCM_ACCESS_REQUEST_USE);
+
+ /* wait for burstcount */
+ stop = jiffies + chip->vendor.timeout_a;
+ do {
+ if (check_locality(chip, l))
+ return l;
+ msleep(TCM_TIMEOUT);
+ } while (time_before(jiffies, stop));
+
+ return -1;
+}
+
+static void release_locality(struct tcm_chip *chip, int l, int force)
+{
+ int ret;
+ u8 access;
+
+ ret = tcm_tis_readb(chip->dev, TCM_ACCESS(l), &access);
+ if (ret < 0)
+ return;
+ if (force || (access & (TCM_ACCESS_REQUEST_PENDING | TCM_ACCESS_VALID)) ==
+ (TCM_ACCESS_REQUEST_PENDING | TCM_ACCESS_VALID))
+ tcm_tis_writeb(chip->dev,
+ TCM_ACCESS(l), TCM_ACCESS_ACTIVE_LOCALITY);
+}
+
+static u8 tcm_tis_status(struct tcm_chip *chip)
+{
+ int ret;
+ u8 status;
+
+ ret = tcm_tis_readb(chip->dev,
+ TCM_STS(chip->vendor.locality), &status);
+ tcm_dbg("TCM-dbg: status: 0x%x\n", status);
+ if (ret < 0)
+ return 0;
+
+ return status;
+}
+
+static void tcm_tis_ready(struct tcm_chip *chip)
+{
+ /* this causes the current command to be aboreted */
+ tcm_tis_writeb(chip->dev, TCM_STS(chip->vendor.locality),
+ TCM_STS_COMMAND_READY);
+}
+
+static int get_burstcount(struct tcm_chip *chip)
+{
+ int ret;
+ unsigned long stop;
+ u8 tmp, tmp1;
+ int burstcnt = 0;
+
+ /* wait for burstcount */
+ /* which timeout value, spec has 2 answers (c & d) */
+ stop = jiffies + chip->vendor.timeout_d;
+ do {
+ ret = tcm_tis_readb(chip->dev,
+ TCM_STS(chip->vendor.locality) + 1,
+ &tmp);
+ tcm_dbg("TCM-dbg: burstcnt: 0x%x\n", burstcnt);
+ if (ret < 0)
+ return -EINVAL;
+ ret = tcm_tis_readb(chip->dev,
+ (TCM_STS(chip->vendor.locality) + 2),
+ &tmp1);
+ tcm_dbg("TCM-dbg: burstcnt: 0x%x\n", burstcnt);
+ if (ret < 0)
+ return -EINVAL;
+
+ burstcnt = tmp | (tmp1 << 8);
+ if (burstcnt)
+ return burstcnt;
+ msleep(TCM_TIMEOUT);
+ } while (time_before(jiffies, stop));
+
+ return -EBUSY;
+}
+
+static int wait_for_stat(struct tcm_chip *chip, u8 mask,
+ unsigned long timeout,
+ wait_queue_head_t *queue)
+{
+ unsigned long stop;
+ u8 status;
+
+ /* check current status */
+ status = tcm_tis_status(chip);
+ if ((status & mask) == mask)
+ return 0;
+
+ stop = jiffies + timeout;
+ do {
+ msleep(TCM_TIMEOUT);
+ status = tcm_tis_status(chip);
+ if ((status & mask) == mask)
+ return 0;
+ } while (time_before(jiffies, stop));
+
+ return -ETIME;
+}
+
+static int recv_data(struct tcm_chip *chip, u8 *buf, size_t count)
+{
+ int ret;
+ int size = 0, burstcnt;
+
+ while (size < count && wait_for_stat(chip,
+ TCM_STS_DATA_AVAIL | TCM_STS_VALID,
+ chip->vendor.timeout_c,
+ &chip->vendor.read_queue) == 0) {
+ burstcnt = get_burstcount(chip);
+
+ if (burstcnt < 0) {
+ dev_err(chip->dev, "Unable to read burstcount\n");
+ return burstcnt;
+ }
+
+ for (; burstcnt > 0 && size < count; burstcnt--) {
+ ret = tcm_tis_readb(chip->dev,
+ TCM_DATA_FIFO(chip->vendor.locality),
+ &buf[size]);
+ tcm_dbg("TCM-dbg: buf[%d]: 0x%x\n", size, buf[size]);
+ size++;
+ }
+ }
+
+ return size;
+}
+
+static int tcm_tis_recv(struct tcm_chip *chip, u8 *buf, size_t count)
+{
+ int size = 0;
+ int expected, status;
+ unsigned long stop;
+
+ if (count < TCM_HEADER_SIZE) {
+ dev_err(chip->dev, "read size is to small: %d\n", (u32)(count));
+ size = -EIO;
+ goto out;
+ }
+
+ /* read first 10 bytes, including tag, paramsize, and result */
+ size = recv_data(chip, buf, TCM_HEADER_SIZE);
+ if (size < TCM_HEADER_SIZE) {
+ dev_err(chip->dev, "Unable to read header\n");
+ goto out;
+ }
+
+ expected = be32_to_cpu(*(__be32 *)(buf + 2));
+ if (expected > count) {
+ dev_err(chip->dev, "Expected data count\n");
+ size = -EIO;
+ goto out;
+ }
+
+ size += recv_data(chip, &buf[TCM_HEADER_SIZE],
+ expected - TCM_HEADER_SIZE);
+ if (size < expected) {
+ dev_err(chip->dev, "Unable to read remainder of result\n");
+ size = -ETIME;
+ goto out;
+ }
+
+ wait_for_stat(chip, TCM_STS_VALID, chip->vendor.timeout_c,
+ &chip->vendor.int_queue);
+
+ stop = jiffies + chip->vendor.timeout_c;
+ do {
+ msleep(TCM_TIMEOUT);
+ status = tcm_tis_status(chip);
+ if ((status & TCM_STS_DATA_AVAIL) == 0)
+ break;
+
+ } while (time_before(jiffies, stop));
+
+ status = tcm_tis_status(chip);
+ if (status & TCM_STS_DATA_AVAIL) { /* retry? */
+ dev_err(chip->dev, "Error left over data\n");
+ size = -EIO;
+ goto out;
+ }
+
+out:
+ tcm_tis_ready(chip);
+ release_locality(chip, chip->vendor.locality, 0);
+ if (size < 0)
+ tcm_handle_err(chip);
+ return size;
+}
+
+/*
+ * If interrupts are used (signaled by an irq set in the vendor structure)
+ * tcm.c can skip polling for the data to be available as the interrupt is
+ * waited for here
+ */
+static int tcm_tis_send(struct tcm_chip *chip, u8 *buf, size_t len)
+{
+ int rc, status, burstcnt;
+ size_t count = 0;
+ u32 ordinal;
+ unsigned long stop;
+ int send_again = 0;
+
+tcm_tis_send_again:
+ count = 0;
+ if (request_locality(chip, 0) < 0) {
+ dev_err(chip->dev, "send, tcm is busy\n");
+ return -EBUSY;
+ }
+ status = tcm_tis_status(chip);
+
+ if ((status & TCM_STS_COMMAND_READY) == 0) {
+ tcm_tis_ready(chip);
+ if (wait_for_stat(chip, TCM_STS_COMMAND_READY,
+ chip->vendor.timeout_b, &chip->vendor.int_queue) < 0) {
+ dev_err(chip->dev, "send, tcm wait time out1\n");
+ rc = -ETIME;
+ goto out_err;
+ }
+ }
+
+ while (count < len - 1) {
+ burstcnt = get_burstcount(chip);
+ if (burstcnt < 0) {
+ dev_err(chip->dev, "Unable to read burstcount\n");
+ rc = burstcnt;
+ goto out_err;
+ }
+ for (; burstcnt > 0 && count < len - 1; burstcnt--) {
+ tcm_tis_writeb(chip->dev,
+ TCM_DATA_FIFO(chip->vendor.locality), buf[count]);
+ count++;
+ }
+
+ wait_for_stat(chip, TCM_STS_VALID, chip->vendor.timeout_c,
+ &chip->vendor.int_queue);
+ }
+
+ /* write last byte */
+ tcm_tis_writeb(chip->dev,
+ TCM_DATA_FIFO(chip->vendor.locality), buf[count]);
+
+ wait_for_stat(chip, TCM_STS_VALID,
+ chip->vendor.timeout_c, &chip->vendor.int_queue);
+ stop = jiffies + chip->vendor.timeout_c;
+ do {
+ msleep(TCM_TIMEOUT);
+ status = tcm_tis_status(chip);
+ if ((status & TCM_STS_DATA_EXPECT) == 0)
+ break;
+
+ } while (time_before(jiffies, stop));
+
+ if ((status & TCM_STS_DATA_EXPECT) != 0) {
+ dev_err(chip->dev, "send, tcm expect data\n");
+ rc = -EIO;
+ goto out_err;
+ }
+
+ /* go and do it */
+ tcm_tis_writeb(chip->dev, TCM_STS(chip->vendor.locality), TCM_STS_GO);
+
+ ordinal = be32_to_cpu(*((__be32 *)(buf + 6)));
+ if (wait_for_stat(chip, TCM_STS_DATA_AVAIL | TCM_STS_VALID,
+ tcm_calc_ordinal_duration(chip, ordinal),
+ &chip->vendor.read_queue) < 0) {
+ dev_err(chip->dev, "send, tcm wait time out2\n");
+ rc = -ETIME;
+ goto out_err;
+ }
+
+ return len;
+
+out_err:
+ tcm_tis_ready(chip);
+ release_locality(chip, chip->vendor.locality, 0);
+ tcm_handle_err(chip);
+ if (send_again++ < 3) {
+ goto tcm_tis_send_again;
+ }
+
+ dev_err(chip->dev, "kylin send, err: %d\n", rc);
+ return rc;
+}
+
+static struct file_operations tis_ops = {
+ .owner = THIS_MODULE,
+ .llseek = no_llseek,
+ .open = tcm_open,
+ .read = tcm_read,
+ .write = tcm_write,
+ .release = tcm_release,
+};
+
+static DEVICE_ATTR(pubek, S_IRUGO, tcm_show_pubek, NULL);
+static DEVICE_ATTR(pcrs, S_IRUGO, tcm_show_pcrs, NULL);
+static DEVICE_ATTR(enabled, S_IRUGO, tcm_show_enabled, NULL);
+static DEVICE_ATTR(active, S_IRUGO, tcm_show_active, NULL);
+static DEVICE_ATTR(owned, S_IRUGO, tcm_show_owned, NULL);
+static DEVICE_ATTR(temp_deactivated, S_IRUGO, tcm_show_temp_deactivated,
+ NULL);
+static DEVICE_ATTR(caps, S_IRUGO, tcm_show_caps, NULL);
+static DEVICE_ATTR(cancel, S_IWUSR | S_IWGRP, NULL, tcm_store_cancel);
+
+static struct attribute *tis_attrs[] = {
+ &dev_attr_pubek.attr,
+ &dev_attr_pcrs.attr,
+ &dev_attr_enabled.attr,
+ &dev_attr_active.attr,
+ &dev_attr_owned.attr,
+ &dev_attr_temp_deactivated.attr,
+ &dev_attr_caps.attr,
+ &dev_attr_cancel.attr, NULL,
+};
+
+static struct attribute_group tis_attr_grp = {
+ .attrs = tis_attrs
+};
+
+static struct tcm_vendor_specific tcm_tis = {
+ .status = tcm_tis_status,
+ .recv = tcm_tis_recv,
+ .send = tcm_tis_send,
+ .cancel = tcm_tis_ready,
+ .req_complete_mask = TCM_STS_DATA_AVAIL | TCM_STS_VALID,
+ .req_complete_val = TCM_STS_DATA_AVAIL | TCM_STS_VALID,
+ .req_canceled = TCM_STS_COMMAND_READY,
+ .attr_group = &tis_attr_grp,
+ .miscdev = {
+ .fops = &tis_ops,
+ },
+};
+
+static struct tcm_chip *chip;
+static int tcm_tis_spi_probe(struct spi_device *spi)
+{
+ int ret;
+ u8 revid;
+ u32 vendor, intfcaps;
+ struct tcm_tis_spi_phy *phy;
+ struct chip_data *spi_chip;
+
+ pr_info("TCM(ky): __func__(v=%d) ..\n",
+ 10);
+
+ tcm_dbg("TCM-dbg: %s/%d, enter\n", __func__, __LINE__);
+ phy = devm_kzalloc(&spi->dev, sizeof(struct tcm_tis_spi_phy),
+ GFP_KERNEL);
+ if (!phy)
+ return -ENOMEM;
+
+ phy->iobuf = devm_kmalloc(&spi->dev, MAX_SPI_FRAMESIZE, GFP_KERNEL);
+ if (!phy->iobuf)
+ return -ENOMEM;
+
+ phy->spi_device = spi;
+ init_completion(&phy->ready);
+
+ tcm_dbg("TCM-dbg: %s/%d\n", __func__, __LINE__);
+ /* init spi dev */
+ spi->chip_select = 0; /* cs0 */
+ spi->mode = SPI_MODE_0;
+ spi->bits_per_word = 8;
+ spi->max_speed_hz = spi->max_speed_hz ? : 24000000;
+ spi_setup(spi);
+
+ spi_chip = spi_get_ctldata(spi);
+ if (!spi_chip) {
+ pr_err("There was wrong in spi master\n");
+ return -ENODEV;
+ }
+ /* tcm does not support interrupt mode, we use poll mode instead. */
+ spi_chip->poll_mode = 1;
+
+ tcm_dbg("TCM-dbg: %s/%d\n", __func__, __LINE__);
+ /* regiter tcm hw */
+ chip = tcm_register_hardware(&spi->dev, &tcm_tis);
+ if (!chip) {
+ dev_err(chip->dev, "tcm register hardware err\n");
+ return -ENODEV;
+ }
+
+ dev_set_drvdata(chip->dev, phy);
+
+ /**
+ * phytium2000a4 spi controller's clk clk level is unstable,
+ * so it is solved by using the low level of gpio output.
+ **/
+ if (is_ft_all() && (spi->chip_select == 0)) {
+ /* reuse conf reg base */
+ reuse_conf_reg = ioremap(REUSE_CONF_REG_BASE, 0x10);
+ if (!reuse_conf_reg) {
+ dev_err(&spi->dev, "Failed to ioremap reuse conf reg\n");
+ ret = -ENOMEM;
+ goto out_err;
+ }
+
+ /* gpio1 a5 base addr */
+ gpio1_a5 = ioremap(REUSE_GPIO1_A5_BASE, 0x10);
+ if (!gpio1_a5) {
+ dev_err(&spi->dev, "Failed to ioremap gpio1 a5\n");
+ ret = -ENOMEM;
+ goto out_err;
+ }
+
+ /* reuse cs0 to gpio1_a5 */
+ iowrite32((ioread32(reuse_conf_reg) | 0xFFFF0) & 0xFFF9004F,
+ reuse_conf_reg);
+ /* set gpio1 a5 to output */
+ iowrite32(0x20, gpio1_a5 + 0x4);
+ }
+
+ tcm_dbg("TCM-dbg: %s/%d\n",
+ __func__, __LINE__);
+ ret = tcm_tis_readl(chip->dev, TCM_DID_VID(0), &vendor);
+ if (ret < 0)
+ goto out_err;
+
+ tcm_dbg("TCM-dbg: %s/%d, vendor: 0x%x\n",
+ __func__, __LINE__, vendor);
+ if ((vendor & 0xffff) != 0x19f5 && (vendor & 0xffff) != 0x1B4E) {
+ dev_err(chip->dev, "there is no Nationz TCM on you computer\n");
+ goto out_err;
+ }
+
+ ret = tcm_tis_readb(chip->dev, TCM_RID(0), &revid);
+ tcm_dbg("TCM-dbg: %s/%d, revid: 0x%x\n",
+ __func__, __LINE__, revid);
+ if (ret < 0)
+ goto out_err;
+ dev_info(chip->dev, "kylin: 2019-09-21 1.2 TCM "
+ "(device-id 0x%X, rev-id %d)\n",
+ vendor >> 16, revid);
+
+ /* Default timeouts */
+ chip->vendor.timeout_a = msecs_to_jiffies(TIS_SHORT_TIMEOUT);
+ chip->vendor.timeout_b = msecs_to_jiffies(TIS_LONG_TIMEOUT);
+ chip->vendor.timeout_c = msecs_to_jiffies(TIS_SHORT_TIMEOUT);
+ chip->vendor.timeout_d = msecs_to_jiffies(TIS_SHORT_TIMEOUT);
+
+ tcm_dbg("TCM-dbg: %s/%d\n",
+ __func__, __LINE__);
+ /* Figure out the capabilities */
+ ret = tcm_tis_readl(chip->dev,
+ TCM_INTF_CAPS(chip->vendor.locality), &intfcaps);
+ if (ret < 0)
+ goto out_err;
+
+ tcm_dbg("TCM-dbg: %s/%d, intfcaps: 0x%x\n",
+ __func__, __LINE__, intfcaps);
+ if (request_locality(chip, 0) != 0) {
+ dev_err(chip->dev, "tcm request_locality err\n");
+ ret = -ENODEV;
+ goto out_err;
+ }
+
+ INIT_LIST_HEAD(&chip->vendor.list);
+ spin_lock(&tis_lock);
+ list_add(&chip->vendor.list, &tis_chips);
+ spin_unlock(&tis_lock);
+
+ tcm_get_timeouts(chip);
+ tcm_startup(chip);
+
+ tcm_dbg("TCM-dbg: %s/%d, exit\n", __func__, __LINE__);
+ return 0;
+
+out_err:
+ if (is_ft_all()) {
+ if (reuse_conf_reg)
+ iounmap(reuse_conf_reg);
+ if (gpio1_a5)
+ iounmap(gpio1_a5);
+ }
+ tcm_dbg("TCM-dbg: %s/%d, error\n", __func__, __LINE__);
+ dev_set_drvdata(chip->dev, chip);
+ tcm_remove_hardware(chip->dev);
+
+ return ret;
+}
+
+static int tcm_tis_spi_remove(struct spi_device *dev)
+{
+ if (is_ft_all()) {
+ if (reuse_conf_reg)
+ iounmap(reuse_conf_reg);
+ if (gpio1_a5)
+ iounmap(gpio1_a5);
+ }
+
+ dev_info(&dev->dev, "%s\n", __func__);
+ dev_set_drvdata(chip->dev, chip);
+ tcm_remove_hardware(&dev->dev);
+
+ return 0;
+}
+
+static const struct acpi_device_id tcm_tis_spi_acpi_match[] = {
+ {"TCMS0001", 0},
+ {"SMO0768", 0},
+ {"ZIC0601", 0},
+ {}
+};
+MODULE_DEVICE_TABLE(acpi, tcm_tis_spi_acpi_match);
+
+static const struct spi_device_id tcm_tis_spi_id_table[] = {
+ {"SMO0768", 0},
+ {"ZIC0601", 0},
+ {}
+};
+MODULE_DEVICE_TABLE(spi, tcm_tis_spi_id_table);
+
+static struct spi_driver tcm_tis_spi_drv = {
+ .driver = {
+ .name = "tcm_tis_spi",
+ .acpi_match_table = ACPI_PTR(tcm_tis_spi_acpi_match),
+ },
+ .id_table = tcm_tis_spi_id_table,
+ .probe = tcm_tis_spi_probe,
+ .remove = tcm_tis_spi_remove,
+};
+
+#if 1
+module_spi_driver(tcm_tis_spi_drv);
+#else/*0*/
+
+static int __init __spi_driver_init(void)
+{
+ pr_info("TCM(ky): __init __func__(ver=%2d)\n",
+ 10);
+ return spi_register_driver(&tcm_tis_spi_drv);
+}
+
+static void __exit __spi_driver_exit(void)
+{
+ pr_info("TCM(ky): __exit __func__\n");
+ spi_unregister_driver(&tcm_tis_spi_drv);
+}
+
+module_init(__spi_driver_init);
+module_exit(__spi_driver_exit);
+#endif/*0*/
+
+MODULE_AUTHOR("xiongxin<xiongxin(a)tj.kylinos.cn>");
+MODULE_DESCRIPTION("TCM Driver Base Spi");
+MODULE_VERSION("6.1.0.2");
+MODULE_LICENSE("GPL");
--
2.23.0
1
1
23 Feb '21
1
0
23 Feb '21
From: Sang Yan <sangyan(a)huawei.com>
hulk inclusion
category: feature
bugzilla: 48159
CVE: N/A
Introducing a feature of CPU PARK in order to save time
of cpus down and up during kexec, which may cost 250ms of
per cpu's down and 30ms of up.
As a result, for 128 cores, it costs more than 30 seconds
to down and up cpus during kexec. Think about 256 cores and more.
CPU PARK is a state that cpu power-on and staying in spin loop, polling
for exit chances, such as writing exit address.
Reserving a block of memory, to fill with cpu park text section,
exit address and park-magic-flag of each cpu. In implementation,
reserved one page for one cpu core.
Cpus going to park state instead of down in machine_shutdown().
Cpus going out of park state in smp_init instead of brought up.
One of cpu park sections in pre-reserved memory blocks,:
+--------------+
+ exit address +
+--------------+
+ park magic +
+--------------+
+ park codes +
+ . +
+ . +
+ . +
+--------------+
Signed-off-by: Sang Yan <sangyan(a)huawei.com>
---
arch/arm64/Kconfig | 12 ++
arch/arm64/include/asm/kexec.h | 6 +
arch/arm64/include/asm/smp.h | 15 +++
arch/arm64/kernel/Makefile | 1 +
arch/arm64/kernel/cpu-park.S | 59 ++++++++++
arch/arm64/kernel/machine_kexec.c | 2 +-
arch/arm64/kernel/process.c | 4 +
arch/arm64/kernel/smp.c | 229 ++++++++++++++++++++++++++++++++++++++
arch/arm64/mm/init.c | 55 +++++++++
9 files changed, 382 insertions(+), 1 deletion(-)
create mode 100644 arch/arm64/kernel/cpu-park.S
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b9c5654..0885668 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -345,6 +345,18 @@ config KASAN_SHADOW_OFFSET
default 0xeffffff900000000 if ARM64_VA_BITS_36 && KASAN_SW_TAGS
default 0xffffffffffffffff
+config ARM64_CPU_PARK
+ bool "Support CPU PARK on kexec"
+ depends on SMP
+ depends on KEXEC_CORE
+ help
+ This enables support for CPU PARK feature in
+ order to save time of cpu down to up.
+ CPU park is a state through kexec, spin loop
+ instead of cpu die before jumping to new kernel,
+ jumping out from loop to new kernel entry in
+ smp_init.
+
source "arch/arm64/Kconfig.platforms"
menu "Kernel Features"
diff --git a/arch/arm64/include/asm/kexec.h b/arch/arm64/include/asm/kexec.h
index 79909ae..a133889 100644
--- a/arch/arm64/include/asm/kexec.h
+++ b/arch/arm64/include/asm/kexec.h
@@ -36,6 +36,11 @@
#define CRASH_ADDR_HIGH_MAX MEMBLOCK_ALLOC_ACCESSIBLE
+#ifdef CONFIG_ARM64_CPU_PARK
+/* CPU park state flag: "park" */
+#define PARK_MAGIC 0x7061726b
+#endif
+
#ifndef __ASSEMBLY__
/**
@@ -104,6 +109,7 @@ static inline void crash_post_resume(void) {}
#ifdef CONFIG_KEXEC_CORE
extern void __init reserve_crashkernel(void);
#endif
+void machine_kexec_mask_interrupts(void);
#ifdef CONFIG_KEXEC_FILE
#define ARCH_HAS_KIMAGE_ARCH
diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h
index 2e7f529..8c5d2d6 100644
--- a/arch/arm64/include/asm/smp.h
+++ b/arch/arm64/include/asm/smp.h
@@ -145,6 +145,21 @@ bool cpus_are_stuck_in_kernel(void);
extern void crash_smp_send_stop(void);
extern bool smp_crash_stop_failed(void);
+#ifdef CONFIG_ARM64_CPU_PARK
+#define PARK_SECTION_SIZE 1024
+struct cpu_park_info {
+ /* Physical address of reserved park memory. */
+ unsigned long start;
+ /* park reserve mem len should be PARK_SECTION_SIZE * NR_CPUS */
+ unsigned long len;
+ /* Virtual address of reserved park memory. */
+ unsigned long start_v;
+};
+extern struct cpu_park_info park_info;
+extern void enter_cpu_park(unsigned long text, unsigned long exit);
+extern void do_cpu_park(unsigned long exit);
+extern int kexec_smp_send_park(void);
+#endif
#endif /* ifndef __ASSEMBLY__ */
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 2621d5c..60478d2 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -54,6 +54,7 @@ obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o
obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o \
cpu-reset.o
+obj-$(CONFIG_ARM64_CPU_PARK) += cpu-park.o
obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o
obj-$(CONFIG_ARM64_RELOC_TEST) += arm64-reloc-test.o
arm64-reloc-test-y := reloc_test_core.o reloc_test_syms.o
diff --git a/arch/arm64/kernel/cpu-park.S b/arch/arm64/kernel/cpu-park.S
new file mode 100644
index 0000000..10c685c
--- /dev/null
+++ b/arch/arm64/kernel/cpu-park.S
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * CPU park routines
+ *
+ * Copyright (C) 2020 Huawei Technologies., Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+#include <asm/kexec.h>
+#include <asm/sysreg.h>
+#include <asm/virt.h>
+
+.text
+.pushsection .idmap.text, "awx"
+
+/* cpu park helper in idmap section */
+SYM_CODE_START(enter_cpu_park)
+ /* Clear sctlr_el1 flags. */
+ mrs x12, sctlr_el1
+ mov_q x13, SCTLR_ELx_FLAGS
+ bic x12, x12, x13
+ pre_disable_mmu_workaround
+ msr sctlr_el1, x12 /* disable mmu */
+ isb
+
+ mov x18, x0
+ mov x0, x1 /* secondary_entry addr */
+ br x18 /* call do_cpu_park of each cpu */
+SYM_CODE_END(enter_cpu_park)
+
+.popsection
+
+SYM_CODE_START(do_cpu_park)
+ ldr x18, =PARK_MAGIC /* magic number "park" */
+ add x1, x0, #8
+ str x18, [x1] /* set on-park flag */
+ dc civac, x1 /* flush cache of "park" */
+ dsb nsh
+ isb
+
+.Lloop:
+ wfe
+ isb
+ ldr x19, [x0]
+ cmp x19, #0 /* test secondary_entry */
+ b.eq .Lloop
+
+ ic iallu /* invalidate the local I-cache */
+ dsb nsh
+ isb
+
+ br x19 /* jump to secondary_entry */
+SYM_CODE_END(do_cpu_park)
+
diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c
index a0b144c..f47ce96 100644
--- a/arch/arm64/kernel/machine_kexec.c
+++ b/arch/arm64/kernel/machine_kexec.c
@@ -213,7 +213,7 @@ void machine_kexec(struct kimage *kimage)
BUG(); /* Should never get here. */
}
-static void machine_kexec_mask_interrupts(void)
+void machine_kexec_mask_interrupts(void)
{
unsigned int i;
struct irq_desc *desc;
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 73e3b32..10cffee 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -146,6 +146,10 @@ void arch_cpu_idle_dead(void)
*/
void machine_shutdown(void)
{
+#ifdef CONFIG_ARM64_CPU_PARK
+ if (kexec_smp_send_park() == 0)
+ return;
+#endif
smp_shutdown_nonboot_cpus(reboot_cpu);
}
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 18e9727..bc475d5 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -32,6 +32,8 @@
#include <linux/irq_work.h>
#include <linux/kernel_stat.h>
#include <linux/kexec.h>
+#include <linux/console.h>
+
#include <linux/kvm_host.h>
#include <asm/alternative.h>
@@ -93,6 +95,167 @@ static inline int op_cpu_kill(unsigned int cpu)
}
#endif
+#ifdef CONFIG_ARM64_CPU_PARK
+struct cpu_park_section {
+ unsigned long exit; /* exit address of park look */
+ unsigned long magic; /* maigc represent park state */
+ char text[0]; /* text section of park */
+};
+
+static int mmap_cpu_park_mem(void)
+{
+ if (!park_info.start)
+ return -ENOMEM;
+
+ if (park_info.start_v)
+ return 0;
+
+ park_info.start_v = (unsigned long)__ioremap(park_info.start,
+ park_info.len,
+ PAGE_KERNEL_EXEC);
+ if (!park_info.start_v) {
+ pr_warn("map park memory failed.");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static inline unsigned long cpu_park_section_v(unsigned int cpu)
+{
+ return park_info.start_v + PARK_SECTION_SIZE * (cpu - 1);
+}
+
+static inline unsigned long cpu_park_section_p(unsigned int cpu)
+{
+ return park_info.start + PARK_SECTION_SIZE * (cpu - 1);
+}
+
+/*
+ * Write the secondary_entry to exit section of park state.
+ * Then the secondary cpu will jump straight into the kernel
+ * by the secondary_entry.
+ */
+static int write_park_exit(unsigned int cpu)
+{
+ struct cpu_park_section *park_section;
+ unsigned long *park_exit;
+ unsigned long *park_text;
+
+ if (mmap_cpu_park_mem() != 0)
+ return -EPERM;
+
+ park_section = (struct cpu_park_section *)cpu_park_section_v(cpu);
+ park_exit = &park_section->exit;
+ park_text = (unsigned long *)park_section->text;
+ pr_debug("park_text 0x%lx : 0x%lx, do_cpu_park text 0x%lx : 0x%lx",
+ (unsigned long)park_text, *park_text,
+ (unsigned long)do_cpu_park,
+ *(unsigned long *)do_cpu_park);
+
+ /*
+ * Test first 8 bytes to determine
+ * whether needs to write cpu park exit.
+ */
+ if (*park_text == *(unsigned long *)do_cpu_park) {
+ writeq_relaxed(__pa_symbol(secondary_entry), park_exit);
+ __flush_dcache_area((__force void *)park_exit,
+ sizeof(unsigned long));
+ flush_icache_range((unsigned long)park_exit,
+ (unsigned long)(park_exit + 1));
+ sev();
+ dsb(sy);
+ isb();
+
+ pr_debug("Write cpu %u secondary entry 0x%lx to 0x%lx.",
+ cpu, *park_exit, (unsigned long)park_exit);
+ pr_info("Boot cpu %u from PARK state.", cpu);
+ return 0;
+ }
+
+ return -EPERM;
+}
+
+/* Install cpu park sections for the specific cpu. */
+static int install_cpu_park(unsigned int cpu)
+{
+ struct cpu_park_section *park_section;
+ unsigned long *park_exit;
+ unsigned long *park_magic;
+ unsigned long park_text_len;
+
+ park_section = (struct cpu_park_section *)cpu_park_section_v(cpu);
+ pr_debug("Install cpu park on cpu %u park exit 0x%lx park text 0x%lx",
+ cpu, (unsigned long)park_section,
+ (unsigned long)(park_section->text));
+
+ park_exit = &park_section->exit;
+ park_magic = &park_section->magic;
+ park_text_len = PARK_SECTION_SIZE - sizeof(struct cpu_park_section);
+
+ *park_exit = 0UL;
+ *park_magic = 0UL;
+ memcpy((void *)park_section->text, do_cpu_park, park_text_len);
+ __flush_dcache_area((void *)park_section, PARK_SECTION_SIZE);
+
+ return 0;
+}
+
+static int uninstall_cpu_park(unsigned int cpu)
+{
+ unsigned long park_section;
+
+ if (mmap_cpu_park_mem() != 0)
+ return -EPERM;
+
+ park_section = cpu_park_section_v(cpu);
+ memset((void *)park_section, 0, PARK_SECTION_SIZE);
+ __flush_dcache_area((void *)park_section, PARK_SECTION_SIZE);
+
+ return 0;
+}
+
+static int cpu_wait_park(unsigned int cpu)
+{
+ long timeout;
+ struct cpu_park_section *park_section;
+
+ volatile unsigned long *park_magic;
+
+ park_section = (struct cpu_park_section *)cpu_park_section_v(cpu);
+ park_magic = &park_section->magic;
+
+ timeout = USEC_PER_SEC;
+ while (*park_magic != PARK_MAGIC && timeout--)
+ udelay(1);
+
+ if (timeout > 0)
+ pr_debug("cpu %u park done.", cpu);
+ else
+ pr_err("cpu %u park failed.", cpu);
+
+ return *park_magic == PARK_MAGIC;
+}
+
+static void cpu_park(unsigned int cpu)
+{
+ unsigned long park_section_p;
+ unsigned long park_exit_phy;
+ unsigned long do_park;
+ typeof(enter_cpu_park) *park;
+
+ park_section_p = cpu_park_section_p(cpu);
+ park_exit_phy = park_section_p;
+ pr_debug("Go to park cpu %u exit address 0x%lx", cpu, park_exit_phy);
+
+ do_park = park_section_p + sizeof(struct cpu_park_section);
+ park = (void *)__pa_symbol(enter_cpu_park);
+
+ cpu_install_idmap();
+ park(do_park, park_exit_phy);
+ unreachable();
+}
+#endif
/*
* Boot a secondary CPU, and assign it the specified idle task.
@@ -102,6 +265,10 @@ static int boot_secondary(unsigned int cpu, struct task_struct *idle)
{
const struct cpu_operations *ops = get_cpu_ops(cpu);
+#ifdef CONFIG_ARM64_CPU_PARK
+ if (write_park_exit(cpu) == 0)
+ return 0;
+#endif
if (ops->cpu_boot)
return ops->cpu_boot(cpu);
@@ -131,6 +298,9 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
return ret;
}
+#ifdef CONFIG_ARM64_CPU_PARK
+ uninstall_cpu_park(cpu);
+#endif
/*
* CPU was successfully started, wait for it to come online or
* time out.
@@ -844,10 +1014,30 @@ void arch_irq_work_raise(void)
static void local_cpu_stop(void)
{
+ int cpu;
+ const struct cpu_operations *ops = NULL;
+
set_cpu_online(smp_processor_id(), false);
local_daif_mask();
sdei_mask_local_cpu();
+
+#ifdef CONFIG_ARM64_CPU_PARK
+ /*
+ * Go to cpu park state.
+ * Otherwise go to cpu die.
+ */
+ cpu = smp_processor_id();
+ if (kexec_in_progress && park_info.start_v) {
+ machine_kexec_mask_interrupts();
+ cpu_park(cpu);
+
+ ops = get_cpu_ops(cpu);
+ if (ops && ops->cpu_die)
+ ops->cpu_die(cpu);
+ }
+#endif
+
cpu_park_loop();
}
@@ -1053,6 +1243,45 @@ void smp_send_stop(void)
sdei_mask_local_cpu();
}
+#ifdef CONFIG_ARM64_CPU_PARK
+int kexec_smp_send_park(void)
+{
+ unsigned long cpu;
+
+ if (WARN_ON(!kexec_in_progress)) {
+ pr_crit("%s called not in kexec progress.", __func__);
+ return -EPERM;
+ }
+
+ if (mmap_cpu_park_mem() != 0) {
+ pr_info("no cpuparkmem, goto normal way.");
+ return -EPERM;
+ }
+
+ local_irq_disable();
+
+ if (num_online_cpus() > 1) {
+ cpumask_t mask;
+
+ cpumask_copy(&mask, cpu_online_mask);
+ cpumask_clear_cpu(smp_processor_id(), &mask);
+
+ for_each_cpu(cpu, &mask)
+ install_cpu_park(cpu);
+ smp_cross_call(&mask, IPI_CPU_STOP);
+
+ /* Wait for other CPUs to park */
+ for_each_cpu(cpu, &mask)
+ cpu_wait_park(cpu);
+ pr_info("smp park other cpus done\n");
+ }
+
+ sdei_mask_local_cpu();
+
+ return 0;
+}
+#endif
+
#ifdef CONFIG_KEXEC_CORE
void crash_smp_send_stop(void)
{
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 794f992..d01259c 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -236,6 +236,57 @@ static void __init fdt_enforce_memory_region(void)
memblock_add(usable_rgns[1].base, usable_rgns[1].size);
}
+#ifdef CONFIG_ARM64_CPU_PARK
+struct cpu_park_info park_info = {
+ .start = 0,
+ .len = PARK_SECTION_SIZE * NR_CPUS,
+ .start_v = 0,
+};
+
+static int __init parse_park_mem(char *p)
+{
+ if (!p)
+ return 0;
+
+ park_info.start = PAGE_ALIGN(memparse(p, NULL));
+ if (park_info.start == 0)
+ pr_info("cpu park mem params[%s]", p);
+
+ return 0;
+}
+early_param("cpuparkmem", parse_park_mem);
+
+static int __init reserve_park_mem(void)
+{
+ if (park_info.start == 0 || park_info.len == 0)
+ return 0;
+
+ park_info.start = PAGE_ALIGN(park_info.start);
+ park_info.len = PAGE_ALIGN(park_info.len);
+
+ if (!memblock_is_region_memory(park_info.start, park_info.len)) {
+ pr_warn("cannot reserve park mem: region is not memory!");
+ goto out;
+ }
+
+ if (memblock_is_region_reserved(park_info.start, park_info.len)) {
+ pr_warn("cannot reserve park mem: region overlaps reserved memory!");
+ goto out;
+ }
+
+ memblock_remove(park_info.start, park_info.len);
+ pr_info("cpu park mem reserved: 0x%016lx - 0x%016lx (%ld MB)",
+ park_info.start, park_info.start + park_info.len,
+ park_info.len >> 20);
+
+ return 0;
+out:
+ park_info.start = 0;
+ park_info.len = 0;
+ return -EINVAL;
+}
+#endif
+
void __init arm64_memblock_init(void)
{
const s64 linear_region_size = BIT(vabits_actual - 1);
@@ -357,6 +408,10 @@ void __init arm64_memblock_init(void)
reserve_crashkernel();
+#ifdef CONFIG_ARM64_CPU_PARK
+ reserve_park_mem();
+#endif
+
reserve_elfcorehdr();
high_memory = __va(memblock_end_of_DRAM() - 1) + 1;
--
2.9.5
2
3
22 Feb '21
From: Sang Yan <sangyan(a)huawei.com>
hulk inclusion
category: feature
bugzilla: 48159
CVE: N/A
In normal kexec, relocating kernel may cost 5 ~ 10 seconds, to
copy all segments from vmalloced memory to kernel boot memory,
because of disabled mmu.
We introduce quick kexec to save time of copying memory as above,
just like kdump(kexec on crash), by using reserved memory
"Quick Kexec".
Constructing quick kimage as the same as crash kernel,
then simply copy all segments of kimage to reserved memroy.
We also add this support in syscall kexec_load using flags
of KEXEC_QUICK.
Signed-off-by: Sang Yan <sangyan(a)huawei.com>
---
arch/Kconfig | 10 ++++++++++
include/linux/ioport.h | 1 +
include/linux/kexec.h | 11 ++++++++++-
include/uapi/linux/kexec.h | 1 +
kernel/kexec.c | 10 ++++++++++
kernel/kexec_core.c | 42 +++++++++++++++++++++++++++++++++---------
6 files changed, 65 insertions(+), 10 deletions(-)
diff --git a/arch/Kconfig b/arch/Kconfig
index 2592b4b..7811eee 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -18,6 +18,16 @@ config KEXEC_CORE
select CRASH_CORE
bool
+config QUICK_KEXEC
+ bool "Support for quick kexec"
+ depends on KEXEC_CORE
+ help
+ It uses pre-reserved memory to accelerate kexec, just like
+ crash kexec, loads new kernel and initrd to reserved memory,
+ and boots new kernel on that memory. It will save the time
+ of relocating kernel.
+
+
config KEXEC_ELF
bool
diff --git a/include/linux/ioport.h b/include/linux/ioport.h
index 5135d4b..84a716f 100644
--- a/include/linux/ioport.h
+++ b/include/linux/ioport.h
@@ -139,6 +139,7 @@ enum {
IORES_DESC_DEVICE_PRIVATE_MEMORY = 6,
IORES_DESC_RESERVED = 7,
IORES_DESC_SOFT_RESERVED = 8,
+ IORES_DESC_QUICK_KEXEC = 9,
};
/*
diff --git a/include/linux/kexec.h b/include/linux/kexec.h
index f301f2f..7fff410 100644
--- a/include/linux/kexec.h
+++ b/include/linux/kexec.h
@@ -269,9 +269,10 @@ struct kimage {
unsigned long control_page;
/* Flags to indicate special processing */
- unsigned int type : 1;
+ unsigned int type : 2;
#define KEXEC_TYPE_DEFAULT 0
#define KEXEC_TYPE_CRASH 1
+#define KEXEC_TYPE_QUICK 2
unsigned int preserve_context : 1;
/* If set, we are using file mode kexec syscall */
unsigned int file_mode:1;
@@ -331,6 +332,11 @@ extern int kexec_load_disabled;
#define KEXEC_FLAGS (KEXEC_ON_CRASH | KEXEC_PRESERVE_CONTEXT)
#endif
+#ifdef CONFIG_QUICK_KEXEC
+#undef KEXEC_FLAGS
+#define KEXEC_FLAGS (KEXEC_ON_CRASH | KEXEC_QUICK)
+#endif
+
/* List of defined/legal kexec file flags */
#define KEXEC_FILE_FLAGS (KEXEC_FILE_UNLOAD | KEXEC_FILE_ON_CRASH | \
KEXEC_FILE_NO_INITRAMFS)
@@ -338,6 +344,9 @@ extern int kexec_load_disabled;
/* Location of a reserved region to hold the crash kernel.
*/
extern note_buf_t __percpu *crash_notes;
+#ifdef CONFIG_QUICK_KEXEC
+extern struct resource quick_kexec_res;
+#endif
/* flag to track if kexec reboot is in progress */
extern bool kexec_in_progress;
diff --git a/include/uapi/linux/kexec.h b/include/uapi/linux/kexec.h
index 05669c8..d891d80 100644
--- a/include/uapi/linux/kexec.h
+++ b/include/uapi/linux/kexec.h
@@ -12,6 +12,7 @@
/* kexec flags for different usage scenarios */
#define KEXEC_ON_CRASH 0x00000001
#define KEXEC_PRESERVE_CONTEXT 0x00000002
+#define KEXEC_QUICK 0x00000004
#define KEXEC_ARCH_MASK 0xffff0000
/*
diff --git a/kernel/kexec.c b/kernel/kexec.c
index c82c6c0..4acc909 100644
--- a/kernel/kexec.c
+++ b/kernel/kexec.c
@@ -44,6 +44,9 @@ static int kimage_alloc_init(struct kimage **rimage, unsigned long entry,
int ret;
struct kimage *image;
bool kexec_on_panic = flags & KEXEC_ON_CRASH;
+#ifdef CONFIG_QUICK_KEXEC
+ bool kexec_on_quick = flags & KEXEC_QUICK;
+#endif
if (kexec_on_panic) {
/* Verify we have a valid entry point */
@@ -69,6 +72,13 @@ static int kimage_alloc_init(struct kimage **rimage, unsigned long entry,
image->type = KEXEC_TYPE_CRASH;
}
+#ifdef CONFIG_QUICK_KEXEC
+ if (kexec_on_quick) {
+ image->control_page = quick_kexec_res.start;
+ image->type = KEXEC_TYPE_QUICK;
+ }
+#endif
+
ret = sanity_check_segment_list(image);
if (ret)
goto out_free_image;
diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c
index 2ca8875..c7e2aa2 100644
--- a/kernel/kexec_core.c
+++ b/kernel/kexec_core.c
@@ -53,6 +53,17 @@ note_buf_t __percpu *crash_notes;
/* Flag to indicate we are going to kexec a new kernel */
bool kexec_in_progress = false;
+/* Resource for quick kexec */
+#ifdef CONFIG_QUICK_KEXEC
+struct resource quick_kexec_res = {
+ .name = "Quick kexec",
+ .start = 0,
+ .end = 0,
+ .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM,
+ .desc = IORES_DESC_QUICK_KEXEC
+};
+#endif
+
int kexec_should_crash(struct task_struct *p)
{
/*
@@ -396,8 +407,9 @@ static struct page *kimage_alloc_normal_control_pages(struct kimage *image,
return pages;
}
-static struct page *kimage_alloc_crash_control_pages(struct kimage *image,
- unsigned int order)
+static struct page *kimage_alloc_special_control_pages(struct kimage *image,
+ unsigned int order,
+ unsigned long end)
{
/* Control pages are special, they are the intermediaries
* that are needed while we copy the rest of the pages
@@ -427,7 +439,7 @@ static struct page *kimage_alloc_crash_control_pages(struct kimage *image,
size = (1 << order) << PAGE_SHIFT;
hole_start = (image->control_page + (size - 1)) & ~(size - 1);
hole_end = hole_start + size - 1;
- while (hole_end <= crashk_res.end) {
+ while (hole_end <= end) {
unsigned long i;
cond_resched();
@@ -462,7 +474,6 @@ static struct page *kimage_alloc_crash_control_pages(struct kimage *image,
return pages;
}
-
struct page *kimage_alloc_control_pages(struct kimage *image,
unsigned int order)
{
@@ -473,8 +484,15 @@ struct page *kimage_alloc_control_pages(struct kimage *image,
pages = kimage_alloc_normal_control_pages(image, order);
break;
case KEXEC_TYPE_CRASH:
- pages = kimage_alloc_crash_control_pages(image, order);
+ pages = kimage_alloc_special_control_pages(image, order,
+ crashk_res.end);
+ break;
+#ifdef CONFIG_QUICK_KEXEC
+ case KEXEC_TYPE_QUICK:
+ pages = kimage_alloc_special_control_pages(image, order,
+ quick_kexec_res.end);
break;
+#endif
}
return pages;
@@ -830,11 +848,12 @@ static int kimage_load_normal_segment(struct kimage *image,
return result;
}
-static int kimage_load_crash_segment(struct kimage *image,
+static int kimage_load_special_segment(struct kimage *image,
struct kexec_segment *segment)
{
- /* For crash dumps kernels we simply copy the data from
- * user space to it's destination.
+ /*
+ * For crash dumps kernels and quick kexec kernels
+ * we simply copy the data from user space to it's destination.
* We do things a page at a time for the sake of kmap.
*/
unsigned long maddr;
@@ -908,8 +927,13 @@ int kimage_load_segment(struct kimage *image,
result = kimage_load_normal_segment(image, segment);
break;
case KEXEC_TYPE_CRASH:
- result = kimage_load_crash_segment(image, segment);
+ result = kimage_load_special_segment(image, segment);
break;
+#ifdef CONFIG_QUICK_KEXEC
+ case KEXEC_TYPE_QUICK:
+ result = kimage_load_special_segment(image, segment);
+ break;
+#endif
}
return result;
--
2.9.5
2
3
bugfix for 20.03 @ 2021/02/22
Aichun Li (7):
netpoll: remove dev argument from netpoll_send_skb_on_dev()
netpoll: move netpoll_send_skb() out of line
netpoll: netpoll_send_skb() returns transmit status
netpoll: accept NULL np argument in netpoll_send_skb()
bonding: add an option to specify a delay between peer notifications
bonding: fix value exported by Netlink for peer_notif_delay
bonding: add documentation for peer_notif_delay
Akilesh Kailash (1):
dm snapshot: flush merged data before committing metadata
Al Viro (2):
don't dump the threads that had been already exiting when zapped.
dump_common_audit_data(): fix racy accesses to ->d_name
Aleksandr Nogikh (1):
netem: fix zero division in tabledist
Alexander Duyck (1):
tcp: Set INET_ECN_xmit configuration in tcp_reinit_congestion_control
Alexander Lobakin (1):
skbuff: back tiny skbs with kmalloc() in __netdev_alloc_skb() too
Alexey Dobriyan (2):
proc: change ->nlink under proc_subdir_lock
proc: fix lookup in /proc/net subdirectories after setns(2)
Alexey Kardashevskiy (1):
serial_core: Check for port state when tty is in error state
Amit Cohen (1):
mlxsw: core: Fix use-after-free in mlxsw_emad_trans_finish()
Andy Shevchenko (2):
device property: Keep secondary firmware node secondary by type
device property: Don't clear secondary pointer for shared primary
firmware node
Antoine Tenart (6):
netfilter: bridge: reset skb->pkt_type after NF_INET_POST_ROUTING
traversal
net: ip6_gre: set dev->hard_header_len when using header_ops
net-sysfs: take the rtnl lock when storing xps_cpus
net-sysfs: take the rtnl lock when accessing xps_cpus_map and num_tc
net-sysfs: take the rtnl lock when storing xps_rxqs
net-sysfs: take the rtnl lock when accessing xps_rxqs_map and num_tc
Ard Biesheuvel (3):
efivarfs: revert "fix memory leak in efivarfs_create()"
arm64, mm, efi: Account for GICv3 LPI tables in static memblock
reserve table
efi/arm: Revert "Defer persistent reservations until after
paging_init()"
Arjun Roy (1):
tcp: Prevent low rmem stalls with SO_RCVLOWAT.
Arnaldo Carvalho de Melo (1):
perf scripting python: Avoid declaring function pointers with a
visibility attribute
Axel Lin (1):
ASoC: msm8916-wcd-digital: Select REGMAP_MMIO to fix build error
Aya Levin (1):
net: ipv6: Validate GSO SKB before finish IPv6 processing
Bart Van Assche (1):
scsi: scsi_transport_spi: Set RQF_PM for domain validation commands
Bharat Gooty (1):
PCI: iproc: Fix out-of-bound array accesses
Bixuan Cui (2):
mmap: fix a compiling error for 'MAP_CHECKNODE'
powerpc: fix a compiling error for 'access_ok'
Bjorn Helgaas (1):
PCI: Bounds-check command-line resource alignment requests
Björn Töpel (1):
ixgbe: avoid premature Rx buffer reuse
Boqun Feng (1):
fcntl: Fix potential deadlock in send_sig{io, urg}()
Boris Protopopov (1):
Convert trailing spaces and periods in path components
Brian Foster (1):
xfs: flush new eof page on truncate to avoid post-eof corruption
Calum Mackay (1):
lockd: don't use interval-based rebinding over TCP
Chen Zhou (1):
selinux: Fix error return code in sel_ib_pkey_sid_slow()
Chenguangli (1):
scsi/hifc:Fix the bug that the system may be oops during unintall hifc
module.
Cheng Lin (1):
nfs_common: need lock during iterate through the list
Christoph Hellwig (2):
nbd: fix a block_device refcount leak in nbd_release
xfs: fix a missing unlock on error in xfs_fs_map_blocks
Chunguang Xu (1):
ext4: fix a memory leak of ext4_free_data
Chunyan Zhang (1):
tick/common: Touch watchdog in tick_unfreeze() on all CPUs
Colin Ian King (1):
PCI: Fix overflow in command-line resource alignment requests
Cong Wang (1):
erspan: fix version 1 check in gre_parse_header()
Damien Le Moal (1):
null_blk: Fix zone size initialization
Dan Carpenter (1):
futex: Don't enable IRQs unconditionally in put_pi_state()
Daniel Scally (1):
Revert "ACPI / resources: Use AE_CTRL_TERMINATE to terminate resources
walks"
Darrick J. Wong (12):
xfs: fix realtime bitmap/summary file truncation when growing rt
volume
xfs: don't free rt blocks when we're doing a REMAP bunmapi call
xfs: set xefi_discard when creating a deferred agfl free log intent
item
xfs: fix scrub flagging rtinherit even if there is no rt device
xfs: fix flags argument to rmap lookup when converting shared file
rmaps
xfs: set the unwritten bit in rmap lookup flags in
xchk_bmap_get_rmapextents
xfs: fix rmap key and record comparison functions
xfs: fix brainos in the refcount scrubber's rmap fragment processor
vfs: remove lockdep bogosity in __sb_start_write
xfs: fix the minrecs logic when dealing with inode root child blocks
xfs: strengthen rmap record flags checking
xfs: revert "xfs: fix rmap key and record comparison functions"
Dave Wysochanski (1):
NFS4: Fix use-after-free in trace_event_raw_event_nfs4_set_lock
Dexuan Cui (1):
ACPI: scan: Harden acpi_device_add() against device ID overflows
Dinghao Liu (4):
ext4: fix error handling code in add_new_gdb
net/mlx5e: Fix memleak in mlx5e_create_l2_table_groups
net/mlx5e: Fix two double free cases
netfilter: nf_nat: Fix memleak in nf_nat_init
Dongdong Wang (1):
lwt: Disable BH too in run_lwt_bpf()
Dongli Zhang (1):
page_frag: Recover from memory pressure
Douglas Gilbert (1):
sgl_alloc_order: fix memory leak
Eddy Wu (1):
fork: fix copy_process(CLONE_PARENT) race with the exiting
->real_parent
Eran Ben Elisha (1):
net/mlx5: Fix wrong address reclaim when command interface is down
Eric Auger (1):
vfio/pci: Move dummy_resources_list init in vfio_pci_probe()
Eric Biggers (1):
ext4: fix leaking sysfs kobject after failed mount
Eric Dumazet (4):
tcp: select sane initial rcvq_space.space for big MSS
net: avoid 32 x truesize under-estimation for tiny skbs
net_sched: avoid shift-out-of-bounds in tcindex_set_parms()
net_sched: reject silly cell_log in qdisc_get_rtab()
Fang Lijun (3):
arm64/ascend: mm: Add MAP_CHECKNODE flag to check node hugetlb
arm64/ascend: mm: Fix arm32 compile warnings
arm64/ascend: mm: Fix hugetlb check node error
Fangrui Song (1):
arm64: Change .weak to SYM_FUNC_START_WEAK_PI for
arch/arm64/lib/mem*.S
Florian Fainelli (1):
net: Have netpoll bring-up DSA management interface
Florian Westphal (4):
netfilter: nf_tables: avoid false-postive lockdep splat
netfilter: xt_RATEEST: reject non-null terminated string from
userspace
net: ip: always refragment ip defragmented packets
net: fix pmtu check in nopmtudisc mode
Gabriel Krisman Bertazi (2):
blk-cgroup: Fix memleak on error path
blk-cgroup: Pre-allocate tree node on blkg_conf_prep
George Spelvin (1):
random32: make prandom_u32() output unpredictable
Gerald Schaefer (1):
mm/userfaultfd: do not access vma->vm_mm after calling
handle_userfault()
Guillaume Nault (4):
ipv4: Fix tos mask in inet_rtm_getroute()
ipv4: Ignore ECN bits for fib lookups in fib_compute_spec_dst()
netfilter: rpfilter: mask ecn bits before fib lookup
udp: mask TOS bits in udp_v4_early_demux()
Hanjun Guo (1):
clocksource/drivers/arch_timer: Fix vdso_fix compile error for arm32
Hannes Reinecke (1):
dm: avoid filesystem lookup in dm_get_dev_t()
Hans de Goede (1):
ACPI: scan: Make acpi_bus_get_device() clear return pointer on error
Heiner Kallweit (1):
net: bridge: add missing counters to ndo_get_stats64 callback
Hoang Le (1):
tipc: fix NULL deref in tipc_link_xmit()
Huang Shijie (1):
lib/genalloc: fix the overflow when size is too big
Huang Ying (1):
mm: fix a race during THP splitting
Hugh Dickins (2):
mlock: fix unevictable_pgs event counts on THP
mm: fix check_move_unevictable_pages() on THP
Hui Wang (1):
ACPI: PNP: compare the string length in the matching_id()
Hyeongseok Kim (1):
dm verity: skip verity work if I/O error when system is shutting down
Ido Schimmel (2):
mlxsw: core: Fix memory leak on module removal
mlxsw: core: Use variable timeout for EMAD retries
Ilya Dryomov (1):
libceph: clear con->out_msg on Policy::stateful_server faults
Jakub Kicinski (1):
net: vlan: avoid leaks on register_vlan_dev() failures
Jamie Iles (1):
bonding: wait for sysfs kobject destruction before freeing struct
slave
Jan Kara (8):
ext4: Detect already used quota file early
ext4: fix bogus warning in ext4_update_dx_flag()
ext4: Protect superblock modifications with a buffer lock
ext4: fix deadlock with fs freezing and EA inodes
ext4: don't remount read-only with errors=continue on reboot
quota: Don't overflow quota file offsets
bfq: Fix computation of shallow depth
ext4: fix superblock checksum failure when setting password salt
Jann Horn (1):
mm, slub: consider rest of partial list if acquire_slab() fails
Jason A. Donenfeld (3):
netfilter: use actual socket sk rather than skb sk when routing harder
net: introduce skb_list_walk_safe for skb segment walking
net: skbuff: disambiguate argument and member for skb_list_walk_safe
helper
Jeff Dike (1):
virtio_net: Fix recursive call to cpus_read_lock()
Jens Axboe (1):
proc: don't allow async path resolution of /proc/self components
Jesper Dangaard Brouer (1):
netfilter: conntrack: fix reading nf_conntrack_buckets
Jessica Yu (1):
module: delay kobject uevent until after module init call
Jiri Olsa (2):
perf python scripting: Fix printable strings in python3 scripts
perf tools: Add missing swap for ino_generation
Johannes Thumshirn (1):
block: factor out requeue handling from dispatch code
Jonathan Cameron (1):
ACPI: Add out of bounds and numa_off protections to pxm_to_node()
Joseph Qi (1):
ext4: unlock xattr_sem properly in ext4_inline_data_truncate()
Jubin Zhong (1):
PCI: Fix pci_slot_release() NULL pointer dereference
Kaixu Xia (1):
ext4: correctly report "not supported" for {usr, grp}jquota when
!CONFIG_QUOTA
Keqian Zhu (1):
clocksource/drivers/arm_arch_timer: Correct fault programming of
CNTKCTL_EL1.EVNTI
Kirill Tkhai (1):
mm: move nr_deactivate accounting to shrink_active_list()
Lang Dai (1):
uio: free uio id after uio file node is freed
Lecopzer Chen (2):
kasan: fix unaligned address is unhandled in kasan_remove_zero_shadow
kasan: fix incorrect arguments passing in kasan_add_zero_shadow
Lee Duncan (1):
scsi: libiscsi: Fix NOP race condition
Lee Jones (1):
Fonts: Replace discarded const qualifier
Leo Yan (1):
perf lock: Don't free "lock_seq_stat" if read_count isn't zero
Leon Romanovsky (1):
net/mlx5: Properly convey driver version to firmware
Lijie (1):
config: enable CONFIG_NVME_MULTIPATH by default
Liu Shixin (3):
config: set default value of CONFIG_TEST_FREE_PAGES
mm: memcontrol: add struct mem_cgroup_extension
mm: fix kabi broken
Lorenzo Pieralisi (1):
asm-generic/io.h: Fix !CONFIG_GENERIC_IOMAP pci_iounmap()
implementation
Lu Jialin (1):
fs: fix files.usage bug when move tasks
Luc Van Oostenryck (1):
xsk: Fix xsk_poll()'s return type
Luo Meng (2):
ext4: fix invalid inode checksum
fail_function: Remove a redundant mutex unlock
Mao Wenan (1):
net: Update window_clamp if SOCK_RCVBUF is set
Marc Zyngier (2):
arm64: Run ARCH_WORKAROUND_1 enabling code on all CPUs
genirq/irqdomain: Don't try to free an interrupt that has no mapping
Mark Rutland (3):
arm64: syscall: exit userspace before unmasking exceptions
arm64: module: rework special section handling
arm64: module/ftrace: intialize PLT at load time
Martin Wilck (1):
scsi: core: Fix VPD LUN ID designator priorities
Mateusz Nosek (1):
futex: Fix incorrect should_fail_futex() handling
Matteo Croce (4):
Revert "kernel/reboot.c: convert simple_strtoul to kstrtoint"
reboot: fix overflow parsing reboot cpu number
ipv6: create multicast route with RTPROT_KERNEL
ipv6: set multicast flag on the multicast route
Matthew Wilcox (Oracle) (1):
mm/page_alloc.c: fix freeing non-compound pages
Maurizio Lombardi (2):
scsi: target: remove boilerplate code
scsi: target: fix hang when multiple threads try to destroy the same
iscsi session
Maxim Mikityanskiy (1):
net/tls: Protect from calling tls_dev_del for TLS RX twice
Miaohe Lin (1):
mm/hugetlb: fix potential missing huge page size info
Michael Schaller (1):
efivarfs: Replace invalid slashes with exclamation marks in dentries.
Mike Christie (1):
scsi: target: iscsi: Fix cmd abort fabric stop race
Mike Galbraith (1):
futex: Handle transient "ownerless" rtmutex state correctly
Miklos Szeredi (1):
fuse: fix page dereference after free
Mikulas Patocka (3):
dm integrity: fix the maximum number of arguments
dm integrity: fix flush with external metadata device
dm integrity: fix a crash if "recalculate" used without
"internal_hash"
Ming Lei (2):
scsi: core: Don't start concurrent async scan on same host
block: fix use-after-free in disk_part_iter_next
Minwoo Im (1):
nvme: free sq/cq dbbuf pointers when dbbuf set fails
Miroslav Benes (1):
module: set MODULE_STATE_GOING state when a module fails to load
Moshe Shemesh (2):
net/mlx4_en: Avoid scheduling restart task if it is already running
net/mlx4_en: Handle TX error CQE
Naoya Horiguchi (1):
mm, hwpoison: double-check page count in __get_any_page()
Naveen N. Rao (1):
ftrace: Fix updating FTRACE_FL_TRAMP
Neal Cardwell (1):
tcp: fix cwnd-limited bug for TSO deferral where we send nothing
NeilBrown (1):
NFS: switch nfsiod to be an UNBOUND workqueue.
Nicholas Piggin (1):
mm: fix exec activate_mm vs TLB shootdown and lazy tlb switching race
Oleg Nesterov (1):
ptrace: fix task_join_group_stop() for the case when current is traced
Oliver Herms (1):
IPv6: Set SIT tunnel hard_header_len to zero
Paul Moore (1):
selinux: fix inode_doinit_with_dentry() LABEL_INVALID error handling
Paulo Alcantara (1):
cifs: fix potential use-after-free in cifs_echo_request()
Peng Liu (1):
sched/deadline: Fix sched_dl_global_validate()
Peter Zijlstra (2):
serial: pl011: Fix lockdep splat when handling magic-sysrq interrupt
perf: Fix get_recursion_context()
Petr Malat (1):
sctp: Fix COMM_LOST/CANT_STR_ASSOC err reporting on big-endian
platforms
Qian Cai (1):
mm/swapfile: do not sleep with a spin lock held
Qiujun Huang (2):
ring-buffer: Return 0 on success from ring_buffer_resize()
tracing: Fix out of bounds write in get_trace_buf
Rafael J. Wysocki (1):
driver core: Extend device_is_dependent()
Randy Dunlap (1):
net: sched: prevent invalid Scell_log shift count
Ritika Srivastava (2):
block: Return blk_status_t instead of errno codes
block: better deal with the delayed not supported case in
blk_cloned_rq_check_limits
Ronnie Sahlberg (1):
cifs: handle -EINTR in cifs_setattr
Ryan Sharpelletti (1):
tcp: only postpone PROBE_RTT if RTT is < current min_rtt estimate
Sami Tolvanen (1):
arm64: lse: fix LSE atomics with LLVM's integrated assembler
Sean Tranchetti (1):
net: ipv6: fib: flush exceptions when purging route
Shakeel Butt (2):
mm: swap: fix vmstats for huge pages
mm: swap: memcg: fix memcg stats for huge pages
Shijie Luo (1):
mm: mempolicy: fix potential pte_unmap_unlock pte error
Shin'ichiro Kawasaki (1):
uio: Fix use-after-free in uio_unregister_device()
Stefano Brivio (1):
netfilter: ipset: Update byte and packet counters regardless of
whether they match
Steven Rostedt (VMware) (4):
ring-buffer: Fix recursion protection transitions between interrupt
context
ftrace: Fix recursion check for NMI test
ftrace: Handle tracing when switching between context
tracing: Fix userstacktrace option for instances
Subash Abhinov Kasiviswanathan (2):
netfilter: x_tables: Switch synchronization to RCU
netfilter: x_tables: Update remaining dereference to RCU
Sven Eckelmann (2):
vxlan: Add needed_headroom for lower device
vxlan: Copy needed_tailroom from lowerdev
Sylwester Dziedziuch (2):
i40e: Fix removing driver while bare-metal VFs pass traffic
i40e: Fix Error I40E_AQ_RC_EINVAL when removing VFs
Takashi Iwai (1):
libata: transport: Use scnprintf() for avoiding potential buffer
overflow
Tariq Toukan (1):
net: Disable NETIF_F_HW_TLS_RX when RXCSUM is disabled
Thomas Gleixner (12):
sched: Reenable interrupts in do_sched_yield()
futex: Move futex exit handling into futex code
futex: Replace PF_EXITPIDONE with a state
exit/exec: Seperate mm_release()
futex: Split futex_mm_release() for exit/exec
futex: Set task::futex_state to DEAD right after handling futex exit
futex: Mark the begin of futex exit explicitly
futex: Sanitize exit state handling
futex: Provide state handling for exec() as well
futex: Add mutex around futex exit
futex: Provide distinct return value when owner is exiting
futex: Prevent exit livelock
Tianyue Ren (1):
selinux: fix error initialization in inode_doinit_with_dentry()
Trond Myklebust (5):
SUNRPC: xprt_load_transport() needs to support the netid "rdma6"
NFSv4: Fix a pNFS layout related use-after-free race when freeing the
inode
pNFS: Mark layout for return if return-on-close was not sent
NFS/pNFS: Fix a leak of the layout 'plh_outstanding' counter
NFS: nfs_igrab_and_active must first reference the superblock
Tung Nguyen (1):
tipc: fix memory leak caused by tipc_buf_append()
Tyler Hicks (1):
tpm: efi: Don't create binary_bios_measurements file for an empty log
Uwe Kleine-König (1):
spi: fix resource leak for drivers without .remove callback
Vadim Fedorenko (1):
net/tls: missing received data after fast remote close
Valentin Schneider (1):
arm64: topology: Stop using MPIDR for topology information
Vamshi K Sthambamkadi (1):
efivarfs: fix memory leak in efivarfs_create()
Vasily Averin (2):
netfilter: ipset: fix shift-out-of-bounds in htable_bits()
net: drop bogus skb with CHECKSUM_PARTIAL and offset beyond end of
trimmed packet
Vincenzo Frascino (1):
arm64: lse: Fix LSE atomics with LLVM
Vladyslav Tarasiuk (1):
net/mlx5: Disable QoS when min_rates on all VFs are zero
Wang Hai (4):
devlink: Add missing genlmsg_cancel() in
devlink_nl_sb_port_pool_fill()
inet_diag: Fix error path to cancel the meseage in
inet_req_diag_fill()
tipc: fix memory leak in tipc_topsrv_start()
ipv6: addrlabel: fix possible memory leak in ip6addrlbl_net_init
Wang Wensheng (1):
sbsa_gwdt: Add WDIOF_PRETIMEOUT flag to watchdog_info at defination
Wei Li (1):
irqchip/gic-v3: Fix compiling error on ARM32 with GICv3
Wei Yang (1):
mm: thp: don't need care deferred split queue in memcg charge move
path
Weilong Chen (1):
hugetlbfs: Add dependency with ascend memory features
Wengang Wang (1):
ocfs2: initialize ip_next_orphan
Will Deacon (3):
arm64: psci: Avoid printing in cpu_psci_cpu_die()
arm64: pgtable: Fix pte_accessible()
arm64: pgtable: Ensure dirty bit is preserved across pte_wrprotect()
Willem de Bruijn (1):
sock: set sk_err to ee_errno on dequeue from errq
Wu Bo (1):
scsi: libiscsi: fix task hung when iscsid deamon exited
Xie XiuQi (1):
cputime: fix undefined reference to get_idle_time when CONFIG_PROC_FS
disabled
Xin Long (1):
sctp: change to hold/put transport for proto_unreach_timer
Xiongfeng Wang (1):
arm64: fix compile error when CONFIG_HOTPLUG_CPU is disabled
Xiubo Li (1):
nbd: make the config put is called before the notifying the waiter
Xu Qiang (4):
NMI: Enable arm-pmu interrupt as NMI in Acensed.
irqchip/gic-v3-its: Unconditionally save/restore the ITS state on
suspend.
irqchip/irq-gic-v3: Add workaround bindings in device tree to init ts
core GICR.
Document: In the binding document, add enable-init-all-GICR field
description.
Yang Shi (6):
mm: list_lru: set shrinker map bit when child nr_items is not zero
mm: thp: extract split_queue_* into a struct
mm: move mem_cgroup_uncharge out of __page_cache_release()
mm: shrinker: make shrinker not depend on memcg kmem
mm: thp: make deferred split shrinker memcg aware
mm: vmscan: protect shrinker idr replace with CONFIG_MEMCG
Yang Yingliang (5):
arm64: arch_timer: only do cntvct workaround on VDSO path on D05
armv7 fix compile error
Kconfig: disable KTASK by default
futex: sched: fix kabi broken in task_struct
futex: sched: fix UAF when free futex_exit_mutex in free_task()
Yi-Hung Wei (1):
ip_tunnels: Set tunnel option flag when tunnel metadata is present
Yicong Yang (1):
libfs: fix error cast of negative value in simple_attr_write()
Yu Kuai (2):
blk-cgroup: prevent rcu_sched detected stalls warnings in
blkg_destroy_all()
blk-throttle: don't check whether or not lower limit is valid if
CONFIG_BLK_DEV_THROTTLING_LOW is off
Yufen Yu (2):
bdi: fix compiler error in bdi_get_dev_name()
scsi: do quiesce for enclosure driver
Yunfeng Ye (1):
workqueue: Kick a worker based on the actual activation of delayed
works
Yunjian Wang (2):
net: hns: fix return value check in __lb_other_process()
vhost_net: fix ubuf refcount incorrectly when sendmsg fails
Yunsheng Lin (1):
net: sch_generic: fix the missing new qdisc assignment bug
Zeng Tao (1):
time: Prevent undefined behaviour in timespec64_to_ns()
Zhang Changzhong (2):
ah6: fix error return code in ah6_input()
net: bridge: vlan: fix error return code in __vlan_add()
Zhao Heming (2):
md/cluster: block reshape with remote resync job
md/cluster: fix deadlock when node is doing resync job
Zhengyuan Liu (3):
arm64/mm: return cpu_all_mask when node is NUMA_NO_NODE
hifc: remove unnecessary __init specifier
mmap: fix a compiling error for 'MAP_PA32BIT'
Zhou Guanghui (2):
memcg/ascend: Check sysctl oom config for memcg oom
memcg/ascend: enable kmem cgroup by default for ascend
Zqiang (1):
kthread_worker: prevent queuing delayed work from timer_fn when it is
being canceled
chenmaodong (1):
fix virtio_gpu use-after-free while creating dumb
j.nixdorf(a)avm.de (1):
net: sunrpc: interpret the return value of kstrtou32 correctly
lijinlin (1):
ext4: add ext3 report error to userspace by netlink
miaoyubo (1):
KVM: Enable PUD huge mappings only on 1620
yangerkun (1):
ext4: fix bug for rename with RENAME_WHITEOUT
zhuoliang zhang (1):
net: xfrm: fix a race condition during allocing spi
.../interrupt-controller/arm,gic-v3.txt | 4 +
Documentation/networking/bonding.txt | 16 +-
arch/Kconfig | 7 +
arch/alpha/include/uapi/asm/mman.h | 2 +
arch/arm/kernel/perf_event_v7.c | 16 +-
arch/arm64/Kconfig | 2 +
arch/arm64/configs/euleros_defconfig | 1 +
arch/arm64/configs/hulk_defconfig | 1 +
arch/arm64/configs/openeuler_defconfig | 3 +-
arch/arm64/include/asm/atomic_lse.h | 76 ++-
arch/arm64/include/asm/cpufeature.h | 4 +
arch/arm64/include/asm/lse.h | 6 +-
arch/arm64/include/asm/memory.h | 11 +
arch/arm64/include/asm/numa.h | 3 +
arch/arm64/include/asm/pgtable.h | 34 +-
arch/arm64/kernel/cpu_errata.c | 8 +
arch/arm64/kernel/cpufeature.c | 2 +-
arch/arm64/kernel/ftrace.c | 50 +-
arch/arm64/kernel/module.c | 47 +-
arch/arm64/kernel/psci.c | 5 +-
arch/arm64/kernel/setup.c | 5 +-
arch/arm64/kernel/syscall.c | 2 +-
arch/arm64/kernel/topology.c | 43 +-
arch/arm64/lib/memcpy.S | 3 +-
arch/arm64/lib/memmove.S | 3 +-
arch/arm64/lib/memset.S | 3 +-
arch/arm64/mm/init.c | 17 +-
arch/arm64/mm/numa.c | 6 +-
arch/mips/include/uapi/asm/mman.h | 2 +
arch/parisc/include/uapi/asm/mman.h | 2 +
arch/powerpc/include/asm/uaccess.h | 4 +-
arch/powerpc/include/uapi/asm/mman.h | 2 +
arch/sparc/include/uapi/asm/mman.h | 2 +
arch/x86/configs/hulk_defconfig | 1 +
arch/x86/configs/openeuler_defconfig | 1 +
arch/xtensa/include/uapi/asm/mman.h | 2 +
block/bfq-iosched.c | 8 +-
block/blk-cgroup.c | 30 +-
block/blk-core.c | 39 +-
block/blk-mq.c | 29 +-
block/blk-throttle.c | 6 +
block/genhd.c | 9 +-
drivers/acpi/acpi_pnp.c | 3 +
drivers/acpi/internal.h | 2 +-
drivers/acpi/numa.c | 2 +-
drivers/acpi/resource.c | 2 +-
drivers/acpi/scan.c | 17 +-
drivers/ata/libata-transport.c | 10 +-
drivers/base/core.c | 21 +-
drivers/block/nbd.c | 3 +-
drivers/block/null_blk_zoned.c | 20 +-
drivers/char/Kconfig | 2 +-
drivers/char/random.c | 1 -
drivers/char/tpm/eventlog/efi.c | 5 +
drivers/clocksource/arm_arch_timer.c | 36 +-
drivers/firmware/efi/efi.c | 4 -
drivers/firmware/efi/libstub/arm-stub.c | 3 -
drivers/gpu/drm/virtio/virtgpu_gem.c | 4 +-
drivers/irqchip/irq-gic-v3-its.c | 30 +-
drivers/irqchip/irq-gic-v3.c | 18 +-
drivers/md/dm-bufio.c | 6 +
drivers/md/dm-integrity.c | 58 ++-
drivers/md/dm-snap.c | 24 +
drivers/md/dm-table.c | 15 +-
drivers/md/dm-verity-target.c | 12 +-
drivers/md/md-cluster.c | 67 +--
drivers/md/md.c | 14 +-
drivers/mtd/hisilicon/sfc/hrd_sfc_driver.c | 4 +-
drivers/net/bonding/bond_main.c | 92 ++--
drivers/net/bonding/bond_netlink.c | 14 +
drivers/net/bonding/bond_options.c | 71 ++-
drivers/net/bonding/bond_procfs.c | 2 +
drivers/net/bonding/bond_sysfs.c | 13 +
drivers/net/bonding/bond_sysfs_slave.c | 18 +-
.../net/ethernet/hisilicon/hns/hns_ethtool.c | 4 +
drivers/net/ethernet/intel/i40e/i40e.h | 4 +
drivers/net/ethernet/intel/i40e/i40e_main.c | 32 +-
.../ethernet/intel/i40e/i40e_virtchnl_pf.c | 30 +-
drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 24 +-
.../net/ethernet/mellanox/mlx4/en_netdev.c | 21 +-
drivers/net/ethernet/mellanox/mlx4/en_tx.c | 40 +-
drivers/net/ethernet/mellanox/mlx4/mlx4_en.h | 12 +-
.../net/ethernet/mellanox/mlx5/core/en_fs.c | 3 +
.../net/ethernet/mellanox/mlx5/core/eswitch.c | 15 +-
.../net/ethernet/mellanox/mlx5/core/main.c | 6 +-
.../ethernet/mellanox/mlx5/core/pagealloc.c | 21 +-
drivers/net/ethernet/mellanox/mlxsw/core.c | 8 +-
drivers/net/geneve.c | 3 +-
drivers/net/macvlan.c | 5 +-
drivers/net/virtio_net.c | 12 +-
drivers/net/vxlan.c | 3 +
drivers/nvme/host/pci.c | 15 +
drivers/pci/controller/pcie-iproc.c | 10 +-
drivers/pci/pci.c | 14 +-
drivers/pci/slot.c | 6 +-
drivers/scsi/huawei/hifc/unf_common.h | 2 +-
drivers/scsi/huawei/hifc/unf_scsi.c | 23 +
drivers/scsi/libiscsi.c | 33 +-
drivers/scsi/scsi_lib.c | 126 +++--
drivers/scsi/scsi_scan.c | 11 +-
drivers/scsi/scsi_transport_spi.c | 27 +-
drivers/spi/spi.c | 19 +-
drivers/target/iscsi/iscsi_target.c | 96 ++--
drivers/target/iscsi/iscsi_target.h | 1 -
drivers/target/iscsi/iscsi_target_configfs.c | 5 +-
drivers/target/iscsi/iscsi_target_login.c | 5 +-
drivers/tty/serial/amba-pl011.c | 11 +-
drivers/tty/serial/serial_core.c | 4 +
drivers/uio/uio.c | 12 +-
drivers/vfio/pci/vfio_pci.c | 3 +-
drivers/vhost/net.c | 6 +-
drivers/watchdog/sbsa_gwdt.c | 6 +-
fs/cifs/cifs_unicode.c | 8 +-
fs/cifs/connect.c | 2 +
fs/cifs/inode.c | 13 +-
fs/efivarfs/inode.c | 2 +
fs/efivarfs/super.c | 3 +
fs/exec.c | 17 +-
fs/ext4/ext4.h | 6 +-
fs/ext4/ext4_jbd2.c | 1 -
fs/ext4/file.c | 1 +
fs/ext4/inline.c | 1 +
fs/ext4/inode.c | 31 +-
fs/ext4/ioctl.c | 3 +
fs/ext4/mballoc.c | 1 +
fs/ext4/namei.c | 23 +-
fs/ext4/resize.c | 8 +-
fs/ext4/super.c | 32 +-
fs/ext4/xattr.c | 1 +
fs/fcntl.c | 10 +-
fs/filescontrol.c | 73 +--
fs/fuse/dev.c | 28 +-
fs/hugetlbfs/inode.c | 2 +-
fs/libfs.c | 6 +-
fs/lockd/host.c | 20 +-
fs/nfs/inode.c | 2 +-
fs/nfs/internal.h | 12 +-
fs/nfs/nfs4proc.c | 2 +-
fs/nfs/nfs4super.c | 2 +-
fs/nfs/pnfs.c | 40 +-
fs/nfs/pnfs.h | 5 +
fs/nfs_common/grace.c | 6 +-
fs/ocfs2/super.c | 1 +
fs/proc/generic.c | 55 ++-
fs/proc/internal.h | 7 +
fs/proc/proc_net.c | 16 -
fs/proc/self.c | 7 +
fs/quota/quota_tree.c | 8 +-
fs/super.c | 33 +-
fs/xfs/libxfs/xfs_alloc.c | 1 +
fs/xfs/libxfs/xfs_bmap.c | 19 +-
fs/xfs/libxfs/xfs_bmap.h | 2 +-
fs/xfs/libxfs/xfs_rmap.c | 2 +-
fs/xfs/scrub/bmap.c | 10 +-
fs/xfs/scrub/btree.c | 45 +-
fs/xfs/scrub/inode.c | 3 +-
fs/xfs/scrub/refcount.c | 8 +-
fs/xfs/xfs_iops.c | 10 +
fs/xfs/xfs_pnfs.c | 2 +-
fs/xfs/xfs_rtalloc.c | 10 +-
include/asm-generic/io.h | 39 +-
include/linux/backing-dev.h | 1 +
include/linux/blkdev.h | 1 +
include/linux/compat.h | 2 -
include/linux/dm-bufio.h | 1 +
include/linux/efi.h | 7 -
include/linux/futex.h | 39 +-
include/linux/huge_mm.h | 9 +
include/linux/hugetlb.h | 10 +-
include/linux/if_team.h | 5 +-
include/linux/memblock.h | 3 -
include/linux/memcontrol.h | 32 +-
include/linux/mm.h | 2 +
include/linux/mm_types.h | 1 +
include/linux/mman.h | 15 +
include/linux/mmzone.h | 8 +
include/linux/netfilter/x_tables.h | 5 +-
include/linux/netfilter_ipv4.h | 2 +-
include/linux/netfilter_ipv6.h | 2 +-
include/linux/netpoll.h | 10 +-
include/linux/prandom.h | 36 +-
include/linux/proc_fs.h | 8 +-
include/linux/sched.h | 5 +-
include/linux/sched/cputime.h | 5 +
include/linux/sched/mm.h | 6 +-
include/linux/shrinker.h | 7 +-
include/linux/skbuff.h | 5 +
include/linux/sunrpc/xprt.h | 1 +
include/linux/time64.h | 4 +
include/net/bond_options.h | 1 +
include/net/bonding.h | 14 +-
include/net/ip_tunnels.h | 7 +-
include/net/red.h | 4 +-
include/net/tls.h | 6 +
include/scsi/libiscsi.h | 3 +
include/target/iscsi/iscsi_target_core.h | 2 +-
include/uapi/asm-generic/mman.h | 1 +
include/uapi/linux/if_link.h | 1 +
init/Kconfig | 2 +-
kernel/events/internal.h | 2 +-
kernel/exit.c | 35 +-
kernel/fail_function.c | 5 +-
kernel/fork.c | 61 ++-
kernel/futex.c | 291 +++++++++--
kernel/irq/irqdomain.c | 11 +-
kernel/kthread.c | 3 +-
kernel/module.c | 6 +-
kernel/reboot.c | 28 +-
kernel/sched/core.c | 6 +-
kernel/sched/cputime.c | 6 +
kernel/sched/deadline.c | 5 +-
kernel/sched/sched.h | 42 +-
kernel/signal.c | 19 +-
kernel/time/itimer.c | 4 -
kernel/time/tick-common.c | 2 +
kernel/time/timer.c | 7 -
kernel/trace/ftrace.c | 22 +-
kernel/trace/ring_buffer.c | 66 ++-
kernel/trace/trace.c | 9 +-
kernel/trace/trace.h | 32 +-
kernel/trace/trace_selftest.c | 9 +-
kernel/workqueue.c | 13 +-
lib/Kconfig.debug | 9 +
lib/Makefile | 1 +
lib/fonts/font_10x18.c | 2 +-
lib/fonts/font_6x10.c | 2 +-
lib/fonts/font_6x11.c | 2 +-
lib/fonts/font_7x14.c | 2 +-
lib/fonts/font_8x16.c | 2 +-
lib/fonts/font_8x8.c | 2 +-
lib/fonts/font_acorn_8x8.c | 2 +-
lib/fonts/font_mini_4x6.c | 2 +-
lib/fonts/font_pearl_8x8.c | 2 +-
lib/fonts/font_sun12x22.c | 2 +-
lib/fonts/font_sun8x16.c | 2 +-
lib/genalloc.c | 25 +-
lib/random32.c | 462 +++++++++++-------
lib/scatterlist.c | 2 +-
lib/test_free_pages.c | 42 ++
mm/huge_memory.c | 167 +++++--
mm/hugetlb.c | 5 +-
mm/kasan/kasan_init.c | 23 +-
mm/list_lru.c | 10 +-
mm/memblock.c | 11 +-
mm/memcontrol.c | 34 +-
mm/memory-failure.c | 6 +
mm/mempolicy.c | 6 +-
mm/mlock.c | 25 +-
mm/mmap.c | 27 +-
mm/page_alloc.c | 9 +
mm/slub.c | 2 +-
mm/swap.c | 37 +-
mm/swapfile.c | 4 +-
mm/vmscan.c | 64 +--
net/8021q/vlan.c | 3 +-
net/8021q/vlan_dev.c | 5 +-
net/bridge/br_device.c | 1 +
net/bridge/br_netfilter_hooks.c | 7 +-
net/bridge/br_private.h | 5 +-
net/bridge/br_vlan.c | 4 +-
net/ceph/messenger.c | 5 +
net/core/dev.c | 5 +
net/core/devlink.c | 6 +-
net/core/lwt_bpf.c | 8 +-
net/core/net-sysfs.c | 65 ++-
net/core/netpoll.c | 51 +-
net/core/skbuff.c | 23 +-
net/dsa/slave.c | 5 +-
net/ipv4/fib_frontend.c | 2 +-
net/ipv4/gre_demux.c | 2 +-
net/ipv4/inet_diag.c | 4 +-
net/ipv4/ip_output.c | 2 +-
net/ipv4/ip_tunnel.c | 10 +-
net/ipv4/netfilter.c | 12 +-
net/ipv4/netfilter/arp_tables.c | 16 +-
net/ipv4/netfilter/ip_tables.c | 16 +-
net/ipv4/netfilter/ipt_SYNPROXY.c | 2 +-
net/ipv4/netfilter/ipt_rpfilter.c | 2 +-
net/ipv4/netfilter/iptable_mangle.c | 2 +-
net/ipv4/netfilter/nf_nat_l3proto_ipv4.c | 2 +-
net/ipv4/netfilter/nf_reject_ipv4.c | 2 +-
net/ipv4/netfilter/nft_chain_route_ipv4.c | 2 +-
net/ipv4/route.c | 7 +-
net/ipv4/syncookies.c | 9 +-
net/ipv4/tcp.c | 2 +
net/ipv4/tcp_bbr.c | 2 +-
net/ipv4/tcp_cong.c | 5 +
net/ipv4/tcp_input.c | 6 +-
net/ipv4/tcp_output.c | 9 +-
net/ipv4/udp.c | 3 +-
net/ipv6/addrconf.c | 3 +-
net/ipv6/addrlabel.c | 26 +-
net/ipv6/ah6.c | 3 +-
net/ipv6/ip6_fib.c | 5 +-
net/ipv6/ip6_gre.c | 16 +-
net/ipv6/ip6_output.c | 40 +-
net/ipv6/netfilter.c | 6 +-
net/ipv6/netfilter/ip6_tables.c | 16 +-
net/ipv6/netfilter/ip6table_mangle.c | 2 +-
net/ipv6/netfilter/nf_nat_l3proto_ipv6.c | 2 +-
net/ipv6/netfilter/nft_chain_route_ipv6.c | 2 +-
net/ipv6/sit.c | 2 -
net/ipv6/syncookies.c | 10 +-
net/netfilter/ipset/ip_set_core.c | 3 +-
net/netfilter/ipset/ip_set_hash_gen.h | 20 +-
net/netfilter/ipvs/ip_vs_core.c | 4 +-
net/netfilter/nf_conntrack_standalone.c | 3 +
net/netfilter/nf_nat_core.c | 1 +
net/netfilter/nf_tables_api.c | 3 +-
net/netfilter/x_tables.c | 49 +-
net/netfilter/xt_RATEEST.c | 3 +
net/sched/cls_tcindex.c | 8 +-
net/sched/sch_api.c | 3 +-
net/sched/sch_choke.c | 2 +-
net/sched/sch_generic.c | 3 +
net/sched/sch_gred.c | 2 +-
net/sched/sch_netem.c | 9 +-
net/sched/sch_red.c | 2 +-
net/sched/sch_sfq.c | 2 +-
net/sctp/input.c | 4 +-
net/sctp/sm_sideeffect.c | 8 +-
net/sctp/transport.c | 2 +-
net/sunrpc/addr.c | 2 +-
net/sunrpc/xprt.c | 65 ++-
net/sunrpc/xprtrdma/module.c | 1 +
net/sunrpc/xprtrdma/transport.c | 1 +
net/sunrpc/xprtsock.c | 4 +
net/tipc/link.c | 9 +-
net/tipc/msg.c | 5 +-
net/tipc/topsrv.c | 10 +-
net/tls/tls_device.c | 5 +-
net/tls/tls_sw.c | 6 +
net/xdp/xsk.c | 8 +-
net/xfrm/xfrm_state.c | 8 +-
security/lsm_audit.c | 7 +-
security/selinux/hooks.c | 16 +-
security/selinux/ibpkey.c | 4 +-
sound/soc/codecs/Kconfig | 1 +
tools/include/uapi/linux/if_link.h | 1 +
tools/perf/builtin-lock.c | 2 +-
tools/perf/util/print_binary.c | 2 +-
.../scripting-engines/trace-event-python.c | 7 +-
tools/perf/util/session.c | 1 +
virt/kvm/arm/mmu.c | 7 +
344 files changed, 3424 insertions(+), 1701 deletions(-)
create mode 100644 lib/test_free_pages.c
--
2.25.1
1
316
[PATCH 001/316] blk-cgroup: prevent rcu_sched detected stalls warnings in blkg_destroy_all()
by Cheng Jian 22 Feb '21
by Cheng Jian 22 Feb '21
22 Feb '21
From: Yu Kuai <yukuai3(a)huawei.com>
hulk inclusion
category: bugfix
bugzilla: 46357
CVE: NA
---------------------------
test procedures:
a. create 20000 cgroups, and echo "8:0 10000" to
blkio.throttle.write_bps_device
b. echo 1 > /sys/blocd/sda/device/delete
test result:
rcu: INFO: rcu_sched detected stalls on CPUs/tasks: [5/1143]
rcu: 0-...0: (0 ticks this GP) idle=0f2/1/0x4000000000000000 softirq=15507/15507 fq
rcu: (detected by 6, t=60012 jiffies, g=119977, q=27153)
NMI backtrace for cpu 0
CPU: 0 PID: 443 Comm: bash Not tainted 4.19.95-00061-g0bcc83b30eec #63
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS ?-20190727_073836-buildvm-p4
RIP: 0010:blk_throtl_update_limit_valid.isra.0+0x116/0x2a0
Code: 01 00 00 e8 7c dd 74 ff 48 83 bb 78 01 00 00 00 0f 85 54 01 00 00 48 8d bb 88 01 1
RSP: 0018:ffff8881030bf9f0 EFLAGS: 00000046
RAX: 0000000000000000 RBX: ffff8880b4f37080 RCX: ffffffff95da0afe
RDX: dffffc0000000000 RSI: ffff888100373980 RDI: ffff8880b4f37208
RBP: ffff888100deca00 R08: ffffffff9528f951 R09: 0000000000000001
R10: ffffed10159dbf56 R11: ffff8880acedfab3 R12: ffff8880b9fda498
R13: ffff8880b9fda4f4 R14: 0000000000000050 R15: ffffffff98b622c0
FS: 00007feb51c51700(0000) GS:ffff888106200000(0000) knlGS:0000000000000000
CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
CR2: 0000561619547080 CR3: 0000000102bc9000 CR4: 00000000000006f0
DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
Call Trace:
throtl_pd_offline+0x98/0x100
blkg_destroy+0x133/0x630
? blkcg_deactivate_policy+0x2c0/0x2c0
? lock_timer_base+0x65/0x110
blkg_destroy_all+0x7f/0x100
blkcg_exit_queue+0x3f/0xa5
blk_exit_queue+0x69/0xa0
blk_cleanup_queue+0x226/0x360
__scsi_remove_device+0xb4/0x3c0
scsi_remove_device+0x38/0x60
sdev_store_delete+0x74/0x100
? dev_driver_string+0xb0/0xb0
dev_attr_store+0x41/0x70
sysfs_kf_write+0x89/0xc0
kernfs_fop_write+0x1b6/0x2e0
? sysfs_kf_bin_read+0x130/0x130
__vfs_write+0xca/0x420
? kernel_read+0xc0/0xc0
? __alloc_fd+0x16f/0x2d0
? __fd_install+0x95/0x1a0
? handle_mm_fault+0x3e0/0x560
vfs_write+0x11a/0x2f0
ksys_write+0xb9/0x1e0
? __x64_sys_read+0x60/0x60
? kasan_check_write+0x20/0x30
? filp_close+0xb5/0xf0
__x64_sys_write+0x46/0x60
do_syscall_64+0xd9/0x1f0
entry_SYSCALL_64_after_hwframe+0x44/0xa9
The usage of so many blkg is very rare, however, such problem do exist
in theory. In order to avoid such warnings, release 'q->queue_lock' for
a while when a batch of blkg were destroyed.
Signed-off-by: Yu Kuai <yukuai3(a)huawei.com>
Reviewed-by: Tao Hou <houtao1(a)huawei.com>
Signed-off-by: Yang Yingliang <yangyingliang(a)huawei.com>
Signed-off-by: Cheng Jian <cj.chengjian(a)huawei.com>
---
block/blk-cgroup.c | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index e592167449aa..c64f0afa27dc 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -364,16 +364,31 @@ static void blkg_destroy(struct blkcg_gq *blkg)
*/
static void blkg_destroy_all(struct request_queue *q)
{
+#define BLKG_DESTROY_BATCH 4096
struct blkcg_gq *blkg, *n;
+ int count;
lockdep_assert_held(q->queue_lock);
+again:
+ count = BLKG_DESTROY_BATCH;
list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
struct blkcg *blkcg = blkg->blkcg;
spin_lock(&blkcg->lock);
blkg_destroy(blkg);
spin_unlock(&blkcg->lock);
+ /*
+ * If the list is too long, the loop can took a long time,
+ * thus relese the lock for a while when a batch of blkg
+ * were destroyed.
+ */
+ if (!--count) {
+ spin_unlock_irq(q->queue_lock);
+ cond_resched();
+ spin_lock_irq(q->queue_lock);
+ goto again;
+ }
}
q->root_blkg = NULL;
--
2.25.1
1
13