Kernel
Threads by month
- ----- 2026 -----
- January
- ----- 2025 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2024 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2023 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2022 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2021 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2020 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2019 -----
- December
- 55 participants
- 22620 discussions
From: Andreas Gruenbacher <agruenba(a)redhat.com>
stable inclusion
from stable-v6.12.63
commit edb2b255618621dc83d0ec23150e16b2c697077f
category: bugfix
bugzilla: https://atomgit.com/src-openeuler/kernel/issues/12714
CVE: CVE-2025-68356
Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id…
--------------------------------
[ Upstream commit 2c5f4a53476e3cab70adc77b38942c066bd2c17c ]
Function new_inode() returns a new inode with inode->i_mapping->gfp_mask
set to GFP_HIGHUSER_MOVABLE. This value includes the __GFP_FS flag, so
allocations in that address space can recurse into filesystem memory
reclaim. We don't want that to happen because it can consume a
significant amount of stack memory.
Worse than that is that it can also deadlock: for example, in several
places, gfs2_unstuff_dinode() is called inside filesystem transactions.
This calls filemap_grab_folio(), which can allocate a new folio, which
can trigger memory reclaim. If memory reclaim recurses into the
filesystem and starts another transaction, a deadlock will ensue.
To fix these kinds of problems, prevent memory reclaim from recursing
into filesystem code by making sure that the gfp_mask of inode address
spaces doesn't include __GFP_FS.
The "meta" and resource group address spaces were already using GFP_NOFS
as their gfp_mask (which doesn't include __GFP_FS). The default value
of GFP_HIGHUSER_MOVABLE is less restrictive than GFP_NOFS, though. To
avoid being overly limiting, use the default value and only knock off
the __GFP_FS flag. I'm not sure if this will actually make a
difference, but it also shouldn't hurt.
This patch is loosely based on commit ad22c7a043c2 ("xfs: prevent stack
overflows from page cache allocation").
Fixes xfstest generic/273.
Fixes: dc0b9435238c ("gfs: Don't use GFP_NOFS in gfs2_unstuff_dinode")
Reviewed-by: Andrew Price <anprice(a)redhat.com>
Signed-off-by: Andreas Gruenbacher <agruenba(a)redhat.com>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
Conflicts:
fs/gfs2/glock.c
fs/gfs2/ops_fstype.c
[Context Conflicts]
Signed-off-by: Lin Ruifeng <linruifeng4(a)huawei.com>
---
fs/gfs2/glock.c | 5 ++++-
fs/gfs2/inode.c | 15 +++++++++++++++
fs/gfs2/inode.h | 1 +
fs/gfs2/ops_fstype.c | 2 +-
4 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index 7606e983a2ca..8fb9f1bba384 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -1254,10 +1254,13 @@ int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
mapping = gfs2_glock2aspace(gl);
if (mapping) {
+ gfp_t gfp_mask;
+
mapping->a_ops = &gfs2_meta_aops;
mapping->host = s->s_bdev->bd_inode;
mapping->flags = 0;
- mapping_set_gfp_mask(mapping, GFP_NOFS);
+ gfp_mask = mapping_gfp_mask(&sdp->sd_aspace);
+ mapping_set_gfp_mask(mapping, gfp_mask);
mapping->private_data = NULL;
mapping->writeback_index = 0;
}
diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c
index 1cb5ce63fbf6..8627c85c6bf0 100644
--- a/fs/gfs2/inode.c
+++ b/fs/gfs2/inode.c
@@ -89,6 +89,19 @@ static int iget_set(struct inode *inode, void *opaque)
return 0;
}
+void gfs2_setup_inode(struct inode *inode)
+{
+ gfp_t gfp_mask;
+
+ /*
+ * Ensure all page cache allocations are done from GFP_NOFS context to
+ * prevent direct reclaim recursion back into the filesystem and blowing
+ * stacks or deadlocking.
+ */
+ gfp_mask = mapping_gfp_mask(inode->i_mapping);
+ mapping_set_gfp_mask(inode->i_mapping, gfp_mask & ~__GFP_FS);
+}
+
/**
* gfs2_inode_lookup - Lookup an inode
* @sb: The super block
@@ -132,6 +145,7 @@ struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type,
struct gfs2_glock *io_gl;
int extra_flags = 0;
+ gfs2_setup_inode(inode);
error = gfs2_glock_get(sdp, no_addr, &gfs2_inode_glops, CREATE,
&ip->i_gl);
if (unlikely(error))
@@ -684,6 +698,7 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
error = -ENOMEM;
if (!inode)
goto fail_gunlock;
+ gfs2_setup_inode(inode);
ip = GFS2_I(inode);
error = posix_acl_create(dir, &mode, &default_acl, &acl);
diff --git a/fs/gfs2/inode.h b/fs/gfs2/inode.h
index ce70cf26b497..6f7c9ec21bbf 100644
--- a/fs/gfs2/inode.h
+++ b/fs/gfs2/inode.h
@@ -88,6 +88,7 @@ static inline int gfs2_check_internal_file_size(struct inode *inode,
return -EIO;
}
+void gfs2_setup_inode(struct inode *inode);
struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned type,
u64 no_addr, u64 no_formal_ino,
unsigned int blktype);
diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c
index f4c066aa24b9..1f6dd91180e0 100644
--- a/fs/gfs2/ops_fstype.c
+++ b/fs/gfs2/ops_fstype.c
@@ -116,7 +116,7 @@ static struct gfs2_sbd *init_sbd(struct super_block *sb)
mapping->a_ops = &gfs2_rgrp_aops;
mapping->host = sb->s_bdev->bd_inode;
mapping->flags = 0;
- mapping_set_gfp_mask(mapping, GFP_NOFS);
+ gfs2_setup_inode(mapping->host);
mapping->private_data = NULL;
mapping->writeback_index = 0;
--
2.43.0
2
1
*** fix CVE-2025-38062 ***
Jason Gunthorpe (1):
genirq/msi: Store the IOMMU IOVA directly in msi_desc instead of
iommu_cookie
Lin Ruifeng (1):
genirq/msi:: Fix kabi break of struct msi_desc
drivers/iommu/dma-iommu.c | 28 +++++++++++++---------------
include/linux/msi.h | 35 +++++++++++++++--------------------
2 files changed, 28 insertions(+), 35 deletions(-)
--
2.43.0
2
3
[PATCH OLK-6.6] KVM: SVM: Don't skip unrelated instruction if INT3/INTO is replaced
by Lin Ruifeng 29 Jan '26
by Lin Ruifeng 29 Jan '26
29 Jan '26
From: Omar Sandoval <osandov(a)fb.com>
stable inclusion
from stable-v6.12.62
commit 87cc1622c88a4888959d64fa1fc9ba1e264aa3d4
category: bugfix
bugzilla: https://atomgit.com/src-openeuler/kernel/issues/11544
CVE: CVE-2025-68259
Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id…
--------------------------------
commit 4da3768e1820cf15cced390242d8789aed34f54d upstream.
When re-injecting a soft interrupt from an INT3, INT0, or (select) INTn
instruction, discard the exception and retry the instruction if the code
stream is changed (e.g. by a different vCPU) between when the CPU
executes the instruction and when KVM decodes the instruction to get the
next RIP.
As effectively predicted by commit 6ef88d6e36c2 ("KVM: SVM: Re-inject
INT3/INTO instead of retrying the instruction"), failure to verify that
the correct INTn instruction was decoded can effectively clobber guest
state due to decoding the wrong instruction and thus specifying the
wrong next RIP.
The bug most often manifests as "Oops: int3" panics on static branch
checks in Linux guests. Enabling or disabling a static branch in Linux
uses the kernel's "text poke" code patching mechanism. To modify code
while other CPUs may be executing that code, Linux (temporarily)
replaces the first byte of the original instruction with an int3 (opcode
0xcc), then patches in the new code stream except for the first byte,
and finally replaces the int3 with the first byte of the new code
stream. If a CPU hits the int3, i.e. executes the code while it's being
modified, then the guest kernel must look up the RIP to determine how to
handle the #BP, e.g. by emulating the new instruction. If the RIP is
incorrect, then this lookup fails and the guest kernel panics.
The bug reproduces almost instantly by hacking the guest kernel to
repeatedly check a static branch[1] while running a drgn script[2] on
the host to constantly swap out the memory containing the guest's TSS.
[1]: https://gist.github.com/osandov/44d17c51c28c0ac998ea0334edf90b5a
[2]: https://gist.github.com/osandov/10e45e45afa29b11e0c7209247afc00b
Fixes: 6ef88d6e36c2 ("KVM: SVM: Re-inject INT3/INTO instead of retrying the instruction")
Cc: stable(a)vger.kernel.org
Co-developed-by: Sean Christopherson <seanjc(a)google.com>
Signed-off-by: Omar Sandoval <osandov(a)fb.com>
Link: https://patch.msgid.link/1cc6dcdf36e3add7ee7c8d90ad58414eeb6c3d34.176227876…
Signed-off-by: Sean Christopherson <seanjc(a)google.com>
Signed-off-by: Greg Kroah-Hartman <gregkh(a)linuxfoundation.org>
Conflicts:
arch/x86/kvm/svm/svm.c
[Context Conflicts]
Signed-off-by: Lin Ruifeng <linruifeng4(a)huawei.com>
---
arch/x86/include/asm/kvm_host.h | 9 +++++++++
arch/x86/kvm/svm/svm.c | 24 +++++++++++++-----------
arch/x86/kvm/x86.c | 21 +++++++++++++++++++++
3 files changed, 43 insertions(+), 11 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 3936da6febce..e54edd378230 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1974,6 +1974,11 @@ u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
* the gfn, i.e. retrying the instruction will hit a
* !PRESENT fault, which results in a new shadow page
* and sends KVM back to square one.
+ *
+ * EMULTYPE_SKIP_SOFT_INT - Set in combination with EMULTYPE_SKIP to only skip
+ * an instruction if it could generate a given software
+ * interrupt, which must be encoded via
+ * EMULTYPE_SET_SOFT_INT_VECTOR().
*/
#define EMULTYPE_NO_DECODE (1 << 0)
#define EMULTYPE_TRAP_UD (1 << 1)
@@ -1984,6 +1989,10 @@ u64 vcpu_tsc_khz(struct kvm_vcpu *vcpu);
#define EMULTYPE_PF (1 << 6)
#define EMULTYPE_COMPLETE_USER_EXIT (1 << 7)
#define EMULTYPE_WRITE_PF_TO_SP (1 << 8)
+#define EMULTYPE_SKIP_SOFT_INT (1 << 9)
+
+#define EMULTYPE_SET_SOFT_INT_VECTOR(v) ((u32)((v) & 0xff) << 16)
+#define EMULTYPE_GET_SOFT_INT_VECTOR(e) (((e) >> 16) & 0xff)
int kvm_emulate_instruction(struct kvm_vcpu *vcpu, int emulation_type);
int kvm_emulate_instruction_from_buffer(struct kvm_vcpu *vcpu,
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index dfe7ed5e567f..e6b9118943f1 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -384,6 +384,7 @@ static bool svm_can_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type,
void *insn, int insn_len);
static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu,
+ int emul_type,
bool commit_side_effects)
{
struct vcpu_svm *svm = to_svm(vcpu);
@@ -413,7 +414,7 @@ static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu,
if (unlikely(!commit_side_effects))
old_rflags = svm->vmcb->save.rflags;
- if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
+ if (!kvm_emulate_instruction(vcpu, emul_type))
return 0;
if (unlikely(!commit_side_effects))
@@ -431,11 +432,13 @@ static int __svm_skip_emulated_instruction(struct kvm_vcpu *vcpu,
static int svm_skip_emulated_instruction(struct kvm_vcpu *vcpu)
{
- return __svm_skip_emulated_instruction(vcpu, true);
+ return __svm_skip_emulated_instruction(vcpu, EMULTYPE_SKIP, true);
}
-static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu)
+static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu, u8 vector)
{
+ const int emul_type = EMULTYPE_SKIP | EMULTYPE_SKIP_SOFT_INT |
+ EMULTYPE_SET_SOFT_INT_VECTOR(vector);
unsigned long rip, old_rip = kvm_rip_read(vcpu);
struct vcpu_svm *svm = to_svm(vcpu);
@@ -451,7 +454,7 @@ static int svm_update_soft_interrupt_rip(struct kvm_vcpu *vcpu)
* in use, the skip must not commit any side effects such as clearing
* the interrupt shadow or RFLAGS.RF.
*/
- if (!__svm_skip_emulated_instruction(vcpu, !nrips))
+ if (!__svm_skip_emulated_instruction(vcpu, emul_type, !nrips))
return -EIO;
rip = kvm_rip_read(vcpu);
@@ -487,7 +490,7 @@ static void svm_inject_exception(struct kvm_vcpu *vcpu)
kvm_deliver_exception_payload(vcpu, ex);
if (kvm_exception_is_soft(ex->vector) &&
- svm_update_soft_interrupt_rip(vcpu))
+ svm_update_soft_interrupt_rip(vcpu, ex->vector))
return;
svm->vmcb->control.event_inj = ex->vector
@@ -3713,11 +3716,12 @@ static bool svm_set_vnmi_pending(struct kvm_vcpu *vcpu)
static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
{
+ struct kvm_queued_interrupt *intr = &vcpu->arch.interrupt;
struct vcpu_svm *svm = to_svm(vcpu);
u32 type;
- if (vcpu->arch.interrupt.soft) {
- if (svm_update_soft_interrupt_rip(vcpu))
+ if (intr->soft) {
+ if (svm_update_soft_interrupt_rip(vcpu, intr->nr))
return;
type = SVM_EVTINJ_TYPE_SOFT;
@@ -3725,12 +3729,10 @@ static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected)
type = SVM_EVTINJ_TYPE_INTR;
}
- trace_kvm_inj_virq(vcpu->arch.interrupt.nr,
- vcpu->arch.interrupt.soft, reinjected);
+ trace_kvm_inj_virq(intr->nr, intr->soft, reinjected);
++vcpu->stat.irq_injections;
- svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
- SVM_EVTINJ_VALID | type;
+ svm->vmcb->control.event_inj = intr->nr | SVM_EVTINJ_VALID | type;
}
void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode,
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 2139f728aecc..8f4911acde42 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -9023,6 +9023,23 @@ static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt)
return false;
}
+static bool is_soft_int_instruction(struct x86_emulate_ctxt *ctxt,
+ int emulation_type)
+{
+ u8 vector = EMULTYPE_GET_SOFT_INT_VECTOR(emulation_type);
+
+ switch (ctxt->b) {
+ case 0xcc:
+ return vector == BP_VECTOR;
+ case 0xcd:
+ return vector == ctxt->src.val;
+ case 0xce:
+ return vector == OF_VECTOR;
+ default:
+ return false;
+ }
+}
+
/*
* Decode an instruction for emulation. The caller is responsible for handling
* code breakpoints. Note, manually detecting code breakpoints is unnecessary
@@ -9113,6 +9130,10 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
* injecting single-step #DBs.
*/
if (emulation_type & EMULTYPE_SKIP) {
+ if (emulation_type & EMULTYPE_SKIP_SOFT_INT &&
+ !is_soft_int_instruction(ctxt, emulation_type))
+ return 0;
+
if (ctxt->mode != X86EMUL_MODE_PROT64)
ctxt->eip = (u32)ctxt->_eip;
else
--
2.43.0
2
1
[PATCH OLK-6.6] pinctrl: check the return value of pinmux_ops::get_function_name()
by Lin Ruifeng 29 Jan '26
by Lin Ruifeng 29 Jan '26
29 Jan '26
From: Bartosz Golaszewski <bartosz.golaszewski(a)linaro.org>
stable inclusion
from stable-v6.6.112
commit 1a2ea887a5cd7d47bab599f733d89444df018b1a
category: bugfix
bugzilla: https://atomgit.com/src-openeuler/kernel/issues/12323
CVE: CVE-2025-40030
Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id…
--------------------------------
commit 4002ee98c022d671ecc1e4a84029e9ae7d8a5603 upstream.
While the API contract in docs doesn't specify it explicitly, the
generic implementation of the get_function_name() callback from struct
pinmux_ops - pinmux_generic_get_function_name() - can fail and return
NULL. This is already checked in pinmux_check_ops() so add a similar
check in pinmux_func_name_to_selector() instead of passing the returned
pointer right down to strcmp() where the NULL can get dereferenced. This
is normal operation when adding new pinfunctions.
Cc: stable(a)vger.kernel.org
Tested-by: Neil Armstrong <neil.armstrong(a)linaro.org>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski(a)linaro.org>
Signed-off-by: Linus Walleij <linus.walleij(a)linaro.org>
Signed-off-by: Greg Kroah-Hartman <gregkh(a)linuxfoundation.org>
Signed-off-by: Lin Ruifeng <linruifeng4(a)huawei.com>
---
drivers/pinctrl/pinmux.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/pinctrl/pinmux.c b/drivers/pinctrl/pinmux.c
index ab853d6c586b..1c324d216ad7 100644
--- a/drivers/pinctrl/pinmux.c
+++ b/drivers/pinctrl/pinmux.c
@@ -339,7 +339,7 @@ static int pinmux_func_name_to_selector(struct pinctrl_dev *pctldev,
while (selector < nfuncs) {
const char *fname = ops->get_function_name(pctldev, selector);
- if (!strcmp(function, fname))
+ if (fname && !strcmp(function, fname))
return selector;
selector++;
--
2.43.0
2
1
[PATCH OLK-6.6] iommu/amd/pgtbl: Fix possible race while increase page table level
by Lin Ruifeng 29 Jan '26
by Lin Ruifeng 29 Jan '26
29 Jan '26
From: Vasant Hegde <vasant.hegde(a)amd.com>
stable inclusion
from stable-v6.6.108
commit 075abf0b1a958acfbea2435003d228e738e90346
category: bugfix
bugzilla: https://atomgit.com/src-openeuler/kernel/issues/7970
CVE: CVE-2025-39961
Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id…
--------------------------------
[ Upstream commit 1e56310b40fd2e7e0b9493da9ff488af145bdd0c ]
The AMD IOMMU host page table implementation supports dynamic page table levels
(up to 6 levels), starting with a 3-level configuration that expands based on
IOVA address. The kernel maintains a root pointer and current page table level
to enable proper page table walks in alloc_pte()/fetch_pte() operations.
The IOMMU IOVA allocator initially starts with 32-bit address and onces its
exhuasted it switches to 64-bit address (max address is determined based
on IOMMU and device DMA capability). To support larger IOVA, AMD IOMMU
driver increases page table level.
But in unmap path (iommu_v1_unmap_pages()), fetch_pte() reads
pgtable->[root/mode] without lock. So its possible that in exteme corner case,
when increase_address_space() is updating pgtable->[root/mode], fetch_pte()
reads wrong page table level (pgtable->mode). It does compare the value with
level encoded in page table and returns NULL. This will result is
iommu_unmap ops to fail and upper layer may retry/log WARN_ON.
CPU 0 CPU 1
------ ------
map pages unmap pages
alloc_pte() -> increase_address_space() iommu_v1_unmap_pages() -> fetch_pte()
pgtable->root = pte (new root value)
READ pgtable->[mode/root]
Reads new root, old mode
Updates mode (pgtable->mode += 1)
Since Page table level updates are infrequent and already synchronized with a
spinlock, implement seqcount to enable lock-free read operations on the read path.
Fixes: 754265bcab7 ("iommu/amd: Fix race in increase_address_space()")
Reported-by: Alejandro Jimenez <alejandro.j.jimenez(a)oracle.com>
Cc: stable(a)vger.kernel.org
Cc: Joao Martins <joao.m.martins(a)oracle.com>
Cc: Suravee Suthikulpanit <suravee.suthikulpanit(a)amd.com>
Signed-off-by: Vasant Hegde <vasant.hegde(a)amd.com>
Signed-off-by: Joerg Roedel <joerg.roedel(a)amd.com>
[ Adapted pgtable->mode and pgtable->root to use domain->iop.mode and domain->iop.root ]
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
Signed-off-by: Greg Kroah-Hartman <gregkh(a)linuxfoundation.org>
Conflicts:
drivers/iommu/amd/io_pgtable.c
[Context Conflicts]
Signed-off-by: Lin Ruifeng <linruifeng4(a)huawei.com>
---
drivers/iommu/amd/amd_iommu_types.h | 1 +
drivers/iommu/amd/io_pgtable.c | 26 ++++++++++++++++++++++----
2 files changed, 23 insertions(+), 4 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index dec4e5c2b66b..be4a5b16b187 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -551,6 +551,7 @@ struct amd_irte_ops;
container_of((x), struct amd_io_pgtable, pgtbl_cfg)
struct amd_io_pgtable {
+ seqcount_t seqcount; /* Protects root/mode update */
struct io_pgtable_cfg pgtbl_cfg;
struct io_pgtable iop;
int mode;
diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c
index 6c0621f6f572..ca41da139d86 100644
--- a/drivers/iommu/amd/io_pgtable.c
+++ b/drivers/iommu/amd/io_pgtable.c
@@ -17,6 +17,7 @@
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/dma-mapping.h>
+#include <linux/seqlock.h>
#include <asm/barrier.h>
@@ -171,8 +172,11 @@ static bool increase_address_space(struct protection_domain *domain,
*pte = PM_LEVEL_PDE(domain->iop.mode, iommu_virt_to_phys(domain->iop.root));
+ write_seqcount_begin(&domain->iop.seqcount);
domain->iop.root = pte;
domain->iop.mode += 1;
+ write_seqcount_end(&domain->iop.seqcount);
+
amd_iommu_update_and_flush_device_table(domain);
amd_iommu_domain_flush_complete(domain);
@@ -199,6 +203,7 @@ static u64 *alloc_pte(struct protection_domain *domain,
gfp_t gfp,
bool *updated)
{
+ unsigned int seqcount;
int level, end_lvl;
u64 *pte, *page;
@@ -214,8 +219,14 @@ static u64 *alloc_pte(struct protection_domain *domain,
}
- level = domain->iop.mode - 1;
- pte = &domain->iop.root[PM_LEVEL_INDEX(level, address)];
+ do {
+ seqcount = read_seqcount_begin(&domain->iop.seqcount);
+
+ level = domain->iop.mode - 1;
+ pte = &domain->iop.root[PM_LEVEL_INDEX(level, address)];
+ } while (read_seqcount_retry(&domain->iop.seqcount, seqcount));
+
+
address = PAGE_SIZE_ALIGN(address, page_size);
end_lvl = PAGE_SIZE_LEVEL(page_size);
@@ -292,6 +303,7 @@ static u64 *fetch_pte(struct amd_io_pgtable *pgtable,
unsigned long *page_size)
{
int level;
+ unsigned int seqcount;
u64 *pte;
*page_size = 0;
@@ -299,8 +311,12 @@ static u64 *fetch_pte(struct amd_io_pgtable *pgtable,
if (address > PM_LEVEL_SIZE(pgtable->mode))
return NULL;
- level = pgtable->mode - 1;
- pte = &pgtable->root[PM_LEVEL_INDEX(level, address)];
+ do {
+ seqcount = read_seqcount_begin(&pgtable->seqcount);
+ level = pgtable->mode - 1;
+ pte = &pgtable->root[PM_LEVEL_INDEX(level, address)];
+ } while (read_seqcount_retry(&pgtable->seqcount, seqcount));
+
*page_size = PTE_LEVEL_PAGE_SIZE(level);
while (level > 0) {
@@ -591,6 +607,8 @@ static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo
cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE,
cfg->tlb = &v1_flush_ops;
+ seqcount_init(&pgtable->seqcount);
+
pgtable->iop.ops.map_pages = iommu_v1_map_pages;
pgtable->iop.ops.unmap_pages = iommu_v1_unmap_pages;
pgtable->iop.ops.iova_to_phys = iommu_v1_iova_to_phys;
--
2.43.0
2
1
29 Jan '26
From: Frank Li <Frank.Li(a)nxp.com>
stable inclusion
from stable-v5.15.63
commit 8e142744f0e96abc69ccd99e6d6c7eb662267f21
category: bugfix
bugzilla: https://atomgit.com/src-openeuler/kernel/issues/10097
CVE: CVE-2022-50151
Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id…
--------------------------------
[ Upstream commit 8659ab3d936fcf0084676f98b75b317017aa8f82 ]
Warning log:
[ 4.141392] Unexpected gfp: 0x4 (GFP_DMA32). Fixing up to gfp: 0xa20 (GFP_ATOMIC). Fix your code!
[ 4.150340] CPU: 1 PID: 175 Comm: 1-0050 Not tainted 5.15.5-00039-g2fd9ae1b568c #20
[ 4.158010] Hardware name: Freescale i.MX8QXP MEK (DT)
[ 4.163155] Call trace:
[ 4.165600] dump_backtrace+0x0/0x1b0
[ 4.169286] show_stack+0x18/0x68
[ 4.172611] dump_stack_lvl+0x68/0x84
[ 4.176286] dump_stack+0x18/0x34
[ 4.179613] kmalloc_fix_flags+0x60/0x88
[ 4.183550] new_slab+0x334/0x370
[ 4.186878] ___slab_alloc.part.108+0x4d4/0x748
[ 4.191419] __slab_alloc.isra.109+0x30/0x78
[ 4.195702] kmem_cache_alloc+0x40c/0x420
[ 4.199725] dma_pool_alloc+0xac/0x1f8
[ 4.203486] cdns3_allocate_trb_pool+0xb4/0xd0
pool_alloc_page(struct dma_pool *pool, gfp_t mem_flags)
{
...
page = kmalloc(sizeof(*page), mem_flags);
page->vaddr = dma_alloc_coherent(pool->dev, pool->allocation,
&page->dma, mem_flags);
...
}
kmalloc was called with mem_flags, which is passed down in
cdns3_allocate_trb_pool() and have GFP_DMA32 flags.
kmall_fix_flags() report warning.
GFP_DMA32 is not useful at all. dma_alloc_coherent() will handle
DMA memory region correctly by pool->dev. GFP_DMA32 can be removed
safely.
Signed-off-by: Frank Li <Frank.Li(a)nxp.com>
Link: https://lore.kernel.org/r/20220609154456.2871672-1-Frank.Li@nxp.com
Signed-off-by: Greg Kroah-Hartman <gregkh(a)linuxfoundation.org>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
Conflicts:
drivers/usb/cdns3/gadget.c
drivers/usb/cdns3/cdns3-gadget.c
[file name is different]
Signed-off-by: Guo Mengqi <guomengqi3(a)huawei.com>
---
drivers/usb/cdns3/gadget.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/usb/cdns3/gadget.c b/drivers/usb/cdns3/gadget.c
index eeea892248b5..2bd24fb806fb 100644
--- a/drivers/usb/cdns3/gadget.c
+++ b/drivers/usb/cdns3/gadget.c
@@ -237,7 +237,7 @@ int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev,
ring_size,
&priv_ep->trb_pool_dma,
- GFP_DMA32 | GFP_ATOMIC);
+ GFP_ATOMIC);
if (!priv_ep->trb_pool)
return -ENOMEM;
--
2.22.0
2
1
29 Jan '26
From: Frank Li <Frank.Li(a)nxp.com>
stable inclusion
from stable-v5.15.63
commit 8e142744f0e96abc69ccd99e6d6c7eb662267f21
category: bugfix
bugzilla: https://gitee.com/src-openeuler/kernel/issues/ICG794
CVE: CVE-2022-50151
Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id…
--------------------------------
[ Upstream commit 8659ab3d936fcf0084676f98b75b317017aa8f82 ]
Warning log:
[ 4.141392] Unexpected gfp: 0x4 (GFP_DMA32). Fixing up to gfp: 0xa20 (GFP_ATOMIC). Fix your code!
[ 4.150340] CPU: 1 PID: 175 Comm: 1-0050 Not tainted 5.15.5-00039-g2fd9ae1b568c #20
[ 4.158010] Hardware name: Freescale i.MX8QXP MEK (DT)
[ 4.163155] Call trace:
[ 4.165600] dump_backtrace+0x0/0x1b0
[ 4.169286] show_stack+0x18/0x68
[ 4.172611] dump_stack_lvl+0x68/0x84
[ 4.176286] dump_stack+0x18/0x34
[ 4.179613] kmalloc_fix_flags+0x60/0x88
[ 4.183550] new_slab+0x334/0x370
[ 4.186878] ___slab_alloc.part.108+0x4d4/0x748
[ 4.191419] __slab_alloc.isra.109+0x30/0x78
[ 4.195702] kmem_cache_alloc+0x40c/0x420
[ 4.199725] dma_pool_alloc+0xac/0x1f8
[ 4.203486] cdns3_allocate_trb_pool+0xb4/0xd0
pool_alloc_page(struct dma_pool *pool, gfp_t mem_flags)
{
...
page = kmalloc(sizeof(*page), mem_flags);
page->vaddr = dma_alloc_coherent(pool->dev, pool->allocation,
&page->dma, mem_flags);
...
}
kmalloc was called with mem_flags, which is passed down in
cdns3_allocate_trb_pool() and have GFP_DMA32 flags.
kmall_fix_flags() report warning.
GFP_DMA32 is not useful at all. dma_alloc_coherent() will handle
DMA memory region correctly by pool->dev. GFP_DMA32 can be removed
safely.
Signed-off-by: Frank Li <Frank.Li(a)nxp.com>
Link: https://lore.kernel.org/r/20220609154456.2871672-1-Frank.Li@nxp.com
Signed-off-by: Greg Kroah-Hartman <gregkh(a)linuxfoundation.org>
Signed-off-by: Sasha Levin <sashal(a)kernel.org>
Conflicts:
drivers/usb/cdns3/gadget.c
drivers/usb/cdns3/cdns3-gadget.c
[file name is different]
Signed-off-by: Guo Mengqi <guomengqi3(a)huawei.com>
---
drivers/usb/cdns3/gadget.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/usb/cdns3/gadget.c b/drivers/usb/cdns3/gadget.c
index eeea892248b5..2bd24fb806fb 100644
--- a/drivers/usb/cdns3/gadget.c
+++ b/drivers/usb/cdns3/gadget.c
@@ -237,7 +237,7 @@ int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev,
ring_size,
&priv_ep->trb_pool_dma,
- GFP_DMA32 | GFP_ATOMIC);
+ GFP_ATOMIC);
if (!priv_ep->trb_pool)
return -ENOMEM;
--
2.22.0
2
2
[PATCH v1 openEuler-26.09] Add copy to/from/in user with vectorization support
by Nikita Panov 29 Jan '26
by Nikita Panov 29 Jan '26
29 Jan '26
From: Artem Kuzin <artem.kuzin(a)huawei.com>
kunpeng inclusion
category: feature
bugzilla: https://atomgit.com/openeuler/kernel/issues/8445
-------------------------------------------------
1. This implementation uses st1/ld1 4-vector instructions which allow to copy 64 bytes at once
2. Copy code is used only if size of data block to copy is more than 128 bytes
4. To use this functionality you need to set configuration switch CONFIG_USE_VECTORIZED_COPY=y
5. Code can be used on any ARMv8 variant
6. In kernel copy functions like memcpy are not supported now, but can be enabled in future
7. For now we use lightweght version of register context saving/restoration (4-registers)
We introduce support of vectorization for copy_from/to/in_user functions. Nowadays it
works in parallel with original FPSIMD/SVE vectorization and doesn't affect it anyhow.
We have special flag in task struct - TIF_KERNEL_FPSIMD, that set if currently we use
lightweight vectorization in kernel. Task struct has been updated by two fields:
user space fpsimd state and kernel fpsimd state. User space fpsimd state used by
kernel_fpsimd_begin(), kernel_fpsimd_end() functions that wrap lightweight FPSIMD
contexts usage in kernel space. Kernel fpsimd state is used to manage threads switch.
Now there is no support of nested calls of kernel_neon_begin()/kernel_fpsimd_begin()
and there is no plans to support this in future. This is not necessary.
We save lightweight FPSIMD context in kernel_fpsimd_begin(), and restore it in
/kernel_fpsimd_end(). On thread switch we preserve kernel FPSIMD context and restore
user space one if any. This prevens curruption of user space FPSIMD state. Before
switching to the next thread we restore it's kernel FPSIMD context if any.
It is allowed to use FPSIMD in bottom halves, due to in case of BH preemption we check
TIF_KERNEL_FPSIMD flag and save/restore contexts.
Context management if quite lightweight and executed only in case of TIF_KERNEL_FPSIMD
flag is set.
To enable this feature, you need to manually modify one of the
appropriate entries:
/proc/sys/vm/copy_from_user_threshold
/proc/sys/vm/copy_in_user_threshold
/proc/sys/vm/copy_to_user_threshold
Allowed values are following:
-1 - feature enabled
0 - feature always enabled
n (n >0) - feature enabled, if copied size is greater than n KB.
P.S.:
What I am personally don't like in current approach:
1. Additional fields and flag in task struct look quite ugly
2. No way to configure the size of chunk to copy using FPSIMD from user space
3. FPSIMD-based memory movement is not generic, need to enable for memmove(),
memcpy() and friends in future.
Co-developed-by: Alexander Kozhevnikov <alexander.kozhevnikov(a)huawei-partners.com>
Signed-off-by: Alexander Kozhevnikov <alexander.kozhevnikov(a)huawei-partners.com>
Co-developed-by: Nikita Panov <panov.nikita(a)huawei.com>
Signed-off-by: Nikita Panov <panov.nikita(a)huawei.com>
Signed-off-by: Artem Kuzin <artem.kuzin(a)huawei.com>
---
arch/arm64/Kconfig | 15 ++
arch/arm64/configs/openeuler_defconfig | 2 +
arch/arm64/include/asm/fpsimd.h | 15 ++
arch/arm64/include/asm/fpsimdmacros.h | 14 ++
arch/arm64/include/asm/neon.h | 28 ++++
arch/arm64/include/asm/processor.h | 10 ++
arch/arm64/include/asm/thread_info.h | 5 +
arch/arm64/include/asm/uaccess.h | 218 ++++++++++++++++++++++++-
arch/arm64/kernel/entry-fpsimd.S | 22 +++
arch/arm64/kernel/fpsimd.c | 102 +++++++++++-
arch/arm64/kernel/process.c | 2 +-
arch/arm64/lib/copy_from_user.S | 30 ++++
arch/arm64/lib/copy_template_fpsimd.S | 180 ++++++++++++++++++++
arch/arm64/lib/copy_to_user.S | 30 ++++
kernel/softirq.c | 34 ++++
kernel/sysctl.c | 34 ++++
16 files changed, 734 insertions(+), 7 deletions(-)
create mode 100644 arch/arm64/lib/copy_template_fpsimd.S
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index c3b38c890b45..8904e6476e3b 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1870,6 +1870,21 @@ config ARM64_ILP32
is an ABI where long and pointers are 32bits but it uses the AARCH64
instruction set.
+config USE_VECTORIZED_COPY
+ bool "Use vectorized instructions in copy_to/from user"
+ depends on KERNEL_MODE_NEON
+ default y
+ help
+ This option turns on vectorization to speed up copy_to/from_user routines.
+
+config VECTORIZED_COPY_VALIDATE
+ bool "Validate result of vectorized copy using regular implementation"
+ depends on KERNEL_MODE_NEON
+ depends on USE_VECTORIZED_COPY
+ default n
+ help
+ This option turns on vectorization to speed up copy_to/from_user routines.
+
menuconfig AARCH32_EL0
bool "Kernel support for 32-bit EL0"
depends on ARM64_4K_PAGES || EXPERT
diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig
index 9e7bc82cba3a..9843dec071bf 100644
--- a/arch/arm64/configs/openeuler_defconfig
+++ b/arch/arm64/configs/openeuler_defconfig
@@ -527,6 +527,8 @@ CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY=y
# CONFIG_RODATA_FULL_DEFAULT_ENABLED is not set
# CONFIG_ARM64_SW_TTBR0_PAN is not set
CONFIG_ARM64_TAGGED_ADDR_ABI=y
+CONFIG_USE_VECTORIZED_COPY=y
+# CONFIG_VECTORIZED_COPY_VALIDATE is not set
CONFIG_AARCH32_EL0=y
# CONFIG_KUSER_HELPERS is not set
# CONFIG_COMPAT_ALIGNMENT_FIXUPS is not set
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index b6c6949984d8..1fc9089b4a47 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -46,6 +46,21 @@
struct task_struct;
+#ifdef CONFIG_USE_VECTORIZED_COPY
+extern void fpsimd_save_state_light(struct fpsimd_state *state);
+extern void fpsimd_load_state_light(struct fpsimd_state *state);
+#else
+static inline void fpsimd_save_state_light(struct fpsimd_state *state)
+{
+ (void) state;
+}
+
+static inline void fpsimd_load_state_light(struct fpsimd_state *state)
+{
+ (void) state;
+}
+#endif
+
extern void fpsimd_save_state(struct user_fpsimd_state *state);
extern void fpsimd_load_state(struct user_fpsimd_state *state);
diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h
index cdf6a35e3994..df9d3ed91931 100644
--- a/arch/arm64/include/asm/fpsimdmacros.h
+++ b/arch/arm64/include/asm/fpsimdmacros.h
@@ -8,6 +8,20 @@
#include <asm/assembler.h>
+#ifdef CONFIG_USE_VECTORIZED_COPY
+/* Lightweight fpsimd context saving/restoration.
+ * Necessary for vectorized kernel memory movement
+ * implementation
+ */
+.macro fpsimd_save_light state
+ st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [\state]
+.endm
+
+.macro fpsimd_restore_light state
+ ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [\state]
+.endm
+#endif
+
.macro fpsimd_save state, tmpnr
stp q0, q1, [\state, #16 * 0]
stp q2, q3, [\state, #16 * 2]
diff --git a/arch/arm64/include/asm/neon.h b/arch/arm64/include/asm/neon.h
index d4b1d172a79b..ab84b194d7b3 100644
--- a/arch/arm64/include/asm/neon.h
+++ b/arch/arm64/include/asm/neon.h
@@ -16,4 +16,32 @@
void kernel_neon_begin(void);
void kernel_neon_end(void);
+#ifdef CONFIG_USE_VECTORIZED_COPY
+bool kernel_fpsimd_begin(void);
+void kernel_fpsimd_end(void);
+/* Functions to use in non-preemptible context */
+void _kernel_fpsimd_save(struct fpsimd_state *state);
+void _kernel_fpsimd_load(struct fpsimd_state *state);
+#else
+bool kernel_fpsimd_begin(void)
+{
+ return false;
+}
+
+void kernel_fpsimd_end(void)
+{
+}
+
+/* Functions to use in non-preemptible context */
+void _kernel_fpsimd_save(struct fpsimd_state *state)
+{
+ (void) state;
+}
+
+void _kernel_fpsimd_load(struct fpsimd_state *state)
+{
+ (void) state;
+}
+#endif
+
#endif /* ! __ASM_NEON_H */
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 9e688b1b13d4..9b81dbcd2126 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -153,6 +153,10 @@ struct cpu_context {
unsigned long pc;
};
+struct fpsimd_state {
+ __uint128_t v[4];
+};
+
struct thread_struct {
struct cpu_context cpu_context; /* cpu context */
@@ -196,6 +200,12 @@ struct thread_struct {
KABI_RESERVE(6)
KABI_RESERVE(7)
KABI_RESERVE(8)
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ KABI_EXTEND(
+ struct fpsimd_state ustate;
+ struct fpsimd_state kstate;
+ )
+#endif
};
static inline unsigned int thread_get_vl(struct thread_struct *thread,
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index 379d24059f5b..60d0be8a2d58 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -89,6 +89,9 @@ void arch_setup_new_exec(void);
#define TIF_SME 27 /* SME in use */
#define TIF_SME_VL_INHERIT 28 /* Inherit SME vl_onexec across exec */
#define TIF_32BIT_AARCH64 29 /* 32 bit process on AArch64(ILP32) */
+#define TIF_KERNEL_FPSIMD 31 /* Use FPSIMD in kernel */
+#define TIF_PRIV_UACC_ENABLED 32 /* Whether priviliged uaccess was manually enabled */
+
#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
@@ -107,6 +110,8 @@ void arch_setup_new_exec(void);
#define _TIF_MTE_ASYNC_FAULT (1 << TIF_MTE_ASYNC_FAULT)
#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)
#define _TIF_32BIT_AARCH64 (1 << TIF_32BIT_AARCH64)
+#define _TIF_KERNEL_FPSIMD (1 << TIF_KERNEL_FPSIMD)
+#define _TIF_PRIV_UACC_ENABLED (1 << TIF_PRIV_UACC_ENABLED)
#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
_TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index dd0877a75922..fc9f1a40624d 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -26,6 +26,10 @@
#include <asm/memory.h>
#include <asm/extable.h>
+#ifndef __GENKSYMS__
+#include <asm/neon.h>
+#endif
+
static inline int __access_ok(const void __user *ptr, unsigned long size);
/*
@@ -134,7 +138,7 @@ static inline void __uaccess_enable_hw_pan(void)
CONFIG_ARM64_PAN));
}
-static inline void uaccess_disable_privileged(void)
+static inline void __uaccess_disable_privileged(void)
{
mte_disable_tco();
@@ -144,7 +148,22 @@ static inline void uaccess_disable_privileged(void)
__uaccess_enable_hw_pan();
}
-static inline void uaccess_enable_privileged(void)
+static inline void uaccess_disable_privileged(void)
+{
+ preempt_disable();
+
+ if (!test_and_clear_thread_flag(TIF_PRIV_UACC_ENABLED)) {
+ WARN_ON(1);
+ preempt_enable();
+ return;
+ }
+
+ __uaccess_disable_privileged();
+
+ preempt_enable();
+}
+
+static inline void __uaccess_enable_privileged(void)
{
mte_enable_tco();
@@ -154,6 +173,47 @@ static inline void uaccess_enable_privileged(void)
__uaccess_disable_hw_pan();
}
+static inline void uaccess_enable_privileged(void)
+{
+ preempt_disable();
+
+ if (test_and_set_thread_flag(TIF_PRIV_UACC_ENABLED)) {
+ WARN_ON(1);
+ preempt_enable();
+ return;
+ }
+
+ __uaccess_enable_privileged();
+
+ preempt_enable();
+}
+
+static inline void uaccess_priviliged_context_switch(struct task_struct *next)
+{
+ bool curr_enabled = !!test_thread_flag(TIF_PRIV_UACC_ENABLED);
+ bool next_enabled = !!test_ti_thread_flag(&next->thread_info, TIF_PRIV_UACC_ENABLED);
+
+ if (curr_enabled == next_enabled)
+ return;
+
+ if (curr_enabled)
+ __uaccess_disable_privileged();
+ else
+ __uaccess_enable_privileged();
+}
+
+static inline void uaccess_priviliged_state_save(void)
+{
+ if (test_thread_flag(TIF_PRIV_UACC_ENABLED))
+ __uaccess_disable_privileged();
+}
+
+static inline void uaccess_priviliged_state_restore(void)
+{
+ if (test_thread_flag(TIF_PRIV_UACC_ENABLED))
+ __uaccess_enable_privileged();
+}
+
/*
* Sanitize a uaccess pointer such that it cannot reach any kernel address.
*
@@ -391,7 +451,97 @@ do { \
} while (0); \
} while(0)
-extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
+#define USER_COPY_CHUNK_SIZE 4096
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+
+extern int sysctl_copy_from_user_threshold;
+
+#define verify_fpsimd_copy(to, from, n, ret) \
+({ \
+ unsigned long __verify_ret = 0; \
+ __verify_ret = memcmp(to, from, ret ? n - ret : n); \
+ if (__verify_ret) \
+ pr_err("FPSIMD:%s inconsistent state\n", __func__); \
+ if (ret) \
+ pr_err("FPSIMD:%s failed to copy data, expected=%lu, copied=%lu\n", __func__, n, n - ret); \
+ __verify_ret |= ret; \
+ __verify_ret; \
+})
+
+#define compare_fpsimd_copy(to, from, n, ret_fpsimd, ret) \
+({ \
+ unsigned long __verify_ret = 0; \
+ __verify_ret = memcmp(to, from, ret ? n - ret : n); \
+ if (__verify_ret) \
+ pr_err("FIXUP:%s inconsistent state\n", __func__); \
+ if (ret) \
+ pr_err("FIXUP:%s failed to copy data, expected=%lu, copied=%lu\n", __func__, n, n - ret); \
+ __verify_ret |= ret; \
+ if (ret_fpsimd != ret) { \
+ pr_err("FIXUP:%s difference between FPSIMD %lu and regular %lu\n", __func__, n - ret_fpsimd, n - ret); \
+ __verify_ret |= 1; \
+ } else { \
+ __verify_ret = 0; \
+ } \
+ __verify_ret; \
+})
+
+extern unsigned long __must_check
+__arch_copy_from_user(void *to, const void __user *from, unsigned long n);
+
+extern unsigned long __must_check
+__arch_copy_from_user_fpsimd(void *to, const void __user *from, unsigned long n);
+
+static __always_inline unsigned long __must_check
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+ unsigned long __acfu_ret;
+
+ if (sysctl_copy_from_user_threshold == -1 || n < sysctl_copy_from_user_threshold) {
+ uaccess_ttbr0_enable();
+ __acfu_ret = __arch_copy_from_user(to,
+ __uaccess_mask_ptr(from), n);
+ uaccess_ttbr0_disable();
+ } else {
+ if (kernel_fpsimd_begin()) {
+ unsigned long __acfu_ret_fpsimd;
+
+ uaccess_enable_privileged();
+ __acfu_ret_fpsimd = __arch_copy_from_user_fpsimd((to),
+ __uaccess_mask_ptr(from), n);
+ uaccess_disable_privileged();
+
+ __acfu_ret = __acfu_ret_fpsimd;
+ kernel_fpsimd_end();
+#ifdef CONFIG_VECTORIZED_COPY_VALIDATE
+ if (verify_fpsimd_copy(to, __uaccess_mask_ptr(from), n,
+ __acfu_ret)) {
+
+ uaccess_ttbr0_enable();
+ __acfu_ret = __arch_copy_from_user((to),
+ __uaccess_mask_ptr(from), n);
+ uaccess_ttbr0_disable();
+
+ compare_fpsimd_copy(to, __uaccess_mask_ptr(from), n,
+ __acfu_ret_fpsimd, __acfu_ret);
+ }
+#endif
+ } else {
+ uaccess_ttbr0_enable();
+ __acfu_ret = __arch_copy_from_user((to),
+ __uaccess_mask_ptr(from), n);
+ uaccess_ttbr0_disable();
+ }
+ }
+
+
+ return __acfu_ret;
+}
+#else
+extern unsigned long __must_check
+__arch_copy_from_user(void *to, const void __user *from, unsigned long n);
+
#define raw_copy_from_user(to, from, n) \
({ \
unsigned long __acfu_ret; \
@@ -402,7 +552,66 @@ extern unsigned long __must_check __arch_copy_from_user(void *to, const void __u
__acfu_ret; \
})
-extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
+#endif
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+
+extern int sysctl_copy_to_user_threshold;
+
+extern unsigned long __must_check
+__arch_copy_to_user(void __user *to, const void *from, unsigned long n);
+
+extern unsigned long __must_check
+__arch_copy_to_user_fpsimd(void __user *to, const void *from, unsigned long n);
+
+static __always_inline unsigned long __must_check
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+ unsigned long __actu_ret;
+
+
+ if (sysctl_copy_to_user_threshold == -1 || n < sysctl_copy_to_user_threshold) {
+ uaccess_ttbr0_enable();
+ __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),
+ from, n);
+ uaccess_ttbr0_disable();
+ } else {
+ if (kernel_fpsimd_begin()) {
+ unsigned long __actu_ret_fpsimd;
+
+ uaccess_enable_privileged();
+ __actu_ret_fpsimd = __arch_copy_to_user_fpsimd(__uaccess_mask_ptr(to),
+ from, n);
+ uaccess_disable_privileged();
+
+ kernel_fpsimd_end();
+ __actu_ret = __actu_ret_fpsimd;
+#ifdef CONFIG_VECTORIZED_COPY_VALIDATE
+ if (verify_fpsimd_copy(__uaccess_mask_ptr(to), from, n,
+ __actu_ret)) {
+ uaccess_ttbr0_enable();
+ __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),
+ from, n);
+ uaccess_ttbr0_disable();
+
+ compare_fpsimd_copy(__uaccess_mask_ptr(to), from, n,
+ __actu_ret_fpsimd, __actu_ret);
+ }
+#endif
+ } else {
+ uaccess_ttbr0_enable();
+ __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),
+ from, n);
+ uaccess_ttbr0_disable();
+ }
+ }
+
+ return __actu_ret;
+}
+#else
+extern unsigned long __must_check
+__arch_copy_to_user(void __user *to, const void *from, unsigned long n);
+
#define raw_copy_to_user(to, from, n) \
({ \
unsigned long __actu_ret; \
@@ -412,6 +621,7 @@ extern unsigned long __must_check __arch_copy_to_user(void __user *to, const voi
uaccess_ttbr0_disable(); \
__actu_ret; \
})
+#endif
static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len)
{
diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S
index 6325db1a2179..6660465f1b7c 100644
--- a/arch/arm64/kernel/entry-fpsimd.S
+++ b/arch/arm64/kernel/entry-fpsimd.S
@@ -11,6 +11,28 @@
#include <asm/assembler.h>
#include <asm/fpsimdmacros.h>
+#ifdef CONFIG_USE_VECTORIZED_COPY
+/*
+ * Save the FP registers.
+ *
+ * x0 - pointer to struct fpsimd_state_light
+ */
+SYM_FUNC_START(fpsimd_save_state_light)
+ fpsimd_save_light x0
+ ret
+SYM_FUNC_END(fpsimd_save_state_light)
+
+/*
+ * Load the FP registers.
+ *
+ * x0 - pointer to struct fpsimd_state_light
+ */
+SYM_FUNC_START(fpsimd_load_state_light)
+ fpsimd_restore_light x0
+ ret
+SYM_FUNC_END(fpsimd_load_state_light)
+#endif
+
/*
* Save the FP registers.
*
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 998906b75075..1b6b1accfbbc 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -1579,6 +1579,11 @@ void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs)
current);
}
+#ifdef CONFIG_USE_VECTORIZED_COPY
+static void kernel_fpsimd_rollback_changes(void);
+static void kernel_fpsimd_restore_changes(struct task_struct *tsk);
+#endif
+
void fpsimd_thread_switch(struct task_struct *next)
{
bool wrong_task, wrong_cpu;
@@ -1587,10 +1592,11 @@ void fpsimd_thread_switch(struct task_struct *next)
return;
__get_cpu_fpsimd_context();
-
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ kernel_fpsimd_rollback_changes();
+#endif
/* Save unsaved fpsimd state, if any: */
fpsimd_save();
-
/*
* Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
* state. For kernel threads, FPSIMD registers are never loaded
@@ -1603,6 +1609,9 @@ void fpsimd_thread_switch(struct task_struct *next)
update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
wrong_task || wrong_cpu);
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ kernel_fpsimd_restore_changes(next);
+#endif
__put_cpu_fpsimd_context();
}
@@ -1933,6 +1942,95 @@ void kernel_neon_end(void)
}
EXPORT_SYMBOL_GPL(kernel_neon_end);
+#ifdef CONFIG_USE_VECTORIZED_COPY
+bool kernel_fpsimd_begin(void)
+{
+ if (WARN_ON(!system_capabilities_finalized()) ||
+ !system_supports_fpsimd() ||
+ in_irq() || irqs_disabled() || in_nmi())
+ return false;
+
+ preempt_disable();
+ if (test_and_set_thread_flag(TIF_KERNEL_FPSIMD)) {
+ preempt_enable();
+
+ WARN_ON(1);
+ return false;
+ }
+
+ /*
+ * Leaving streaming mode enabled will cause issues for any kernel
+ * NEON and leaving streaming mode or ZA enabled may increase power
+ * consumption.
+ */
+ if (system_supports_sme())
+ sme_smstop();
+
+ fpsimd_save_state_light(¤t->thread.ustate);
+ preempt_enable();
+
+ return true;
+}
+EXPORT_SYMBOL(kernel_fpsimd_begin);
+
+void kernel_fpsimd_end(void)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ preempt_disable();
+ if (test_and_clear_thread_flag(TIF_KERNEL_FPSIMD))
+ fpsimd_load_state_light(¤t->thread.ustate);
+
+ preempt_enable();
+}
+EXPORT_SYMBOL(kernel_fpsimd_end);
+
+void _kernel_fpsimd_save(struct fpsimd_state *state)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ BUG_ON(preemptible());
+ if (test_thread_flag(TIF_KERNEL_FPSIMD))
+ fpsimd_save_state_light(state);
+}
+
+void _kernel_fpsimd_load(struct fpsimd_state *state)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ BUG_ON(preemptible());
+ if (test_thread_flag(TIF_KERNEL_FPSIMD))
+ fpsimd_load_state_light(state);
+}
+
+static void kernel_fpsimd_rollback_changes(void)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ BUG_ON(preemptible());
+ if (test_thread_flag(TIF_KERNEL_FPSIMD)) {
+ fpsimd_save_state_light(¤t->thread.kstate);
+ fpsimd_load_state_light(¤t->thread.ustate);
+ }
+}
+
+static void kernel_fpsimd_restore_changes(struct task_struct *tsk)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ BUG_ON(preemptible());
+ if (test_ti_thread_flag(task_thread_info(tsk), TIF_KERNEL_FPSIMD)) {
+ fpsimd_save_state_light(&tsk->thread.ustate);
+ fpsimd_load_state_light(&tsk->thread.kstate);
+ }
+}
+#endif
+
#ifdef CONFIG_EFI
static DEFINE_PER_CPU(struct user_fpsimd_state, efi_fpsimd_state);
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index e9e5ce956f15..fd895189cb7e 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -529,7 +529,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
struct task_struct *next)
{
struct task_struct *last;
-
+ uaccess_priviliged_context_switch(next);
fpsimd_thread_switch(next);
tls_thread_switch(next);
hw_breakpoint_thread_switch(next);
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 34e317907524..60dc63e10233 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -71,3 +71,33 @@ USER(9998f, ldtrb tmp1w, [srcin])
ret
SYM_FUNC_END(__arch_copy_from_user)
EXPORT_SYMBOL(__arch_copy_from_user)
+
+
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ .macro ldsve reg1, reg2, reg3, reg4, ptr
+ USER(9997f, ld1 {\reg1, \reg2, \reg3, \reg4}, [\ptr])
+ .endm
+
+ .macro stsve reg1, reg2, reg3, reg4, ptr
+ KERNEL_ME_SAFE(9998f, st1 {\reg1, \reg2, \reg3, \reg4}, [\ptr])
+ .endm
+
+SYM_FUNC_START(__arch_copy_from_user_fpsimd)
+ add end, x0, x2
+ mov srcin, x1
+#include "copy_template_fpsimd.S"
+ mov x0, #0 // Nothing to copy
+ ret
+
+ // Exception fixups
+9997: cmp dst, dstin
+ b.ne 9998f
+ // Before being absolutely sure we couldn't copy anything, try harder
+USER(9998f, ldtrb tmp1w, [srcin])
+ strb tmp1w, [dst], #1
+9998: sub x0, end, dst // bytes not copied
+ ret
+SYM_FUNC_END(__arch_copy_from_user_fpsimd)
+EXPORT_SYMBOL(__arch_copy_from_user_fpsimd)
+#endif
\ No newline at end of file
diff --git a/arch/arm64/lib/copy_template_fpsimd.S b/arch/arm64/lib/copy_template_fpsimd.S
new file mode 100644
index 000000000000..9b2e7ce1e4d2
--- /dev/null
+++ b/arch/arm64/lib/copy_template_fpsimd.S
@@ -0,0 +1,180 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
+ */
+
+/*
+ * Copy a buffer from src to dest (alignment handled by the hardware)
+ *
+ * Parameters:
+ * x0 - dest
+ * x1 - src
+ * x2 - n
+ * Returns:
+ * x0 - dest
+ */
+dstin .req x0
+src .req x1
+count .req x2
+tmp1 .req x3
+tmp1w .req w3
+tmp2 .req x4
+tmp2w .req w4
+dst .req x6
+
+A_l .req x7
+A_h .req x8
+B_l .req x9
+B_h .req x10
+C_l .req x11
+C_h .req x12
+D_l .req x13
+D_h .req x14
+
+V_a .req v20
+V_b .req v21
+V_c .req v22
+V_d .req v23
+
+ mov dst, dstin
+ cmp count, #16
+ /*When memory length is less than 16, the accessed are not aligned.*/
+ b.lo .Ltiny15_fpsimd
+
+ neg tmp2, src
+ ands tmp2, tmp2, #15/* Bytes to reach alignment. */
+ b.eq .LSrcAligned_fpsimd
+ sub count, count, tmp2
+ /*
+ * Copy the leading memory data from src to dst in an increasing
+ * address order.By this way,the risk of overwriting the source
+ * memory data is eliminated when the distance between src and
+ * dst is less than 16. The memory accesses here are alignment.
+ */
+ tbz tmp2, #0, 1f
+ ldrb1 tmp1w, src, #1
+ strb1 tmp1w, dst, #1
+1:
+ tbz tmp2, #1, 2f
+ ldrh1 tmp1w, src, #2
+ strh1 tmp1w, dst, #2
+2:
+ tbz tmp2, #2, 3f
+ ldr1 tmp1w, src, #4
+ str1 tmp1w, dst, #4
+3:
+ tbz tmp2, #3, .LSrcAligned_fpsimd
+ ldr1 tmp1, src, #8
+ str1 tmp1, dst, #8
+
+.LSrcAligned_fpsimd:
+ cmp count, #64
+ b.ge .Lcpy_over64_fpsimd
+ /*
+ * Deal with small copies quickly by dropping straight into the
+ * exit block.
+ */
+.Ltail63_fpsimd:
+ /*
+ * Copy up to 48 bytes of data. At this point we only need the
+ * bottom 6 bits of count to be accurate.
+ */
+ ands tmp1, count, #0x30
+ b.eq .Ltiny15_fpsimd
+ cmp tmp1w, #0x20
+ b.eq 1f
+ b.lt 2f
+ ldp1 A_l, A_h, src, #16
+ stp1 A_l, A_h, dst, #16
+1:
+ ldp1 A_l, A_h, src, #16
+ stp1 A_l, A_h, dst, #16
+2:
+ ldp1 A_l, A_h, src, #16
+ stp1 A_l, A_h, dst, #16
+.Ltiny15_fpsimd:
+ /*
+ * Prefer to break one ldp/stp into several load/store to access
+ * memory in an increasing address order,rather than to load/store 16
+ * bytes from (src-16) to (dst-16) and to backward the src to aligned
+ * address,which way is used in original cortex memcpy. If keeping
+ * the original memcpy process here, memmove need to satisfy the
+ * precondition that src address is at least 16 bytes bigger than dst
+ * address,otherwise some source data will be overwritten when memove
+ * call memcpy directly. To make memmove simpler and decouple the
+ * memcpy's dependency on memmove, withdrew the original process.
+ */
+ tbz count, #3, 1f
+ ldr1 tmp1, src, #8
+ str1 tmp1, dst, #8
+1:
+ tbz count, #2, 2f
+ ldr1 tmp1w, src, #4
+ str1 tmp1w, dst, #4
+2:
+ tbz count, #1, 3f
+ ldrh1 tmp1w, src, #2
+ strh1 tmp1w, dst, #2
+3:
+ tbz count, #0, .Lexitfunc_fpsimd
+ ldrb1 tmp1w, src, #1
+ strb1 tmp1w, dst, #1
+
+ b .Lexitfunc_fpsimd
+
+.Lcpy_over64_fpsimd:
+ subs count, count, #128
+ b.ge .Lcpy_body_large_fpsimd
+ /*
+ * Less than 128 bytes to copy, so handle 64 here and then jump
+ * to the tail.
+ */
+ ldp1 A_l, A_h, src, #16
+ stp1 A_l, A_h, dst, #16
+ ldp1 B_l, B_h, src, #16
+ ldp1 C_l, C_h, src, #16
+ stp1 B_l, B_h, dst, #16
+ stp1 C_l, C_h, dst, #16
+ ldp1 D_l, D_h, src, #16
+ stp1 D_l, D_h, dst, #16
+
+ tst count, #0x3f
+ b.ne .Ltail63_fpsimd
+ b .Lexitfunc_fpsimd
+
+ /*
+ * Critical loop. Start at a new cache line boundary. Assuming
+ * 64 bytes per line this ensures the entire loop is in one line.
+ */
+ .p2align L1_CACHE_SHIFT
+.Lcpy_body_large_fpsimd:
+ /* pre-get 64 bytes data. */
+ ldsve V_a.16b, V_b.16b, V_c.16b, V_d.16b, src
+ add src, src, #64
+
+1:
+ /*
+ * interlace the load of next 64 bytes data block with store of the last
+ * loaded 64 bytes data.
+ */
+ stsve V_a.16b, V_b.16b, V_c.16b, V_d.16b, dst
+ ldsve V_a.16b, V_b.16b, V_c.16b, V_d.16b, src
+ add dst, dst, #64
+ add src, src, #64
+
+ subs count, count, #64
+ b.ge 1b
+
+ stsve V_a.16b, V_b.16b, V_c.16b, V_d.16b, dst
+ add dst, dst, #64
+
+ tst count, #0x3f
+ b.ne .Ltail63_fpsimd
+.Lexitfunc_fpsimd:
diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
index 2ac716c0d6d8..c190e5f8a989 100644
--- a/arch/arm64/lib/copy_to_user.S
+++ b/arch/arm64/lib/copy_to_user.S
@@ -71,3 +71,33 @@ USER(9998f, sttrb tmp1w, [dst])
ret
SYM_FUNC_END(__arch_copy_to_user)
EXPORT_SYMBOL(__arch_copy_to_user)
+
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ .macro stsve reg1, reg2, reg3, reg4, ptr
+ USER(9997f, st1 {\reg1, \reg2, \reg3, \reg4}, [\ptr])
+ .endm
+
+ .macro ldsve reg1, reg2, reg3, reg4, ptr
+ KERNEL_ME_SAFE(9998f, ld1 {\reg1, \reg2, \reg3, \reg4}, [\ptr])
+ .endm
+
+SYM_FUNC_START(__arch_copy_to_user_fpsimd)
+ add end, x0, x2
+ mov srcin, x1
+#include "copy_template_fpsimd.S"
+ mov x0, #0
+ ret
+
+ // Exception fixups
+9997: cmp dst, dstin
+ b.ne 9998f
+ // Before being absolutely sure we couldn't copy anything, try harder
+KERNEL_ME_SAFE(9998f, ldrb tmp1w, [srcin])
+USER(9998f, sttrb tmp1w, [dst])
+ add dst, dst, #1
+9998: sub x0, end, dst // bytes not copied
+ ret
+SYM_FUNC_END(__arch_copy_to_user_fpsimd)
+EXPORT_SYMBOL(__arch_copy_to_user_fpsimd)
+#endif
diff --git a/kernel/softirq.c b/kernel/softirq.c
index f8cf88cc46c6..39b84ffbf4e5 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -30,6 +30,10 @@
#include <asm/softirq_stack.h>
+#ifdef CONFIG_USE_VECTORIZED_COPY
+#include <asm/fpsimd.h>
+#endif
+
#define CREATE_TRACE_POINTS
#include <trace/events/irq.h>
@@ -524,6 +528,9 @@ static void handle_softirqs(bool ksirqd)
__u32 pending;
int softirq_bit;
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ struct fpsimd_state state;
+#endif
/*
* Mask out PF_MEMALLOC as the current task context is borrowed for the
* softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
@@ -533,10 +540,16 @@ static void handle_softirqs(bool ksirqd)
pending = local_softirq_pending();
+
softirq_handle_begin();
in_hardirq = lockdep_softirq_start();
account_softirq_enter(current);
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ _kernel_fpsimd_save(&state);
+ uaccess_priviliged_state_save();
+#endif
+
restart:
/* Reset the pending bitmask before enabling irqs */
set_softirq_pending(0);
@@ -585,7 +598,14 @@ static void handle_softirqs(bool ksirqd)
account_softirq_exit(current);
lockdep_softirq_end(in_hardirq);
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ uaccess_priviliged_state_restore();
+ _kernel_fpsimd_load(&state);
+#endif
+
softirq_handle_end();
+
current_restore_flags(old_flags, PF_MEMALLOC);
}
@@ -819,12 +839,21 @@ static void tasklet_action_common(struct softirq_action *a,
{
struct tasklet_struct *list;
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ struct fpsimd_state state;
+#endif
+
local_irq_disable();
list = tl_head->head;
tl_head->head = NULL;
tl_head->tail = &tl_head->head;
local_irq_enable();
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ _kernel_fpsimd_save(&state);
+ uaccess_priviliged_state_save();
+#endif
+
while (list) {
struct tasklet_struct *t = list;
@@ -856,6 +885,11 @@ static void tasklet_action_common(struct softirq_action *a,
__raise_softirq_irqoff(softirq_nr);
local_irq_enable();
}
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ uaccess_priviliged_state_restore();
+ _kernel_fpsimd_load(&state);
+#endif
}
static __latent_entropy void tasklet_action(struct softirq_action *a)
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index e84df0818517..6f8e22102bdc 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -137,6 +137,17 @@ int sysctl_legacy_va_layout;
#endif /* CONFIG_SYSCTL */
+#ifdef CONFIG_USE_VECTORIZED_COPY
+int sysctl_copy_to_user_threshold = -1;
+EXPORT_SYMBOL(sysctl_copy_to_user_threshold);
+
+int sysctl_copy_from_user_threshold = -1;
+EXPORT_SYMBOL(sysctl_copy_from_user_threshold);
+
+int sysctl_copy_in_user_threshold = -1;
+EXPORT_SYMBOL(sysctl_copy_in_user_threshold);
+#endif
+
/*
* /proc/sys support
*/
@@ -2250,6 +2261,29 @@ static struct ctl_table vm_table[] = {
.extra1 = (void *)&mmap_rnd_compat_bits_min,
.extra2 = (void *)&mmap_rnd_compat_bits_max,
},
+#endif
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ {
+ .procname = "copy_to_user_threshold",
+ .data = &sysctl_copy_to_user_threshold,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec
+ },
+ {
+ .procname = "copy_from_user_threshold",
+ .data = &sysctl_copy_from_user_threshold,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec
+ },
+ {
+ .procname = "copy_in_user_threshold",
+ .data = &sysctl_copy_in_user_threshold,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec
+ },
#endif
{ }
};
--
2.34.1
2
1
[PATCH v1 openEuler-25.03] Add copy to/from/in user with vectorization support
by Nikita Panov 28 Jan '26
by Nikita Panov 28 Jan '26
28 Jan '26
From: Artem Kuzin <artem.kuzin(a)huawei.com>
kunpeng inclusion
category: feature
bugzilla: https://atomgit.com/openeuler/kernel/issues/8445
-------------------------------------------------
1. This implementation uses st1/ld1 4-vector instructions which allow to copy 64 bytes at once
2. Copy code is used only if size of data block to copy is more than 128 bytes
4. To use this functionality you need to set configuration switch CONFIG_USE_VECTORIZED_COPY=y
5. Code can be used on any ARMv8 variant
6. In kernel copy functions like memcpy are not supported now, but can be enabled in future
7. For now we use lightweght version of register context saving/restoration (4-registers)
We introduce support of vectorization for copy_from/to/in_user functions. Nowadays it
works in parallel with original FPSIMD/SVE vectorization and doesn't affect it anyhow.
We have special flag in task struct - TIF_KERNEL_FPSIMD, that set if currently we use
lightweight vectorization in kernel. Task struct has been updated by two fields:
user space fpsimd state and kernel fpsimd state. User space fpsimd state used by
kernel_fpsimd_begin(), kernel_fpsimd_end() functions that wrap lightweight FPSIMD
contexts usage in kernel space. Kernel fpsimd state is used to manage threads switch.
Now there is no support of nested calls of kernel_neon_begin()/kernel_fpsimd_begin()
and there is no plans to support this in future. This is not necessary.
We save lightweight FPSIMD context in kernel_fpsimd_begin(), and restore it in
/kernel_fpsimd_end(). On thread switch we preserve kernel FPSIMD context and restore
user space one if any. This prevens curruption of user space FPSIMD state. Before
switching to the next thread we restore it's kernel FPSIMD context if any.
It is allowed to use FPSIMD in bottom halves, due to in case of BH preemption we check
TIF_KERNEL_FPSIMD flag and save/restore contexts.
Context management if quite lightweight and executed only in case of TIF_KERNEL_FPSIMD
flag is set.
To enable this feature, you need to manually modify one of the
appropriate entries:
/proc/sys/vm/copy_from_user_threshold
/proc/sys/vm/copy_in_user_threshold
/proc/sys/vm/copy_to_user_threshold
Allowed values are following:
-1 - feature enabled
0 - feature always enabled
n (n >0) - feature enabled, if copied size is greater than n KB.
P.S.:
What I am personally don't like in current approach:
1. Additional fields and flag in task struct look quite ugly
2. No way to configure the size of chunk to copy using FPSIMD from user space
3. FPSIMD-based memory movement is not generic, need to enable for memmove(),
memcpy() and friends in future.
Co-developed-by: Alexander Kozhevnikov <alexander.kozhevnikov(a)huawei-partners.com>
Signed-off-by: Alexander Kozhevnikov <alexander.kozhevnikov(a)huawei-partners.com>
Co-developed-by: Nikita Panov <panov.nikita(a)huawei.com>
Signed-off-by: Nikita Panov <panov.nikita(a)huawei.com>
Signed-off-by: Artem Kuzin <artem.kuzin(a)huawei.com>
---
arch/arm64/Kconfig | 15 ++
arch/arm64/configs/openeuler_defconfig | 2 +
arch/arm64/include/asm/fpsimd.h | 15 ++
arch/arm64/include/asm/fpsimdmacros.h | 14 ++
arch/arm64/include/asm/neon.h | 28 ++++
arch/arm64/include/asm/processor.h | 10 ++
arch/arm64/include/asm/thread_info.h | 5 +
arch/arm64/include/asm/uaccess.h | 218 ++++++++++++++++++++++++-
arch/arm64/kernel/entry-fpsimd.S | 22 +++
arch/arm64/kernel/fpsimd.c | 102 +++++++++++-
arch/arm64/kernel/process.c | 2 +-
arch/arm64/lib/copy_from_user.S | 30 ++++
arch/arm64/lib/copy_template_fpsimd.S | 180 ++++++++++++++++++++
arch/arm64/lib/copy_to_user.S | 30 ++++
kernel/softirq.c | 34 ++++
kernel/sysctl.c | 34 ++++
16 files changed, 734 insertions(+), 7 deletions(-)
create mode 100644 arch/arm64/lib/copy_template_fpsimd.S
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index d3ce44c166ce..0cf5ab2d7574 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1828,6 +1828,21 @@ config ARM64_ILP32
is an ABI where long and pointers are 32bits but it uses the AARCH64
instruction set.
+config USE_VECTORIZED_COPY
+ bool "Use vectorized instructions in copy_to/from user"
+ depends on KERNEL_MODE_NEON
+ default y
+ help
+ This option turns on vectorization to speed up copy_to/from_user routines.
+
+config VECTORIZED_COPY_VALIDATE
+ bool "Validate result of vectorized copy using regular implementation"
+ depends on KERNEL_MODE_NEON
+ depends on USE_VECTORIZED_COPY
+ default n
+ help
+ This option turns on vectorization to speed up copy_to/from_user routines.
+
menuconfig AARCH32_EL0
bool "Kernel support for 32-bit EL0"
depends on ARM64_4K_PAGES || EXPERT
diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig
index 8f97574813ca..dbad22bcbd57 100644
--- a/arch/arm64/configs/openeuler_defconfig
+++ b/arch/arm64/configs/openeuler_defconfig
@@ -499,6 +499,8 @@ CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY=y
# CONFIG_RODATA_FULL_DEFAULT_ENABLED is not set
# CONFIG_ARM64_SW_TTBR0_PAN is not set
CONFIG_ARM64_TAGGED_ADDR_ABI=y
+CONFIG_USE_VECTORIZED_COPY=y
+# CONFIG_VECTORIZED_COPY_VALIDATE is not set
CONFIG_AARCH32_EL0=y
# CONFIG_KUSER_HELPERS is not set
# CONFIG_COMPAT_ALIGNMENT_FIXUPS is not set
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 40a99d8607fe..f71b3ac578c1 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -46,6 +46,21 @@
struct task_struct;
+#ifdef CONFIG_USE_VECTORIZED_COPY
+extern void fpsimd_save_state_light(struct fpsimd_state *state);
+extern void fpsimd_load_state_light(struct fpsimd_state *state);
+#else
+static inline void fpsimd_save_state_light(struct fpsimd_state *state)
+{
+ (void) state;
+}
+
+static inline void fpsimd_load_state_light(struct fpsimd_state *state)
+{
+ (void) state;
+}
+#endif
+
extern void fpsimd_save_state(struct user_fpsimd_state *state);
extern void fpsimd_load_state(struct user_fpsimd_state *state);
diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h
index cdf6a35e3994..df9d3ed91931 100644
--- a/arch/arm64/include/asm/fpsimdmacros.h
+++ b/arch/arm64/include/asm/fpsimdmacros.h
@@ -8,6 +8,20 @@
#include <asm/assembler.h>
+#ifdef CONFIG_USE_VECTORIZED_COPY
+/* Lightweight fpsimd context saving/restoration.
+ * Necessary for vectorized kernel memory movement
+ * implementation
+ */
+.macro fpsimd_save_light state
+ st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [\state]
+.endm
+
+.macro fpsimd_restore_light state
+ ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [\state]
+.endm
+#endif
+
.macro fpsimd_save state, tmpnr
stp q0, q1, [\state, #16 * 0]
stp q2, q3, [\state, #16 * 2]
diff --git a/arch/arm64/include/asm/neon.h b/arch/arm64/include/asm/neon.h
index d4b1d172a79b..ab84b194d7b3 100644
--- a/arch/arm64/include/asm/neon.h
+++ b/arch/arm64/include/asm/neon.h
@@ -16,4 +16,32 @@
void kernel_neon_begin(void);
void kernel_neon_end(void);
+#ifdef CONFIG_USE_VECTORIZED_COPY
+bool kernel_fpsimd_begin(void);
+void kernel_fpsimd_end(void);
+/* Functions to use in non-preemptible context */
+void _kernel_fpsimd_save(struct fpsimd_state *state);
+void _kernel_fpsimd_load(struct fpsimd_state *state);
+#else
+bool kernel_fpsimd_begin(void)
+{
+ return false;
+}
+
+void kernel_fpsimd_end(void)
+{
+}
+
+/* Functions to use in non-preemptible context */
+void _kernel_fpsimd_save(struct fpsimd_state *state)
+{
+ (void) state;
+}
+
+void _kernel_fpsimd_load(struct fpsimd_state *state)
+{
+ (void) state;
+}
+#endif
+
#endif /* ! __ASM_NEON_H */
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 9e688b1b13d4..9b81dbcd2126 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -153,6 +153,10 @@ struct cpu_context {
unsigned long pc;
};
+struct fpsimd_state {
+ __uint128_t v[4];
+};
+
struct thread_struct {
struct cpu_context cpu_context; /* cpu context */
@@ -196,6 +200,12 @@ struct thread_struct {
KABI_RESERVE(6)
KABI_RESERVE(7)
KABI_RESERVE(8)
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ KABI_EXTEND(
+ struct fpsimd_state ustate;
+ struct fpsimd_state kstate;
+ )
+#endif
};
static inline unsigned int thread_get_vl(struct thread_struct *thread,
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index 379d24059f5b..60d0be8a2d58 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -89,6 +89,9 @@ void arch_setup_new_exec(void);
#define TIF_SME 27 /* SME in use */
#define TIF_SME_VL_INHERIT 28 /* Inherit SME vl_onexec across exec */
#define TIF_32BIT_AARCH64 29 /* 32 bit process on AArch64(ILP32) */
+#define TIF_KERNEL_FPSIMD 31 /* Use FPSIMD in kernel */
+#define TIF_PRIV_UACC_ENABLED 32 /* Whether priviliged uaccess was manually enabled */
+
#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
@@ -107,6 +110,8 @@ void arch_setup_new_exec(void);
#define _TIF_MTE_ASYNC_FAULT (1 << TIF_MTE_ASYNC_FAULT)
#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)
#define _TIF_32BIT_AARCH64 (1 << TIF_32BIT_AARCH64)
+#define _TIF_KERNEL_FPSIMD (1 << TIF_KERNEL_FPSIMD)
+#define _TIF_PRIV_UACC_ENABLED (1 << TIF_PRIV_UACC_ENABLED)
#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
_TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index dd0877a75922..fc9f1a40624d 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -26,6 +26,10 @@
#include <asm/memory.h>
#include <asm/extable.h>
+#ifndef __GENKSYMS__
+#include <asm/neon.h>
+#endif
+
static inline int __access_ok(const void __user *ptr, unsigned long size);
/*
@@ -134,7 +138,7 @@ static inline void __uaccess_enable_hw_pan(void)
CONFIG_ARM64_PAN));
}
-static inline void uaccess_disable_privileged(void)
+static inline void __uaccess_disable_privileged(void)
{
mte_disable_tco();
@@ -144,7 +148,22 @@ static inline void uaccess_disable_privileged(void)
__uaccess_enable_hw_pan();
}
-static inline void uaccess_enable_privileged(void)
+static inline void uaccess_disable_privileged(void)
+{
+ preempt_disable();
+
+ if (!test_and_clear_thread_flag(TIF_PRIV_UACC_ENABLED)) {
+ WARN_ON(1);
+ preempt_enable();
+ return;
+ }
+
+ __uaccess_disable_privileged();
+
+ preempt_enable();
+}
+
+static inline void __uaccess_enable_privileged(void)
{
mte_enable_tco();
@@ -154,6 +173,47 @@ static inline void uaccess_enable_privileged(void)
__uaccess_disable_hw_pan();
}
+static inline void uaccess_enable_privileged(void)
+{
+ preempt_disable();
+
+ if (test_and_set_thread_flag(TIF_PRIV_UACC_ENABLED)) {
+ WARN_ON(1);
+ preempt_enable();
+ return;
+ }
+
+ __uaccess_enable_privileged();
+
+ preempt_enable();
+}
+
+static inline void uaccess_priviliged_context_switch(struct task_struct *next)
+{
+ bool curr_enabled = !!test_thread_flag(TIF_PRIV_UACC_ENABLED);
+ bool next_enabled = !!test_ti_thread_flag(&next->thread_info, TIF_PRIV_UACC_ENABLED);
+
+ if (curr_enabled == next_enabled)
+ return;
+
+ if (curr_enabled)
+ __uaccess_disable_privileged();
+ else
+ __uaccess_enable_privileged();
+}
+
+static inline void uaccess_priviliged_state_save(void)
+{
+ if (test_thread_flag(TIF_PRIV_UACC_ENABLED))
+ __uaccess_disable_privileged();
+}
+
+static inline void uaccess_priviliged_state_restore(void)
+{
+ if (test_thread_flag(TIF_PRIV_UACC_ENABLED))
+ __uaccess_enable_privileged();
+}
+
/*
* Sanitize a uaccess pointer such that it cannot reach any kernel address.
*
@@ -391,7 +451,97 @@ do { \
} while (0); \
} while(0)
-extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
+#define USER_COPY_CHUNK_SIZE 4096
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+
+extern int sysctl_copy_from_user_threshold;
+
+#define verify_fpsimd_copy(to, from, n, ret) \
+({ \
+ unsigned long __verify_ret = 0; \
+ __verify_ret = memcmp(to, from, ret ? n - ret : n); \
+ if (__verify_ret) \
+ pr_err("FPSIMD:%s inconsistent state\n", __func__); \
+ if (ret) \
+ pr_err("FPSIMD:%s failed to copy data, expected=%lu, copied=%lu\n", __func__, n, n - ret); \
+ __verify_ret |= ret; \
+ __verify_ret; \
+})
+
+#define compare_fpsimd_copy(to, from, n, ret_fpsimd, ret) \
+({ \
+ unsigned long __verify_ret = 0; \
+ __verify_ret = memcmp(to, from, ret ? n - ret : n); \
+ if (__verify_ret) \
+ pr_err("FIXUP:%s inconsistent state\n", __func__); \
+ if (ret) \
+ pr_err("FIXUP:%s failed to copy data, expected=%lu, copied=%lu\n", __func__, n, n - ret); \
+ __verify_ret |= ret; \
+ if (ret_fpsimd != ret) { \
+ pr_err("FIXUP:%s difference between FPSIMD %lu and regular %lu\n", __func__, n - ret_fpsimd, n - ret); \
+ __verify_ret |= 1; \
+ } else { \
+ __verify_ret = 0; \
+ } \
+ __verify_ret; \
+})
+
+extern unsigned long __must_check
+__arch_copy_from_user(void *to, const void __user *from, unsigned long n);
+
+extern unsigned long __must_check
+__arch_copy_from_user_fpsimd(void *to, const void __user *from, unsigned long n);
+
+static __always_inline unsigned long __must_check
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+ unsigned long __acfu_ret;
+
+ if (sysctl_copy_from_user_threshold == -1 || n < sysctl_copy_from_user_threshold) {
+ uaccess_ttbr0_enable();
+ __acfu_ret = __arch_copy_from_user(to,
+ __uaccess_mask_ptr(from), n);
+ uaccess_ttbr0_disable();
+ } else {
+ if (kernel_fpsimd_begin()) {
+ unsigned long __acfu_ret_fpsimd;
+
+ uaccess_enable_privileged();
+ __acfu_ret_fpsimd = __arch_copy_from_user_fpsimd((to),
+ __uaccess_mask_ptr(from), n);
+ uaccess_disable_privileged();
+
+ __acfu_ret = __acfu_ret_fpsimd;
+ kernel_fpsimd_end();
+#ifdef CONFIG_VECTORIZED_COPY_VALIDATE
+ if (verify_fpsimd_copy(to, __uaccess_mask_ptr(from), n,
+ __acfu_ret)) {
+
+ uaccess_ttbr0_enable();
+ __acfu_ret = __arch_copy_from_user((to),
+ __uaccess_mask_ptr(from), n);
+ uaccess_ttbr0_disable();
+
+ compare_fpsimd_copy(to, __uaccess_mask_ptr(from), n,
+ __acfu_ret_fpsimd, __acfu_ret);
+ }
+#endif
+ } else {
+ uaccess_ttbr0_enable();
+ __acfu_ret = __arch_copy_from_user((to),
+ __uaccess_mask_ptr(from), n);
+ uaccess_ttbr0_disable();
+ }
+ }
+
+
+ return __acfu_ret;
+}
+#else
+extern unsigned long __must_check
+__arch_copy_from_user(void *to, const void __user *from, unsigned long n);
+
#define raw_copy_from_user(to, from, n) \
({ \
unsigned long __acfu_ret; \
@@ -402,7 +552,66 @@ extern unsigned long __must_check __arch_copy_from_user(void *to, const void __u
__acfu_ret; \
})
-extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
+#endif
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+
+extern int sysctl_copy_to_user_threshold;
+
+extern unsigned long __must_check
+__arch_copy_to_user(void __user *to, const void *from, unsigned long n);
+
+extern unsigned long __must_check
+__arch_copy_to_user_fpsimd(void __user *to, const void *from, unsigned long n);
+
+static __always_inline unsigned long __must_check
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+ unsigned long __actu_ret;
+
+
+ if (sysctl_copy_to_user_threshold == -1 || n < sysctl_copy_to_user_threshold) {
+ uaccess_ttbr0_enable();
+ __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),
+ from, n);
+ uaccess_ttbr0_disable();
+ } else {
+ if (kernel_fpsimd_begin()) {
+ unsigned long __actu_ret_fpsimd;
+
+ uaccess_enable_privileged();
+ __actu_ret_fpsimd = __arch_copy_to_user_fpsimd(__uaccess_mask_ptr(to),
+ from, n);
+ uaccess_disable_privileged();
+
+ kernel_fpsimd_end();
+ __actu_ret = __actu_ret_fpsimd;
+#ifdef CONFIG_VECTORIZED_COPY_VALIDATE
+ if (verify_fpsimd_copy(__uaccess_mask_ptr(to), from, n,
+ __actu_ret)) {
+ uaccess_ttbr0_enable();
+ __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),
+ from, n);
+ uaccess_ttbr0_disable();
+
+ compare_fpsimd_copy(__uaccess_mask_ptr(to), from, n,
+ __actu_ret_fpsimd, __actu_ret);
+ }
+#endif
+ } else {
+ uaccess_ttbr0_enable();
+ __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),
+ from, n);
+ uaccess_ttbr0_disable();
+ }
+ }
+
+ return __actu_ret;
+}
+#else
+extern unsigned long __must_check
+__arch_copy_to_user(void __user *to, const void *from, unsigned long n);
+
#define raw_copy_to_user(to, from, n) \
({ \
unsigned long __actu_ret; \
@@ -412,6 +621,7 @@ extern unsigned long __must_check __arch_copy_to_user(void __user *to, const voi
uaccess_ttbr0_disable(); \
__actu_ret; \
})
+#endif
static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len)
{
diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S
index 6325db1a2179..6660465f1b7c 100644
--- a/arch/arm64/kernel/entry-fpsimd.S
+++ b/arch/arm64/kernel/entry-fpsimd.S
@@ -11,6 +11,28 @@
#include <asm/assembler.h>
#include <asm/fpsimdmacros.h>
+#ifdef CONFIG_USE_VECTORIZED_COPY
+/*
+ * Save the FP registers.
+ *
+ * x0 - pointer to struct fpsimd_state_light
+ */
+SYM_FUNC_START(fpsimd_save_state_light)
+ fpsimd_save_light x0
+ ret
+SYM_FUNC_END(fpsimd_save_state_light)
+
+/*
+ * Load the FP registers.
+ *
+ * x0 - pointer to struct fpsimd_state_light
+ */
+SYM_FUNC_START(fpsimd_load_state_light)
+ fpsimd_restore_light x0
+ ret
+SYM_FUNC_END(fpsimd_load_state_light)
+#endif
+
/*
* Save the FP registers.
*
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 0137d987631e..19fcf3a3ac66 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -1577,6 +1577,11 @@ void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs)
current);
}
+#ifdef CONFIG_USE_VECTORIZED_COPY
+static void kernel_fpsimd_rollback_changes(void);
+static void kernel_fpsimd_restore_changes(struct task_struct *tsk);
+#endif
+
void fpsimd_thread_switch(struct task_struct *next)
{
bool wrong_task, wrong_cpu;
@@ -1585,10 +1590,11 @@ void fpsimd_thread_switch(struct task_struct *next)
return;
__get_cpu_fpsimd_context();
-
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ kernel_fpsimd_rollback_changes();
+#endif
/* Save unsaved fpsimd state, if any: */
fpsimd_save();
-
/*
* Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
* state. For kernel threads, FPSIMD registers are never loaded
@@ -1601,6 +1607,9 @@ void fpsimd_thread_switch(struct task_struct *next)
update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
wrong_task || wrong_cpu);
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ kernel_fpsimd_restore_changes(next);
+#endif
__put_cpu_fpsimd_context();
}
@@ -1956,6 +1965,95 @@ void kernel_neon_end(void)
}
EXPORT_SYMBOL_GPL(kernel_neon_end);
+#ifdef CONFIG_USE_VECTORIZED_COPY
+bool kernel_fpsimd_begin(void)
+{
+ if (WARN_ON(!system_capabilities_finalized()) ||
+ !system_supports_fpsimd() ||
+ in_irq() || irqs_disabled() || in_nmi())
+ return false;
+
+ preempt_disable();
+ if (test_and_set_thread_flag(TIF_KERNEL_FPSIMD)) {
+ preempt_enable();
+
+ WARN_ON(1);
+ return false;
+ }
+
+ /*
+ * Leaving streaming mode enabled will cause issues for any kernel
+ * NEON and leaving streaming mode or ZA enabled may increase power
+ * consumption.
+ */
+ if (system_supports_sme())
+ sme_smstop();
+
+ fpsimd_save_state_light(¤t->thread.ustate);
+ preempt_enable();
+
+ return true;
+}
+EXPORT_SYMBOL(kernel_fpsimd_begin);
+
+void kernel_fpsimd_end(void)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ preempt_disable();
+ if (test_and_clear_thread_flag(TIF_KERNEL_FPSIMD))
+ fpsimd_load_state_light(¤t->thread.ustate);
+
+ preempt_enable();
+}
+EXPORT_SYMBOL(kernel_fpsimd_end);
+
+void _kernel_fpsimd_save(struct fpsimd_state *state)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ BUG_ON(preemptible());
+ if (test_thread_flag(TIF_KERNEL_FPSIMD))
+ fpsimd_save_state_light(state);
+}
+
+void _kernel_fpsimd_load(struct fpsimd_state *state)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ BUG_ON(preemptible());
+ if (test_thread_flag(TIF_KERNEL_FPSIMD))
+ fpsimd_load_state_light(state);
+}
+
+static void kernel_fpsimd_rollback_changes(void)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ BUG_ON(preemptible());
+ if (test_thread_flag(TIF_KERNEL_FPSIMD)) {
+ fpsimd_save_state_light(¤t->thread.kstate);
+ fpsimd_load_state_light(¤t->thread.ustate);
+ }
+}
+
+static void kernel_fpsimd_restore_changes(struct task_struct *tsk)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ BUG_ON(preemptible());
+ if (test_ti_thread_flag(task_thread_info(tsk), TIF_KERNEL_FPSIMD)) {
+ fpsimd_save_state_light(&tsk->thread.ustate);
+ fpsimd_load_state_light(&tsk->thread.kstate);
+ }
+}
+#endif
+
#ifdef CONFIG_EFI
static DEFINE_PER_CPU(struct user_fpsimd_state, efi_fpsimd_state);
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 068e5bb2661b..bbeb36e671de 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -524,7 +524,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
struct task_struct *next)
{
struct task_struct *last;
-
+ uaccess_priviliged_context_switch(next);
fpsimd_thread_switch(next);
tls_thread_switch(next);
hw_breakpoint_thread_switch(next);
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 34e317907524..60dc63e10233 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -71,3 +71,33 @@ USER(9998f, ldtrb tmp1w, [srcin])
ret
SYM_FUNC_END(__arch_copy_from_user)
EXPORT_SYMBOL(__arch_copy_from_user)
+
+
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ .macro ldsve reg1, reg2, reg3, reg4, ptr
+ USER(9997f, ld1 {\reg1, \reg2, \reg3, \reg4}, [\ptr])
+ .endm
+
+ .macro stsve reg1, reg2, reg3, reg4, ptr
+ KERNEL_ME_SAFE(9998f, st1 {\reg1, \reg2, \reg3, \reg4}, [\ptr])
+ .endm
+
+SYM_FUNC_START(__arch_copy_from_user_fpsimd)
+ add end, x0, x2
+ mov srcin, x1
+#include "copy_template_fpsimd.S"
+ mov x0, #0 // Nothing to copy
+ ret
+
+ // Exception fixups
+9997: cmp dst, dstin
+ b.ne 9998f
+ // Before being absolutely sure we couldn't copy anything, try harder
+USER(9998f, ldtrb tmp1w, [srcin])
+ strb tmp1w, [dst], #1
+9998: sub x0, end, dst // bytes not copied
+ ret
+SYM_FUNC_END(__arch_copy_from_user_fpsimd)
+EXPORT_SYMBOL(__arch_copy_from_user_fpsimd)
+#endif
\ No newline at end of file
diff --git a/arch/arm64/lib/copy_template_fpsimd.S b/arch/arm64/lib/copy_template_fpsimd.S
new file mode 100644
index 000000000000..9b2e7ce1e4d2
--- /dev/null
+++ b/arch/arm64/lib/copy_template_fpsimd.S
@@ -0,0 +1,180 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
+ */
+
+/*
+ * Copy a buffer from src to dest (alignment handled by the hardware)
+ *
+ * Parameters:
+ * x0 - dest
+ * x1 - src
+ * x2 - n
+ * Returns:
+ * x0 - dest
+ */
+dstin .req x0
+src .req x1
+count .req x2
+tmp1 .req x3
+tmp1w .req w3
+tmp2 .req x4
+tmp2w .req w4
+dst .req x6
+
+A_l .req x7
+A_h .req x8
+B_l .req x9
+B_h .req x10
+C_l .req x11
+C_h .req x12
+D_l .req x13
+D_h .req x14
+
+V_a .req v20
+V_b .req v21
+V_c .req v22
+V_d .req v23
+
+ mov dst, dstin
+ cmp count, #16
+ /*When memory length is less than 16, the accessed are not aligned.*/
+ b.lo .Ltiny15_fpsimd
+
+ neg tmp2, src
+ ands tmp2, tmp2, #15/* Bytes to reach alignment. */
+ b.eq .LSrcAligned_fpsimd
+ sub count, count, tmp2
+ /*
+ * Copy the leading memory data from src to dst in an increasing
+ * address order.By this way,the risk of overwriting the source
+ * memory data is eliminated when the distance between src and
+ * dst is less than 16. The memory accesses here are alignment.
+ */
+ tbz tmp2, #0, 1f
+ ldrb1 tmp1w, src, #1
+ strb1 tmp1w, dst, #1
+1:
+ tbz tmp2, #1, 2f
+ ldrh1 tmp1w, src, #2
+ strh1 tmp1w, dst, #2
+2:
+ tbz tmp2, #2, 3f
+ ldr1 tmp1w, src, #4
+ str1 tmp1w, dst, #4
+3:
+ tbz tmp2, #3, .LSrcAligned_fpsimd
+ ldr1 tmp1, src, #8
+ str1 tmp1, dst, #8
+
+.LSrcAligned_fpsimd:
+ cmp count, #64
+ b.ge .Lcpy_over64_fpsimd
+ /*
+ * Deal with small copies quickly by dropping straight into the
+ * exit block.
+ */
+.Ltail63_fpsimd:
+ /*
+ * Copy up to 48 bytes of data. At this point we only need the
+ * bottom 6 bits of count to be accurate.
+ */
+ ands tmp1, count, #0x30
+ b.eq .Ltiny15_fpsimd
+ cmp tmp1w, #0x20
+ b.eq 1f
+ b.lt 2f
+ ldp1 A_l, A_h, src, #16
+ stp1 A_l, A_h, dst, #16
+1:
+ ldp1 A_l, A_h, src, #16
+ stp1 A_l, A_h, dst, #16
+2:
+ ldp1 A_l, A_h, src, #16
+ stp1 A_l, A_h, dst, #16
+.Ltiny15_fpsimd:
+ /*
+ * Prefer to break one ldp/stp into several load/store to access
+ * memory in an increasing address order,rather than to load/store 16
+ * bytes from (src-16) to (dst-16) and to backward the src to aligned
+ * address,which way is used in original cortex memcpy. If keeping
+ * the original memcpy process here, memmove need to satisfy the
+ * precondition that src address is at least 16 bytes bigger than dst
+ * address,otherwise some source data will be overwritten when memove
+ * call memcpy directly. To make memmove simpler and decouple the
+ * memcpy's dependency on memmove, withdrew the original process.
+ */
+ tbz count, #3, 1f
+ ldr1 tmp1, src, #8
+ str1 tmp1, dst, #8
+1:
+ tbz count, #2, 2f
+ ldr1 tmp1w, src, #4
+ str1 tmp1w, dst, #4
+2:
+ tbz count, #1, 3f
+ ldrh1 tmp1w, src, #2
+ strh1 tmp1w, dst, #2
+3:
+ tbz count, #0, .Lexitfunc_fpsimd
+ ldrb1 tmp1w, src, #1
+ strb1 tmp1w, dst, #1
+
+ b .Lexitfunc_fpsimd
+
+.Lcpy_over64_fpsimd:
+ subs count, count, #128
+ b.ge .Lcpy_body_large_fpsimd
+ /*
+ * Less than 128 bytes to copy, so handle 64 here and then jump
+ * to the tail.
+ */
+ ldp1 A_l, A_h, src, #16
+ stp1 A_l, A_h, dst, #16
+ ldp1 B_l, B_h, src, #16
+ ldp1 C_l, C_h, src, #16
+ stp1 B_l, B_h, dst, #16
+ stp1 C_l, C_h, dst, #16
+ ldp1 D_l, D_h, src, #16
+ stp1 D_l, D_h, dst, #16
+
+ tst count, #0x3f
+ b.ne .Ltail63_fpsimd
+ b .Lexitfunc_fpsimd
+
+ /*
+ * Critical loop. Start at a new cache line boundary. Assuming
+ * 64 bytes per line this ensures the entire loop is in one line.
+ */
+ .p2align L1_CACHE_SHIFT
+.Lcpy_body_large_fpsimd:
+ /* pre-get 64 bytes data. */
+ ldsve V_a.16b, V_b.16b, V_c.16b, V_d.16b, src
+ add src, src, #64
+
+1:
+ /*
+ * interlace the load of next 64 bytes data block with store of the last
+ * loaded 64 bytes data.
+ */
+ stsve V_a.16b, V_b.16b, V_c.16b, V_d.16b, dst
+ ldsve V_a.16b, V_b.16b, V_c.16b, V_d.16b, src
+ add dst, dst, #64
+ add src, src, #64
+
+ subs count, count, #64
+ b.ge 1b
+
+ stsve V_a.16b, V_b.16b, V_c.16b, V_d.16b, dst
+ add dst, dst, #64
+
+ tst count, #0x3f
+ b.ne .Ltail63_fpsimd
+.Lexitfunc_fpsimd:
diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
index 2ac716c0d6d8..c190e5f8a989 100644
--- a/arch/arm64/lib/copy_to_user.S
+++ b/arch/arm64/lib/copy_to_user.S
@@ -71,3 +71,33 @@ USER(9998f, sttrb tmp1w, [dst])
ret
SYM_FUNC_END(__arch_copy_to_user)
EXPORT_SYMBOL(__arch_copy_to_user)
+
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ .macro stsve reg1, reg2, reg3, reg4, ptr
+ USER(9997f, st1 {\reg1, \reg2, \reg3, \reg4}, [\ptr])
+ .endm
+
+ .macro ldsve reg1, reg2, reg3, reg4, ptr
+ KERNEL_ME_SAFE(9998f, ld1 {\reg1, \reg2, \reg3, \reg4}, [\ptr])
+ .endm
+
+SYM_FUNC_START(__arch_copy_to_user_fpsimd)
+ add end, x0, x2
+ mov srcin, x1
+#include "copy_template_fpsimd.S"
+ mov x0, #0
+ ret
+
+ // Exception fixups
+9997: cmp dst, dstin
+ b.ne 9998f
+ // Before being absolutely sure we couldn't copy anything, try harder
+KERNEL_ME_SAFE(9998f, ldrb tmp1w, [srcin])
+USER(9998f, sttrb tmp1w, [dst])
+ add dst, dst, #1
+9998: sub x0, end, dst // bytes not copied
+ ret
+SYM_FUNC_END(__arch_copy_to_user_fpsimd)
+EXPORT_SYMBOL(__arch_copy_to_user_fpsimd)
+#endif
diff --git a/kernel/softirq.c b/kernel/softirq.c
index cd8770b2f76c..e8ce3275a099 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -30,6 +30,10 @@
#include <asm/softirq_stack.h>
+#ifdef CONFIG_USE_VECTORIZED_COPY
+#include <asm/fpsimd.h>
+#endif
+
#define CREATE_TRACE_POINTS
#include <trace/events/irq.h>
@@ -517,6 +521,9 @@ static void handle_softirqs(bool ksirqd)
__u32 pending;
int softirq_bit;
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ struct fpsimd_state state;
+#endif
/*
* Mask out PF_MEMALLOC as the current task context is borrowed for the
* softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
@@ -526,10 +533,16 @@ static void handle_softirqs(bool ksirqd)
pending = local_softirq_pending();
+
softirq_handle_begin();
in_hardirq = lockdep_softirq_start();
account_softirq_enter(current);
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ _kernel_fpsimd_save(&state);
+ uaccess_priviliged_state_save();
+#endif
+
restart:
/* Reset the pending bitmask before enabling irqs */
set_softirq_pending(0);
@@ -578,7 +591,14 @@ static void handle_softirqs(bool ksirqd)
account_softirq_exit(current);
lockdep_softirq_end(in_hardirq);
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ uaccess_priviliged_state_restore();
+ _kernel_fpsimd_load(&state);
+#endif
+
softirq_handle_end();
+
current_restore_flags(old_flags, PF_MEMALLOC);
}
@@ -812,12 +832,21 @@ static void tasklet_action_common(struct softirq_action *a,
{
struct tasklet_struct *list;
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ struct fpsimd_state state;
+#endif
+
local_irq_disable();
list = tl_head->head;
tl_head->head = NULL;
tl_head->tail = &tl_head->head;
local_irq_enable();
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ _kernel_fpsimd_save(&state);
+ uaccess_priviliged_state_save();
+#endif
+
while (list) {
struct tasklet_struct *t = list;
@@ -849,6 +878,11 @@ static void tasklet_action_common(struct softirq_action *a,
__raise_softirq_irqoff(softirq_nr);
local_irq_enable();
}
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ uaccess_priviliged_state_restore();
+ _kernel_fpsimd_load(&state);
+#endif
}
static __latent_entropy void tasklet_action(struct softirq_action *a)
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index e84df0818517..6f8e22102bdc 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -137,6 +137,17 @@ int sysctl_legacy_va_layout;
#endif /* CONFIG_SYSCTL */
+#ifdef CONFIG_USE_VECTORIZED_COPY
+int sysctl_copy_to_user_threshold = -1;
+EXPORT_SYMBOL(sysctl_copy_to_user_threshold);
+
+int sysctl_copy_from_user_threshold = -1;
+EXPORT_SYMBOL(sysctl_copy_from_user_threshold);
+
+int sysctl_copy_in_user_threshold = -1;
+EXPORT_SYMBOL(sysctl_copy_in_user_threshold);
+#endif
+
/*
* /proc/sys support
*/
@@ -2250,6 +2261,29 @@ static struct ctl_table vm_table[] = {
.extra1 = (void *)&mmap_rnd_compat_bits_min,
.extra2 = (void *)&mmap_rnd_compat_bits_max,
},
+#endif
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ {
+ .procname = "copy_to_user_threshold",
+ .data = &sysctl_copy_to_user_threshold,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec
+ },
+ {
+ .procname = "copy_from_user_threshold",
+ .data = &sysctl_copy_from_user_threshold,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec
+ },
+ {
+ .procname = "copy_in_user_threshold",
+ .data = &sysctl_copy_in_user_threshold,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec
+ },
#endif
{ }
};
--
2.34.1
2
1
28 Jan '26
From: Artem Kuzin <artem.kuzin(a)huawei.com>
kunpeng inclusion
category: feature
bugzilla: https://atomgit.com/openeuler/kernel/issues/8445
-------------------------------------------------
1. This implementation uses st1/ld1 4-vector instructions which allow to copy 64 bytes at once
2. Copy code is used only if size of data block to copy is more than 128 bytes
4. To use this functionality you need to set configuration switch CONFIG_USE_VECTORIZED_COPY=y
5. Code can be used on any ARMv8 variant
6. In kernel copy functions like memcpy are not supported now, but can be enabled in future
7. For now we use lightweght version of register context saving/restoration (4-registers)
We introduce support of vectorization for copy_from/to/in_user functions. Nowadays it
works in parallel with original FPSIMD/SVE vectorization and doesn't affect it anyhow.
We have special flag in task struct - TIF_KERNEL_FPSIMD, that set if currently we use
lightweight vectorization in kernel. Task struct has been updated by two fields:
user space fpsimd state and kernel fpsimd state. User space fpsimd state used by
kernel_fpsimd_begin(), kernel_fpsimd_end() functions that wrap lightweight FPSIMD
contexts usage in kernel space. Kernel fpsimd state is used to manage threads switch.
Now there is no support of nested calls of kernel_neon_begin()/kernel_fpsimd_begin()
and there is no plans to support this in future. This is not necessary.
We save lightweight FPSIMD context in kernel_fpsimd_begin(), and restore it in
/kernel_fpsimd_end(). On thread switch we preserve kernel FPSIMD context and restore
user space one if any. This prevens curruption of user space FPSIMD state. Before
switching to the next thread we restore it's kernel FPSIMD context if any.
It is allowed to use FPSIMD in bottom halves, due to in case of BH preemption we check
TIF_KERNEL_FPSIMD flag and save/restore contexts.
Context management if quite lightweight and executed only in case of TIF_KERNEL_FPSIMD
flag is set.
To enable this feature, you need to manually modify one of the
appropriate entries:
/proc/sys/vm/copy_from_user_threshold
/proc/sys/vm/copy_in_user_threshold
/proc/sys/vm/copy_to_user_threshold
Allowed values are following:
-1 - feature enabled
0 - feature always enabled
n (n >0) - feature enabled, if copied size is greater than n KB.
P.S.:
What I am personally don't like in current approach:
1. Additional fields and flag in task struct look quite ugly
2. No way to configure the size of chunk to copy using FPSIMD from user space
3. FPSIMD-based memory movement is not generic, need to enable for memmove(),
memcpy() and friends in future.
Co-developed-by: Alexander Kozhevnikov <alexander.kozhevnikov(a)huawei-partners.com>
Signed-off-by: Alexander Kozhevnikov <alexander.kozhevnikov(a)huawei-partners.com>
Co-developed-by: Nikita Panov <panov.nikita(a)huawei.com>
Signed-off-by: Nikita Panov <panov.nikita(a)huawei.com>
Signed-off-by: Artem Kuzin <artem.kuzin(a)huawei.com>
---
arch/arm64/Kconfig | 15 ++
arch/arm64/configs/openeuler_defconfig | 2 +
arch/arm64/include/asm/fpsimd.h | 15 ++
arch/arm64/include/asm/fpsimdmacros.h | 14 ++
arch/arm64/include/asm/neon.h | 28 ++++
arch/arm64/include/asm/processor.h | 10 ++
arch/arm64/include/asm/thread_info.h | 5 +
arch/arm64/include/asm/uaccess.h | 218 ++++++++++++++++++++++++-
arch/arm64/kernel/entry-fpsimd.S | 22 +++
arch/arm64/kernel/fpsimd.c | 102 +++++++++++-
arch/arm64/kernel/process.c | 2 +-
arch/arm64/lib/copy_from_user.S | 30 ++++
arch/arm64/lib/copy_template_fpsimd.S | 180 ++++++++++++++++++++
arch/arm64/lib/copy_to_user.S | 30 ++++
kernel/softirq.c | 34 ++++
kernel/sysctl.c | 34 ++++
16 files changed, 734 insertions(+), 7 deletions(-)
create mode 100644 arch/arm64/lib/copy_template_fpsimd.S
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index ef8c524a296d..15ec2232994a 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1870,6 +1870,21 @@ config ARM64_ILP32
is an ABI where long and pointers are 32bits but it uses the AARCH64
instruction set.
+config USE_VECTORIZED_COPY
+ bool "Use vectorized instructions in copy_to/from user"
+ depends on KERNEL_MODE_NEON
+ default y
+ help
+ This option turns on vectorization to speed up copy_to/from_user routines.
+
+config VECTORIZED_COPY_VALIDATE
+ bool "Validate result of vectorized copy using regular implementation"
+ depends on KERNEL_MODE_NEON
+ depends on USE_VECTORIZED_COPY
+ default n
+ help
+ This option turns on vectorization to speed up copy_to/from_user routines.
+
menuconfig AARCH32_EL0
bool "Kernel support for 32-bit EL0"
depends on ARM64_4K_PAGES || EXPERT
diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig
index 425616aa8422..331077d556ca 100644
--- a/arch/arm64/configs/openeuler_defconfig
+++ b/arch/arm64/configs/openeuler_defconfig
@@ -525,6 +525,8 @@ CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY=y
# CONFIG_RODATA_FULL_DEFAULT_ENABLED is not set
# CONFIG_ARM64_SW_TTBR0_PAN is not set
CONFIG_ARM64_TAGGED_ADDR_ABI=y
+CONFIG_USE_VECTORIZED_COPY=y
+# CONFIG_VECTORIZED_COPY_VALIDATE is not set
CONFIG_AARCH32_EL0=y
# CONFIG_KUSER_HELPERS is not set
# CONFIG_COMPAT_ALIGNMENT_FIXUPS is not set
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index b6c6949984d8..1fc9089b4a47 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -46,6 +46,21 @@
struct task_struct;
+#ifdef CONFIG_USE_VECTORIZED_COPY
+extern void fpsimd_save_state_light(struct fpsimd_state *state);
+extern void fpsimd_load_state_light(struct fpsimd_state *state);
+#else
+static inline void fpsimd_save_state_light(struct fpsimd_state *state)
+{
+ (void) state;
+}
+
+static inline void fpsimd_load_state_light(struct fpsimd_state *state)
+{
+ (void) state;
+}
+#endif
+
extern void fpsimd_save_state(struct user_fpsimd_state *state);
extern void fpsimd_load_state(struct user_fpsimd_state *state);
diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h
index cdf6a35e3994..df9d3ed91931 100644
--- a/arch/arm64/include/asm/fpsimdmacros.h
+++ b/arch/arm64/include/asm/fpsimdmacros.h
@@ -8,6 +8,20 @@
#include <asm/assembler.h>
+#ifdef CONFIG_USE_VECTORIZED_COPY
+/* Lightweight fpsimd context saving/restoration.
+ * Necessary for vectorized kernel memory movement
+ * implementation
+ */
+.macro fpsimd_save_light state
+ st1 {v20.16b, v21.16b, v22.16b, v23.16b}, [\state]
+.endm
+
+.macro fpsimd_restore_light state
+ ld1 {v20.16b, v21.16b, v22.16b, v23.16b}, [\state]
+.endm
+#endif
+
.macro fpsimd_save state, tmpnr
stp q0, q1, [\state, #16 * 0]
stp q2, q3, [\state, #16 * 2]
diff --git a/arch/arm64/include/asm/neon.h b/arch/arm64/include/asm/neon.h
index d4b1d172a79b..ab84b194d7b3 100644
--- a/arch/arm64/include/asm/neon.h
+++ b/arch/arm64/include/asm/neon.h
@@ -16,4 +16,32 @@
void kernel_neon_begin(void);
void kernel_neon_end(void);
+#ifdef CONFIG_USE_VECTORIZED_COPY
+bool kernel_fpsimd_begin(void);
+void kernel_fpsimd_end(void);
+/* Functions to use in non-preemptible context */
+void _kernel_fpsimd_save(struct fpsimd_state *state);
+void _kernel_fpsimd_load(struct fpsimd_state *state);
+#else
+bool kernel_fpsimd_begin(void)
+{
+ return false;
+}
+
+void kernel_fpsimd_end(void)
+{
+}
+
+/* Functions to use in non-preemptible context */
+void _kernel_fpsimd_save(struct fpsimd_state *state)
+{
+ (void) state;
+}
+
+void _kernel_fpsimd_load(struct fpsimd_state *state)
+{
+ (void) state;
+}
+#endif
+
#endif /* ! __ASM_NEON_H */
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 9e688b1b13d4..9b81dbcd2126 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -153,6 +153,10 @@ struct cpu_context {
unsigned long pc;
};
+struct fpsimd_state {
+ __uint128_t v[4];
+};
+
struct thread_struct {
struct cpu_context cpu_context; /* cpu context */
@@ -196,6 +200,12 @@ struct thread_struct {
KABI_RESERVE(6)
KABI_RESERVE(7)
KABI_RESERVE(8)
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ KABI_EXTEND(
+ struct fpsimd_state ustate;
+ struct fpsimd_state kstate;
+ )
+#endif
};
static inline unsigned int thread_get_vl(struct thread_struct *thread,
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index 379d24059f5b..60d0be8a2d58 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -89,6 +89,9 @@ void arch_setup_new_exec(void);
#define TIF_SME 27 /* SME in use */
#define TIF_SME_VL_INHERIT 28 /* Inherit SME vl_onexec across exec */
#define TIF_32BIT_AARCH64 29 /* 32 bit process on AArch64(ILP32) */
+#define TIF_KERNEL_FPSIMD 31 /* Use FPSIMD in kernel */
+#define TIF_PRIV_UACC_ENABLED 32 /* Whether priviliged uaccess was manually enabled */
+
#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
@@ -107,6 +110,8 @@ void arch_setup_new_exec(void);
#define _TIF_MTE_ASYNC_FAULT (1 << TIF_MTE_ASYNC_FAULT)
#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)
#define _TIF_32BIT_AARCH64 (1 << TIF_32BIT_AARCH64)
+#define _TIF_KERNEL_FPSIMD (1 << TIF_KERNEL_FPSIMD)
+#define _TIF_PRIV_UACC_ENABLED (1 << TIF_PRIV_UACC_ENABLED)
#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
_TIF_NOTIFY_RESUME | _TIF_FOREIGN_FPSTATE | \
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index dd0877a75922..fc9f1a40624d 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -26,6 +26,10 @@
#include <asm/memory.h>
#include <asm/extable.h>
+#ifndef __GENKSYMS__
+#include <asm/neon.h>
+#endif
+
static inline int __access_ok(const void __user *ptr, unsigned long size);
/*
@@ -134,7 +138,7 @@ static inline void __uaccess_enable_hw_pan(void)
CONFIG_ARM64_PAN));
}
-static inline void uaccess_disable_privileged(void)
+static inline void __uaccess_disable_privileged(void)
{
mte_disable_tco();
@@ -144,7 +148,22 @@ static inline void uaccess_disable_privileged(void)
__uaccess_enable_hw_pan();
}
-static inline void uaccess_enable_privileged(void)
+static inline void uaccess_disable_privileged(void)
+{
+ preempt_disable();
+
+ if (!test_and_clear_thread_flag(TIF_PRIV_UACC_ENABLED)) {
+ WARN_ON(1);
+ preempt_enable();
+ return;
+ }
+
+ __uaccess_disable_privileged();
+
+ preempt_enable();
+}
+
+static inline void __uaccess_enable_privileged(void)
{
mte_enable_tco();
@@ -154,6 +173,47 @@ static inline void uaccess_enable_privileged(void)
__uaccess_disable_hw_pan();
}
+static inline void uaccess_enable_privileged(void)
+{
+ preempt_disable();
+
+ if (test_and_set_thread_flag(TIF_PRIV_UACC_ENABLED)) {
+ WARN_ON(1);
+ preempt_enable();
+ return;
+ }
+
+ __uaccess_enable_privileged();
+
+ preempt_enable();
+}
+
+static inline void uaccess_priviliged_context_switch(struct task_struct *next)
+{
+ bool curr_enabled = !!test_thread_flag(TIF_PRIV_UACC_ENABLED);
+ bool next_enabled = !!test_ti_thread_flag(&next->thread_info, TIF_PRIV_UACC_ENABLED);
+
+ if (curr_enabled == next_enabled)
+ return;
+
+ if (curr_enabled)
+ __uaccess_disable_privileged();
+ else
+ __uaccess_enable_privileged();
+}
+
+static inline void uaccess_priviliged_state_save(void)
+{
+ if (test_thread_flag(TIF_PRIV_UACC_ENABLED))
+ __uaccess_disable_privileged();
+}
+
+static inline void uaccess_priviliged_state_restore(void)
+{
+ if (test_thread_flag(TIF_PRIV_UACC_ENABLED))
+ __uaccess_enable_privileged();
+}
+
/*
* Sanitize a uaccess pointer such that it cannot reach any kernel address.
*
@@ -391,7 +451,97 @@ do { \
} while (0); \
} while(0)
-extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n);
+#define USER_COPY_CHUNK_SIZE 4096
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+
+extern int sysctl_copy_from_user_threshold;
+
+#define verify_fpsimd_copy(to, from, n, ret) \
+({ \
+ unsigned long __verify_ret = 0; \
+ __verify_ret = memcmp(to, from, ret ? n - ret : n); \
+ if (__verify_ret) \
+ pr_err("FPSIMD:%s inconsistent state\n", __func__); \
+ if (ret) \
+ pr_err("FPSIMD:%s failed to copy data, expected=%lu, copied=%lu\n", __func__, n, n - ret); \
+ __verify_ret |= ret; \
+ __verify_ret; \
+})
+
+#define compare_fpsimd_copy(to, from, n, ret_fpsimd, ret) \
+({ \
+ unsigned long __verify_ret = 0; \
+ __verify_ret = memcmp(to, from, ret ? n - ret : n); \
+ if (__verify_ret) \
+ pr_err("FIXUP:%s inconsistent state\n", __func__); \
+ if (ret) \
+ pr_err("FIXUP:%s failed to copy data, expected=%lu, copied=%lu\n", __func__, n, n - ret); \
+ __verify_ret |= ret; \
+ if (ret_fpsimd != ret) { \
+ pr_err("FIXUP:%s difference between FPSIMD %lu and regular %lu\n", __func__, n - ret_fpsimd, n - ret); \
+ __verify_ret |= 1; \
+ } else { \
+ __verify_ret = 0; \
+ } \
+ __verify_ret; \
+})
+
+extern unsigned long __must_check
+__arch_copy_from_user(void *to, const void __user *from, unsigned long n);
+
+extern unsigned long __must_check
+__arch_copy_from_user_fpsimd(void *to, const void __user *from, unsigned long n);
+
+static __always_inline unsigned long __must_check
+raw_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+ unsigned long __acfu_ret;
+
+ if (sysctl_copy_from_user_threshold == -1 || n < sysctl_copy_from_user_threshold) {
+ uaccess_ttbr0_enable();
+ __acfu_ret = __arch_copy_from_user(to,
+ __uaccess_mask_ptr(from), n);
+ uaccess_ttbr0_disable();
+ } else {
+ if (kernel_fpsimd_begin()) {
+ unsigned long __acfu_ret_fpsimd;
+
+ uaccess_enable_privileged();
+ __acfu_ret_fpsimd = __arch_copy_from_user_fpsimd((to),
+ __uaccess_mask_ptr(from), n);
+ uaccess_disable_privileged();
+
+ __acfu_ret = __acfu_ret_fpsimd;
+ kernel_fpsimd_end();
+#ifdef CONFIG_VECTORIZED_COPY_VALIDATE
+ if (verify_fpsimd_copy(to, __uaccess_mask_ptr(from), n,
+ __acfu_ret)) {
+
+ uaccess_ttbr0_enable();
+ __acfu_ret = __arch_copy_from_user((to),
+ __uaccess_mask_ptr(from), n);
+ uaccess_ttbr0_disable();
+
+ compare_fpsimd_copy(to, __uaccess_mask_ptr(from), n,
+ __acfu_ret_fpsimd, __acfu_ret);
+ }
+#endif
+ } else {
+ uaccess_ttbr0_enable();
+ __acfu_ret = __arch_copy_from_user((to),
+ __uaccess_mask_ptr(from), n);
+ uaccess_ttbr0_disable();
+ }
+ }
+
+
+ return __acfu_ret;
+}
+#else
+extern unsigned long __must_check
+__arch_copy_from_user(void *to, const void __user *from, unsigned long n);
+
#define raw_copy_from_user(to, from, n) \
({ \
unsigned long __acfu_ret; \
@@ -402,7 +552,66 @@ extern unsigned long __must_check __arch_copy_from_user(void *to, const void __u
__acfu_ret; \
})
-extern unsigned long __must_check __arch_copy_to_user(void __user *to, const void *from, unsigned long n);
+#endif
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+
+extern int sysctl_copy_to_user_threshold;
+
+extern unsigned long __must_check
+__arch_copy_to_user(void __user *to, const void *from, unsigned long n);
+
+extern unsigned long __must_check
+__arch_copy_to_user_fpsimd(void __user *to, const void *from, unsigned long n);
+
+static __always_inline unsigned long __must_check
+raw_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+ unsigned long __actu_ret;
+
+
+ if (sysctl_copy_to_user_threshold == -1 || n < sysctl_copy_to_user_threshold) {
+ uaccess_ttbr0_enable();
+ __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),
+ from, n);
+ uaccess_ttbr0_disable();
+ } else {
+ if (kernel_fpsimd_begin()) {
+ unsigned long __actu_ret_fpsimd;
+
+ uaccess_enable_privileged();
+ __actu_ret_fpsimd = __arch_copy_to_user_fpsimd(__uaccess_mask_ptr(to),
+ from, n);
+ uaccess_disable_privileged();
+
+ kernel_fpsimd_end();
+ __actu_ret = __actu_ret_fpsimd;
+#ifdef CONFIG_VECTORIZED_COPY_VALIDATE
+ if (verify_fpsimd_copy(__uaccess_mask_ptr(to), from, n,
+ __actu_ret)) {
+ uaccess_ttbr0_enable();
+ __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),
+ from, n);
+ uaccess_ttbr0_disable();
+
+ compare_fpsimd_copy(__uaccess_mask_ptr(to), from, n,
+ __actu_ret_fpsimd, __actu_ret);
+ }
+#endif
+ } else {
+ uaccess_ttbr0_enable();
+ __actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to),
+ from, n);
+ uaccess_ttbr0_disable();
+ }
+ }
+
+ return __actu_ret;
+}
+#else
+extern unsigned long __must_check
+__arch_copy_to_user(void __user *to, const void *from, unsigned long n);
+
#define raw_copy_to_user(to, from, n) \
({ \
unsigned long __actu_ret; \
@@ -412,6 +621,7 @@ extern unsigned long __must_check __arch_copy_to_user(void __user *to, const voi
uaccess_ttbr0_disable(); \
__actu_ret; \
})
+#endif
static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len)
{
diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S
index 6325db1a2179..6660465f1b7c 100644
--- a/arch/arm64/kernel/entry-fpsimd.S
+++ b/arch/arm64/kernel/entry-fpsimd.S
@@ -11,6 +11,28 @@
#include <asm/assembler.h>
#include <asm/fpsimdmacros.h>
+#ifdef CONFIG_USE_VECTORIZED_COPY
+/*
+ * Save the FP registers.
+ *
+ * x0 - pointer to struct fpsimd_state_light
+ */
+SYM_FUNC_START(fpsimd_save_state_light)
+ fpsimd_save_light x0
+ ret
+SYM_FUNC_END(fpsimd_save_state_light)
+
+/*
+ * Load the FP registers.
+ *
+ * x0 - pointer to struct fpsimd_state_light
+ */
+SYM_FUNC_START(fpsimd_load_state_light)
+ fpsimd_restore_light x0
+ ret
+SYM_FUNC_END(fpsimd_load_state_light)
+#endif
+
/*
* Save the FP registers.
*
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index b86a50646700..103559cccb07 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -1579,6 +1579,11 @@ void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs)
current);
}
+#ifdef CONFIG_USE_VECTORIZED_COPY
+static void kernel_fpsimd_rollback_changes(void);
+static void kernel_fpsimd_restore_changes(struct task_struct *tsk);
+#endif
+
void fpsimd_thread_switch(struct task_struct *next)
{
bool wrong_task, wrong_cpu;
@@ -1587,10 +1592,11 @@ void fpsimd_thread_switch(struct task_struct *next)
return;
__get_cpu_fpsimd_context();
-
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ kernel_fpsimd_rollback_changes();
+#endif
/* Save unsaved fpsimd state, if any: */
fpsimd_save();
-
/*
* Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
* state. For kernel threads, FPSIMD registers are never loaded
@@ -1603,6 +1609,9 @@ void fpsimd_thread_switch(struct task_struct *next)
update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
wrong_task || wrong_cpu);
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ kernel_fpsimd_restore_changes(next);
+#endif
__put_cpu_fpsimd_context();
}
@@ -1933,6 +1942,95 @@ void kernel_neon_end(void)
}
EXPORT_SYMBOL_GPL(kernel_neon_end);
+#ifdef CONFIG_USE_VECTORIZED_COPY
+bool kernel_fpsimd_begin(void)
+{
+ if (WARN_ON(!system_capabilities_finalized()) ||
+ !system_supports_fpsimd() ||
+ in_irq() || irqs_disabled() || in_nmi())
+ return false;
+
+ preempt_disable();
+ if (test_and_set_thread_flag(TIF_KERNEL_FPSIMD)) {
+ preempt_enable();
+
+ WARN_ON(1);
+ return false;
+ }
+
+ /*
+ * Leaving streaming mode enabled will cause issues for any kernel
+ * NEON and leaving streaming mode or ZA enabled may increase power
+ * consumption.
+ */
+ if (system_supports_sme())
+ sme_smstop();
+
+ fpsimd_save_state_light(¤t->thread.ustate);
+ preempt_enable();
+
+ return true;
+}
+EXPORT_SYMBOL(kernel_fpsimd_begin);
+
+void kernel_fpsimd_end(void)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ preempt_disable();
+ if (test_and_clear_thread_flag(TIF_KERNEL_FPSIMD))
+ fpsimd_load_state_light(¤t->thread.ustate);
+
+ preempt_enable();
+}
+EXPORT_SYMBOL(kernel_fpsimd_end);
+
+void _kernel_fpsimd_save(struct fpsimd_state *state)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ BUG_ON(preemptible());
+ if (test_thread_flag(TIF_KERNEL_FPSIMD))
+ fpsimd_save_state_light(state);
+}
+
+void _kernel_fpsimd_load(struct fpsimd_state *state)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ BUG_ON(preemptible());
+ if (test_thread_flag(TIF_KERNEL_FPSIMD))
+ fpsimd_load_state_light(state);
+}
+
+static void kernel_fpsimd_rollback_changes(void)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ BUG_ON(preemptible());
+ if (test_thread_flag(TIF_KERNEL_FPSIMD)) {
+ fpsimd_save_state_light(¤t->thread.kstate);
+ fpsimd_load_state_light(¤t->thread.ustate);
+ }
+}
+
+static void kernel_fpsimd_restore_changes(struct task_struct *tsk)
+{
+ if (!system_supports_fpsimd())
+ return;
+
+ BUG_ON(preemptible());
+ if (test_ti_thread_flag(task_thread_info(tsk), TIF_KERNEL_FPSIMD)) {
+ fpsimd_save_state_light(&tsk->thread.ustate);
+ fpsimd_load_state_light(&tsk->thread.kstate);
+ }
+}
+#endif
+
#ifdef CONFIG_EFI
static DEFINE_PER_CPU(struct user_fpsimd_state, efi_fpsimd_state);
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index e9e5ce956f15..fd895189cb7e 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -529,7 +529,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
struct task_struct *next)
{
struct task_struct *last;
-
+ uaccess_priviliged_context_switch(next);
fpsimd_thread_switch(next);
tls_thread_switch(next);
hw_breakpoint_thread_switch(next);
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 34e317907524..60dc63e10233 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -71,3 +71,33 @@ USER(9998f, ldtrb tmp1w, [srcin])
ret
SYM_FUNC_END(__arch_copy_from_user)
EXPORT_SYMBOL(__arch_copy_from_user)
+
+
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ .macro ldsve reg1, reg2, reg3, reg4, ptr
+ USER(9997f, ld1 {\reg1, \reg2, \reg3, \reg4}, [\ptr])
+ .endm
+
+ .macro stsve reg1, reg2, reg3, reg4, ptr
+ KERNEL_ME_SAFE(9998f, st1 {\reg1, \reg2, \reg3, \reg4}, [\ptr])
+ .endm
+
+SYM_FUNC_START(__arch_copy_from_user_fpsimd)
+ add end, x0, x2
+ mov srcin, x1
+#include "copy_template_fpsimd.S"
+ mov x0, #0 // Nothing to copy
+ ret
+
+ // Exception fixups
+9997: cmp dst, dstin
+ b.ne 9998f
+ // Before being absolutely sure we couldn't copy anything, try harder
+USER(9998f, ldtrb tmp1w, [srcin])
+ strb tmp1w, [dst], #1
+9998: sub x0, end, dst // bytes not copied
+ ret
+SYM_FUNC_END(__arch_copy_from_user_fpsimd)
+EXPORT_SYMBOL(__arch_copy_from_user_fpsimd)
+#endif
\ No newline at end of file
diff --git a/arch/arm64/lib/copy_template_fpsimd.S b/arch/arm64/lib/copy_template_fpsimd.S
new file mode 100644
index 000000000000..9b2e7ce1e4d2
--- /dev/null
+++ b/arch/arm64/lib/copy_template_fpsimd.S
@@ -0,0 +1,180 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2013 ARM Ltd.
+ * Copyright (C) 2013 Linaro.
+ *
+ * This code is based on glibc cortex strings work originally authored by Linaro
+ * be found @
+ *
+ * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
+ * files/head:/src/aarch64/
+ */
+
+/*
+ * Copy a buffer from src to dest (alignment handled by the hardware)
+ *
+ * Parameters:
+ * x0 - dest
+ * x1 - src
+ * x2 - n
+ * Returns:
+ * x0 - dest
+ */
+dstin .req x0
+src .req x1
+count .req x2
+tmp1 .req x3
+tmp1w .req w3
+tmp2 .req x4
+tmp2w .req w4
+dst .req x6
+
+A_l .req x7
+A_h .req x8
+B_l .req x9
+B_h .req x10
+C_l .req x11
+C_h .req x12
+D_l .req x13
+D_h .req x14
+
+V_a .req v20
+V_b .req v21
+V_c .req v22
+V_d .req v23
+
+ mov dst, dstin
+ cmp count, #16
+ /*When memory length is less than 16, the accessed are not aligned.*/
+ b.lo .Ltiny15_fpsimd
+
+ neg tmp2, src
+ ands tmp2, tmp2, #15/* Bytes to reach alignment. */
+ b.eq .LSrcAligned_fpsimd
+ sub count, count, tmp2
+ /*
+ * Copy the leading memory data from src to dst in an increasing
+ * address order.By this way,the risk of overwriting the source
+ * memory data is eliminated when the distance between src and
+ * dst is less than 16. The memory accesses here are alignment.
+ */
+ tbz tmp2, #0, 1f
+ ldrb1 tmp1w, src, #1
+ strb1 tmp1w, dst, #1
+1:
+ tbz tmp2, #1, 2f
+ ldrh1 tmp1w, src, #2
+ strh1 tmp1w, dst, #2
+2:
+ tbz tmp2, #2, 3f
+ ldr1 tmp1w, src, #4
+ str1 tmp1w, dst, #4
+3:
+ tbz tmp2, #3, .LSrcAligned_fpsimd
+ ldr1 tmp1, src, #8
+ str1 tmp1, dst, #8
+
+.LSrcAligned_fpsimd:
+ cmp count, #64
+ b.ge .Lcpy_over64_fpsimd
+ /*
+ * Deal with small copies quickly by dropping straight into the
+ * exit block.
+ */
+.Ltail63_fpsimd:
+ /*
+ * Copy up to 48 bytes of data. At this point we only need the
+ * bottom 6 bits of count to be accurate.
+ */
+ ands tmp1, count, #0x30
+ b.eq .Ltiny15_fpsimd
+ cmp tmp1w, #0x20
+ b.eq 1f
+ b.lt 2f
+ ldp1 A_l, A_h, src, #16
+ stp1 A_l, A_h, dst, #16
+1:
+ ldp1 A_l, A_h, src, #16
+ stp1 A_l, A_h, dst, #16
+2:
+ ldp1 A_l, A_h, src, #16
+ stp1 A_l, A_h, dst, #16
+.Ltiny15_fpsimd:
+ /*
+ * Prefer to break one ldp/stp into several load/store to access
+ * memory in an increasing address order,rather than to load/store 16
+ * bytes from (src-16) to (dst-16) and to backward the src to aligned
+ * address,which way is used in original cortex memcpy. If keeping
+ * the original memcpy process here, memmove need to satisfy the
+ * precondition that src address is at least 16 bytes bigger than dst
+ * address,otherwise some source data will be overwritten when memove
+ * call memcpy directly. To make memmove simpler and decouple the
+ * memcpy's dependency on memmove, withdrew the original process.
+ */
+ tbz count, #3, 1f
+ ldr1 tmp1, src, #8
+ str1 tmp1, dst, #8
+1:
+ tbz count, #2, 2f
+ ldr1 tmp1w, src, #4
+ str1 tmp1w, dst, #4
+2:
+ tbz count, #1, 3f
+ ldrh1 tmp1w, src, #2
+ strh1 tmp1w, dst, #2
+3:
+ tbz count, #0, .Lexitfunc_fpsimd
+ ldrb1 tmp1w, src, #1
+ strb1 tmp1w, dst, #1
+
+ b .Lexitfunc_fpsimd
+
+.Lcpy_over64_fpsimd:
+ subs count, count, #128
+ b.ge .Lcpy_body_large_fpsimd
+ /*
+ * Less than 128 bytes to copy, so handle 64 here and then jump
+ * to the tail.
+ */
+ ldp1 A_l, A_h, src, #16
+ stp1 A_l, A_h, dst, #16
+ ldp1 B_l, B_h, src, #16
+ ldp1 C_l, C_h, src, #16
+ stp1 B_l, B_h, dst, #16
+ stp1 C_l, C_h, dst, #16
+ ldp1 D_l, D_h, src, #16
+ stp1 D_l, D_h, dst, #16
+
+ tst count, #0x3f
+ b.ne .Ltail63_fpsimd
+ b .Lexitfunc_fpsimd
+
+ /*
+ * Critical loop. Start at a new cache line boundary. Assuming
+ * 64 bytes per line this ensures the entire loop is in one line.
+ */
+ .p2align L1_CACHE_SHIFT
+.Lcpy_body_large_fpsimd:
+ /* pre-get 64 bytes data. */
+ ldsve V_a.16b, V_b.16b, V_c.16b, V_d.16b, src
+ add src, src, #64
+
+1:
+ /*
+ * interlace the load of next 64 bytes data block with store of the last
+ * loaded 64 bytes data.
+ */
+ stsve V_a.16b, V_b.16b, V_c.16b, V_d.16b, dst
+ ldsve V_a.16b, V_b.16b, V_c.16b, V_d.16b, src
+ add dst, dst, #64
+ add src, src, #64
+
+ subs count, count, #64
+ b.ge 1b
+
+ stsve V_a.16b, V_b.16b, V_c.16b, V_d.16b, dst
+ add dst, dst, #64
+
+ tst count, #0x3f
+ b.ne .Ltail63_fpsimd
+.Lexitfunc_fpsimd:
diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
index 2ac716c0d6d8..c190e5f8a989 100644
--- a/arch/arm64/lib/copy_to_user.S
+++ b/arch/arm64/lib/copy_to_user.S
@@ -71,3 +71,33 @@ USER(9998f, sttrb tmp1w, [dst])
ret
SYM_FUNC_END(__arch_copy_to_user)
EXPORT_SYMBOL(__arch_copy_to_user)
+
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ .macro stsve reg1, reg2, reg3, reg4, ptr
+ USER(9997f, st1 {\reg1, \reg2, \reg3, \reg4}, [\ptr])
+ .endm
+
+ .macro ldsve reg1, reg2, reg3, reg4, ptr
+ KERNEL_ME_SAFE(9998f, ld1 {\reg1, \reg2, \reg3, \reg4}, [\ptr])
+ .endm
+
+SYM_FUNC_START(__arch_copy_to_user_fpsimd)
+ add end, x0, x2
+ mov srcin, x1
+#include "copy_template_fpsimd.S"
+ mov x0, #0
+ ret
+
+ // Exception fixups
+9997: cmp dst, dstin
+ b.ne 9998f
+ // Before being absolutely sure we couldn't copy anything, try harder
+KERNEL_ME_SAFE(9998f, ldrb tmp1w, [srcin])
+USER(9998f, sttrb tmp1w, [dst])
+ add dst, dst, #1
+9998: sub x0, end, dst // bytes not copied
+ ret
+SYM_FUNC_END(__arch_copy_to_user_fpsimd)
+EXPORT_SYMBOL(__arch_copy_to_user_fpsimd)
+#endif
diff --git a/kernel/softirq.c b/kernel/softirq.c
index bd10ff418865..9935a11be1e8 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -30,6 +30,10 @@
#include <asm/softirq_stack.h>
+#ifdef CONFIG_USE_VECTORIZED_COPY
+#include <asm/fpsimd.h>
+#endif
+
#define CREATE_TRACE_POINTS
#include <trace/events/irq.h>
@@ -542,6 +546,9 @@ static void handle_softirqs(bool ksirqd)
__u32 pending;
int softirq_bit;
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ struct fpsimd_state state;
+#endif
/*
* Mask out PF_MEMALLOC as the current task context is borrowed for the
* softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
@@ -551,10 +558,16 @@ static void handle_softirqs(bool ksirqd)
pending = local_softirq_pending();
+
softirq_handle_begin();
in_hardirq = lockdep_softirq_start();
account_softirq_enter(current);
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ _kernel_fpsimd_save(&state);
+ uaccess_priviliged_state_save();
+#endif
+
restart:
/* Reset the pending bitmask before enabling irqs */
set_softirq_pending(0);
@@ -603,7 +616,14 @@ static void handle_softirqs(bool ksirqd)
account_softirq_exit(current);
lockdep_softirq_end(in_hardirq);
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ uaccess_priviliged_state_restore();
+ _kernel_fpsimd_load(&state);
+#endif
+
softirq_handle_end();
+
current_restore_flags(old_flags, PF_MEMALLOC);
}
@@ -837,12 +857,21 @@ static void tasklet_action_common(struct softirq_action *a,
{
struct tasklet_struct *list;
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ struct fpsimd_state state;
+#endif
+
local_irq_disable();
list = tl_head->head;
tl_head->head = NULL;
tl_head->tail = &tl_head->head;
local_irq_enable();
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ _kernel_fpsimd_save(&state);
+ uaccess_priviliged_state_save();
+#endif
+
while (list) {
struct tasklet_struct *t = list;
@@ -874,6 +903,11 @@ static void tasklet_action_common(struct softirq_action *a,
__raise_softirq_irqoff(softirq_nr);
local_irq_enable();
}
+
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ uaccess_priviliged_state_restore();
+ _kernel_fpsimd_load(&state);
+#endif
}
static __latent_entropy void tasklet_action(struct softirq_action *a)
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index e84df0818517..6f8e22102bdc 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -137,6 +137,17 @@ int sysctl_legacy_va_layout;
#endif /* CONFIG_SYSCTL */
+#ifdef CONFIG_USE_VECTORIZED_COPY
+int sysctl_copy_to_user_threshold = -1;
+EXPORT_SYMBOL(sysctl_copy_to_user_threshold);
+
+int sysctl_copy_from_user_threshold = -1;
+EXPORT_SYMBOL(sysctl_copy_from_user_threshold);
+
+int sysctl_copy_in_user_threshold = -1;
+EXPORT_SYMBOL(sysctl_copy_in_user_threshold);
+#endif
+
/*
* /proc/sys support
*/
@@ -2250,6 +2261,29 @@ static struct ctl_table vm_table[] = {
.extra1 = (void *)&mmap_rnd_compat_bits_min,
.extra2 = (void *)&mmap_rnd_compat_bits_max,
},
+#endif
+#ifdef CONFIG_USE_VECTORIZED_COPY
+ {
+ .procname = "copy_to_user_threshold",
+ .data = &sysctl_copy_to_user_threshold,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec
+ },
+ {
+ .procname = "copy_from_user_threshold",
+ .data = &sysctl_copy_from_user_threshold,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec
+ },
+ {
+ .procname = "copy_in_user_threshold",
+ .data = &sysctl_copy_in_user_threshold,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec
+ },
#endif
{ }
};
--
2.34.1
2
1