In the test of execl, shell1 and shell8 of UnixBench, L3 false sharing occurs between rwsem_try_write_lock_unqueued() and filemap_map_pages().
The offset between address_space.host and address_space.i_mmap_rwsem is 48. It may occur L3 false sharing. Their offsets in struct ext4_inode_info is 696 and 744, so when the address of ext4_inode_info after L3 aligned, it may occur L3 false sharing in the following condition:
[0x00 ~ 0x10] false sharing [0x18 ~ 0x40] no false sharing [0x48 ~ 0x80] false sharing
Change the offset of 'vfs_inode' from 320 to 360 in ext4_inode_info and make the address of ext4_inode_info L3 aligned, so the offset of host and i_mmap_rwsem in ext4_inode_info is changed to 736 and 784, it can make them in different L3 to avoid false sharing.
./Run -c 96 -i 3 execl
Before this patch: System Benchmarks Partial Index BASELINE RESULT INDEX Execl Throughput 43.0 24238.0 5636.8 ======== System Benchmarks Index Score (Partial Only) 5636.8
After this patch: System Benchmarks Partial Index BASELINE RESULT INDEX Execl Throughput 43.0 29363.7 6828.8 ======== System Benchmarks Index Score (Partial Only) 6828.8
Yang Yingliang (2): ext4: mitigatin cacheline false sharing in struct ext4_inode_info enable MITIGATION_FALSE_SHARING by default
arch/arm64/configs/openeuler_defconfig | 1 + arch/x86/configs/openeuler_defconfig | 1 + fs/ext4/Kconfig | 9 +++++++++ fs/ext4/ext4.h | 5 +++++ fs/ext4/super.c | 4 ++++ 5 files changed, 20 insertions(+)
hulk inclusion category: performance bugzilla: https://gitee.com/openeuler/kernel/issues/I7ADMY
--------------------------------
In the test of execl, shell1 and shell8 of UnixBench, L3 false sharing occurs between rwsem_try_write_lock_unqueued() and filemap_map_pages().
The offset between address_space.host and address_space.i_mmap_rwsem is 48. It may occur L3 false sharing. Their offsets in struct ext4_inode_info is 696 and 744, so when the address of ext4_inode_info after L3 aligned, it may occur L3 false sharing in the following condition:
[0x00 ~ 0x10] false sharing [0x18 ~ 0x40] no false sharing [0x48 ~ 0x80] false sharing
Change the offset of 'vfs_inode' from 320 to 360 in ext4_inode_info and make the address of ext4_inode_info L3 aligned, so the offset of host and i_mmap_rwsem in ext4_inode_info is changed to 736 and 784, it can make them in different L3 to avoid false sharing.
./Run -c 96 -i 3 execl
Before this patch: System Benchmarks Partial Index BASELINE RESULT INDEX Execl Throughput 43.0 24238.0 5636.8 ======== System Benchmarks Index Score (Partial Only) 5636.8
After this patch: System Benchmarks Partial Index BASELINE RESULT INDEX Execl Throughput 43.0 29363.7 6828.8 ======== System Benchmarks Index Score (Partial Only) 6828.8
Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- fs/ext4/Kconfig | 9 +++++++++ fs/ext4/ext4.h | 5 +++++ fs/ext4/super.c | 4 ++++ 3 files changed, 18 insertions(+)
diff --git a/fs/ext4/Kconfig b/fs/ext4/Kconfig index 86699c8cab28..62e9a967f0ed 100644 --- a/fs/ext4/Kconfig +++ b/fs/ext4/Kconfig @@ -117,3 +117,12 @@ config EXT4_KUNIT_TESTS to the KUnit documentation in Documentation/dev-tools/kunit/.
If unsure, say N. + +config EXT4_MITIGATION_FALSE_SHARING + bool "mitigation false sharing in ext4 inode" + depends on EXT4_FS + default n + help + Enable this to mitigation cacheline false sharing in ext4 inode info. + + If unsure, say N. diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 5d5ae6f44510..45aac9bcd794 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -1090,7 +1090,9 @@ struct ext4_inode_info { * to occasionally drop it. */ struct rw_semaphore i_mmap_sem; +#ifndef CONFIG_EXT4_MITIGATION_FALSE_SHARING struct inode vfs_inode; +#endif struct jbd2_inode *jinode;
spinlock_t i_raw_lock; /* protects updates to the raw inode */ @@ -1103,6 +1105,9 @@ struct ext4_inode_info {
/* mballoc */ atomic_t i_prealloc_active; +#ifdef CONFIG_EXT4_MITIGATION_FALSE_SHARING + struct inode vfs_inode; +#endif struct list_head i_prealloc_list; spinlock_t i_prealloc_lock;
diff --git a/fs/ext4/super.c b/fs/ext4/super.c index c096cebcea6b..52c0eb7552b6 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1420,7 +1420,11 @@ static void init_once(void *foo) static int __init init_inodecache(void) { ext4_inode_cachep = kmem_cache_create_usercopy("ext4_inode_cache", +#ifdef CONFIG_EXT4_MITIGATION_FALSE_SHARING + sizeof(struct ext4_inode_info), 128, +#else sizeof(struct ext4_inode_info), 0, +#endif (SLAB_RECLAIM_ACCOUNT|SLAB_MEM_SPREAD| SLAB_ACCOUNT), offsetof(struct ext4_inode_info, i_data),
hulk inclusion category: performance bugzilla: https://gitee.com/openeuler/kernel/issues/I7ADMY
--------------------------------
Enable MITIGATION_FALSE_SHARING by default on x86 and arm64
Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- arch/arm64/configs/openeuler_defconfig | 1 + arch/x86/configs/openeuler_defconfig | 1 + 2 files changed, 2 insertions(+)
diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index eaec6afffc27..9b4b46addcc4 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -6208,6 +6208,7 @@ CONFIG_EXT4_USE_FOR_EXT2=y CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_EXT4_FS_SECURITY=y # CONFIG_EXT4_DEBUG is not set +CONFIG_EXT4_MITIGATION_FALSE_SHARING=y CONFIG_JBD2=m # CONFIG_JBD2_DEBUG is not set CONFIG_FS_MBCACHE=m diff --git a/arch/x86/configs/openeuler_defconfig b/arch/x86/configs/openeuler_defconfig index 2f150404db7b..564bd3856579 100644 --- a/arch/x86/configs/openeuler_defconfig +++ b/arch/x86/configs/openeuler_defconfig @@ -7306,6 +7306,7 @@ CONFIG_EXT4_USE_FOR_EXT2=y CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_EXT4_FS_SECURITY=y # CONFIG_EXT4_DEBUG is not set +CONFIG_EXT4_MITIGATION_FALSE_SHARING=y CONFIG_JBD2=m # CONFIG_JBD2_DEBUG is not set CONFIG_FS_MBCACHE=m
-
Yang Yingliang