[PATCH OLK-6.6] add test support for container

16 Sep 2025

---
 tools/container/client_kill.sh   |  8 ++++
 tools/container/create.sh        | 75 ++++++++++++++++++++++++++++++++
 tools/container/edit_client.sh   | 34 +++++++++++++++
 tools/container/env.sh           | 31 +++++++++++++
 tools/container/get_specific.sh  | 19 ++++++++
 tools/container/run_all.sh       | 13 ++++++
 tools/container/run_bk.sh        | 16 +++++++
 tools/container/run_bk_117.sh    |  7 +++
 tools/container/runc_parallel.sh | 41 +++++++++++++++++
 tools/container/usage.rst        | 56 ++++++++++++++++++++++++
 10 files changed, 300 insertions(+)
 create mode 100644 tools/container/client_kill.sh
 create mode 100644 tools/container/create.sh
 create mode 100644 tools/container/edit_client.sh
 create mode 100644 tools/container/env.sh
 create mode 100644 tools/container/get_specific.sh
 create mode 100644 tools/container/run_all.sh
 create mode 100644 tools/container/run_bk.sh
 create mode 100644 tools/container/run_bk_117.sh
 create mode 100644 tools/container/runc_parallel.sh
 create mode 100644 tools/container/usage.rst

diff --git a/tools/container/client_kill.sh b/tools/container/client_kill.sh
new file mode 100644
index 000000000000..a12c5be1edef
--- /dev/null
+++ b/tools/container/client_kill.sh
@@ -0,0 +1,8 @@
+source /home/name/common/env.sh
+
+for IP in "${CLIENT[@]}"
+do
+    echo $IP
+    ssh root@$IP "pgrep -f redis-benchmark | xargs kill -9" 
+done
+ssh root@$CLIENT_OBSERVE "pgrep -f redis-benchmark | xargs kill -9"
diff --git a/tools/container/create.sh b/tools/container/create.sh
new file mode 100644
index 000000000000..3f507bfc39cd
--- /dev/null
+++ b/tools/container/create.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+source /home/name/common/env.sh
+
+nr=$1
+mode=${2:-"sequential"}  # 默认顺序模式，可选：sequential(顺序)或interleave(交错)
+[[ -z $nr ]] && { echo "Usage: $0 <number> [sequential|interleave]"; exit 1; }
+
+docker stop $(docker ps -aq) 2>/dev/null
+docker rm $(docker ps -aq) 2>/dev/null
+
+mkdir -p /var/run/netns
+systemctl stop irqbalance
+
+if [[ $mode == "interleave" ]]; then
+    deployment_order=()
+    for i in $(seq 0 $((nr - 1))); do
+        for j in $(seq $numa_start $numa_end); do
+            deployment_order+=("$j $i")
+        done
+    done
+else
+    deployment_order=()
+    for j in $(seq $numa_start $numa_end); do
+        for i in $(seq 0 $((nr - 1))); do
+            deployment_order+=("$j $i")
+        done
+    done
+fi
+
+ip_count=0
+for deployment in "${deployment_order[@]}"; do
+    j=$(echo $deployment | awk '{print $1}')
+    i=$(echo $deployment | awk '{print $2}')
+    
+    docker run --cpus=2 --cpuset-cpus=${cpus_numa[$j]} --cpuset-mems=$j -m 8g \
+        --cap-add CAP_SYS_ADMIN --privileged=true -itd \
+        --name redis-docker-bridge-numa$j-$i \
+        -v /home/name/redis_image:/home -v /usr:/usr -v /mnt:/mnt \
+        -v /lib/modules:/lib/modules -v /data:/data -v /etc:/etc \
+        openeuler-22.03-lts-sp4 /bin/bash
+
+    pid=$(docker inspect -f '{{.State.Pid}}' redis-docker-bridge-numa$j-$i)
+    vf=$((j * nr + i))
+    ip_count=$((ip_count + 1))
+    IP=${UNREACHABLE_IPS[$((ip_count))]}
+
+    echo "vf: ${vf} IP: ${IP}"   
+ 
+    ln -sf /proc/$pid/ns/net /var/run/netns/redis-docker-bridge-numa$j-$i
+    ip link set enp24s0f0v$vf netns redis-docker-bridge-numa$j-$i
+    docker exec redis-docker-bridge-numa$j-$i ifconfig enp24s0f0v$vf $IP/16
+
+    docker exec -d redis-docker-bridge-numa$j-$i \
+        /home/c00838100/redis-origin/src/redis-server \
+        /home/c00838100/redis.conf --bind 0.0.0.0 --port 6379
+
+    while ! docker exec redis-docker-bridge-numa$j-$i ps -ef | grep -q "[r]edis-server"; do
+        sleep 0.5
+    done
+    
+    pid=$(docker exec redis-docker-bridge-numa$j-$i pgrep -f redis-server)
+    core_array=(${cpus[$j]})
+    cpu_core=${core_array[$((i * CPU_DIST))]}
+    docker exec redis-docker-bridge-numa$j-$i taskset -pc $cpu_core $pid
+
+    vf_num=$vf
+    vf_dev=$(ls -la /sys/class/net/enp24s0f0np0/device/virtfn${vf_num} | awk '{print $11}' | sed "s|../||g")
+    irq_core=$((cpu_core + 1))
+
+    for irq in `cat /proc/interrupts | grep $vf_dev | awk -F ':' '{print $1}'`; do    
+        echo $irq_core > /proc/irq/$irq/smp_affinity_list
+    done
+done
+
+echo "部署完成! 模式: $mode, 容器数量: $((nr * (numa_end - numa_start + 1)))"
diff --git a/tools/container/edit_client.sh b/tools/container/edit_client.sh
new file mode 100644
index 000000000000..7242d49de753
--- /dev/null
+++ b/tools/container/edit_client.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+source /home/name/common/env.sh
+
+connection=40
+data_len=3
+thread=50
+data_num=2000000
+data_num_large=200000000
+
+mod=0
+for IP in "${CLIENT[@]}"
+do
+echo $IP
+    cp run_bk.sh run.sh
+    echo $mod
+    sed -i "s|xx|$mod|g" run.sh
+    sed -i "s|aa|$connection|g" run.sh
+    sed -i "s|bb|$data_len|g" run.sh
+    sed -i "s|cc|$thread|g" run.sh
+    sed -i "s|dd|$data_num_large|g" run.sh
+
+    ssh root@$IP mkdir -p /home/name/common
+    scp /home/name/common/env.sh root@$IP:/home/name/common
+    scp run.sh root@$IP:/home/name/redis/client/
+    mod=$((mod + 1))
+done
+
+cp run_bk_117.sh run_117.sh
+sed -i "s|aa|$connection|g" run_117.sh
+sed -i "s|bb|$data_len|g" run_117.sh
+sed -i "s|cc|$thread|g" run_117.sh
+sed -i "s|dd|$data_num|g" run_117.sh
+scp run_117.sh root@$CLIENT_OBSERVE:/home/name/redis/client/run.sh
diff --git a/tools/container/env.sh b/tools/container/env.sh
new file mode 100644
index 000000000000..8ec051d67708
--- /dev/null
+++ b/tools/container/env.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+CPU_DIST=4
+
+cpus_numa=(
+"0-79" // todo
+)
+
+vf_core=317 // todo
+redis_core=316
+
+numa_start=0
+numa_end=3
+
+cpus=(
+"0 1"// todo
+"80 81"
+)
+
+CLIENT=(
+ip1 // todo
+ip2 // todo
+)
+
+count=80
+CLIENT_OBSERVE=ip3 // todo
+
+UNREACHABLE_IPS=(
+ip1 // todo
+ip2 // todo
+)
diff --git a/tools/container/get_specific.sh b/tools/container/get_specific.sh
new file mode 100644
index 000000000000..4bbbdac6e0b2
--- /dev/null
+++ b/tools/container/get_specific.sh
@@ -0,0 +1,19 @@
+nr=$1
+DIST=4
+
+dir=/home/name/redis_image/test_inbalance_new/result
+echo "****************"
+cat ${dir}/test_1/log_specific | grep throughput | awk '{print $3}'
+for i in $(seq 1 $nr); do
+cat ${dir}/test_$((i * DIST))/log_specific | grep throughput | awk '{print $3}'
+done
+echo "****************"
+cat ${dir}/test_1/log_specific | grep -A 2 "latency summary" | tail -n 1 | awk '{print $1}'
+for i in $(seq 1 $nr); do
+cat ${dir}/test_$((i * DIST))/log_specific | grep -A 2 "latency summary" | tail -n 1 | awk '{print $1}'
+done
+echo "****************"
+cat ${dir}/test_1/log_specific | grep -A 2 "latency summary" | tail -n 1 | awk '{print $5}'
+for i in $(seq 1 $nr); do
+cat ${dir}/test_$((i * DIST))/log_specific | grep -A 2 "latency summary" | tail -n 1 | awk '{print $5}'
+done
diff --git a/tools/container/run_all.sh b/tools/container/run_all.sh
new file mode 100644
index 000000000000..33d40b7d397e
--- /dev/null
+++ b/tools/container/run_all.sh
@@ -0,0 +1,13 @@
+source /home/name/common/env.sh
+
+rm -rf result/test*
+ssh root@$CLIENT_OBSERVE rm -rf /home/name/redis/client/test*
+nr=$1
+
+sh runc_parallel.sh 1
+sleep 3
+
+for i in $(seq 1 $nr); do
+sh runc_parallel.sh $((i * 4))
+sleep 3
+done
diff --git a/tools/container/run_bk.sh b/tools/container/run_bk.sh
new file mode 100644
index 000000000000..794d390b536e
--- /dev/null
+++ b/tools/container/run_bk.sh
@@ -0,0 +1,16 @@
+nr=$1
+
+source /home/name/common/env.sh
+dir=/home/name/redis/client/test_$((nr + 1))
+
+rm -rf $dir
+mkdir $dir
+
+for (( i=1; i<=$nr; i++ ))
+do
+    if [ $((i % 4)) -eq xx ]; then
+        redis-benchmark -h ${UNREACHABLE_IPS[$i]} -p 6379 -c aa -d bb -n dd -r 10000000  -t get --threads cc > $dir/log_${i} &
+#        redis-benchmark -h ${UNREACHABLE_IPS[$((80-i))]} -p 6379 -c aa -d bb -n dd -r 10000000  -t get --threads cc > $dir/log_${i} &
+fi
+done
+wait
diff --git a/tools/container/run_bk_117.sh b/tools/container/run_bk_117.sh
new file mode 100644
index 000000000000..16315a256632
--- /dev/null
+++ b/tools/container/run_bk_117.sh
@@ -0,0 +1,7 @@
+nr=$1
+IP=ip_observe // todo
+
+#sleep 8
+rm -rf /home/name/redis/client/test_$nr
+mkdir /home/name/redis/client/test_$nr
+redis-benchmark -h $IP -p 6379 -c aa -d bb -n dd -r 10000000  -t get --threads cc > /home/name/redis/client/test_$nr/log_specific
diff --git a/tools/container/runc_parallel.sh b/tools/container/runc_parallel.sh
new file mode 100644
index 000000000000..47b48e88e703
--- /dev/null
+++ b/tools/container/runc_parallel.sh
@@ -0,0 +1,41 @@
+nr=$1
+
+source /home/name/common/env.sh
+sh client_kill.sh
+function cleanup() {
+ssh root@$CLIENT_OBSERVE "rm -rf /home/name/redis/client/test_*"
+for IP in "${CLIENT[@]}"
+do
+    ssh root@$IP "rm -rf /home/name/redis/client/test_*"
+done
+}
+
+function copy() {
+scp -r root@$CLIENT_OBSERVE:/home/name/redis/client/test_$((nr + 1)) result
+for IP in "${CLIENT[@]}"
+do
+    scp -r root@$IP:/home/name/redis/client/test_$((nr + 1)) result &> /dev/null
+done
+}
+
+function stress() {
+nr=$1
+for IP in "${CLIENT[@]}"
+do
+echo -n $IP
+    ssh root@$IP "sh /home/name/redis/client/run.sh $nr" &
+echo done
+done
+}
+
+nr=$((nr - 1))
+
+cleanup
+stress $nr
+sleep 2
+ssh root@$CLIENT_OBSERVE "sh /home/name/redis/client/run.sh $((nr + 1))"
+
+sleep 15
+mkdir result
+copy
+cat result/test_$((nr + 1))/log_specific | grep -A 4 throughput
diff --git a/tools/container/usage.rst b/tools/container/usage.rst
new file mode 100644
index 000000000000..beb2593b8142
--- /dev/null
+++ b/tools/container/usage.rst
@@ -0,0 +1,56 @@
+镜像下载：wget https://dl-cdn.openeuler.openatom.cn/openEuler-22.03-LTS-SP4/docker_img/aarc...
+导入容器镜像：docker load < ./openEuler-docker.aarch64.tar.xz
+redis下载：https://redis.io/downloads/  版本6.2
+修改点：
+protected-mode no
+daemonize yes
+cd redis-xx && make && make install
+
+# 1. runc
+基线测试方法： 6.6.0-92.0.0.96.oe2403sp2.aarch64
+1. 硬件：服务器，CX5双网口网卡共160个vf 
+2. redis容器：每个2u8g容器1个核绑定redis实例，另1个核绑定VF中断；且两个核同属一个SMT 
+3. 部署方式：numa 3的最后一个cluster部署1个观察容器，其他背景容器按顺序部署在numa 0-3 
+4. 客户端：一台客户端单独加压观察容器，保证客户端不是瓶颈；其他四台客户端加压背景容器 
+5. 组网方式：每个容器一个VF
+
+echo 80 > /sys/class/net/enp24s0f0np0/device/sriov_numvfs
+echo 80 > /sys/class/net/enp24s0f1np1/device/sriov_numvfs
+cd /home/name/redis_image/test_inbalance_refactor
+sh create_runc_passthrough.sh 20
+
+# 同时跑4个redis
+sh runc_parallel.sh 4
+# 同时跑80个redis
+sh runc_parallel.sh 80
+
+# 2. vm：6.6.0-95.0.0
+1. BIOS配置：开启SMMU、开启GiCv4.1 
+2. 服务器内核配置：开启中断直通kvm-arm.vgic_v4_enable=1、1G大页：default_hugepagesz=1G hugepagesz=1G 
+3. 虚拟机规格：2台虚机，单虚机2个numa，160U，240G内存，40个VF直通网卡 
+4. host和虚拟机内开启调度策略优化（echo IRQ_AVG > /sys/kernel/debug/sched/features）
+
+
+smt开关：Advanced -> AMD CBS-> CPU Common Options  -> performance -> SMU Common Options
+IOMMU开关：Advanced -> AMD CBS-> NBIO Common Options  -> IOMMU/security -> IOMMU
+
+分配大页：  
+echo 120 > /sys/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages  
+echo 120 > /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages  
+echo 120 > /sys/devices/system/node/node2/hugepages/hugepages-1048576kB/nr_hugepages  
+echo 120 > /sys/devices/system/node/node3/hugepages/hugepages-1048576kB/nr_hugepages
+
+virsh create /home/name/vms/vm1/vm1.xml
+virsh create /home/name/vms/vm2/vm2.xml
+
+ssh root@vm_ip1
+cd /home/name/redis_image/test_inbalance_refactor
+sh create_runc_passthrough.sh 20
+
+ssh root@vm_ip2
+cd /home/name/redis_image/test_inbalance_refactor
+sh create_runc_passthrough.sh 20
+
+# host上
+cd /home/name/redis_image/test_inbalance_refactor
+sh runc_parallel.sh 80
-- 
2.33.0

    

[PATCH OLK-6.6] add test support for container

Qi Xi