April 2021 - Compass-ci - mailweb.openeuler.org

[PATCH v4 lkp-tests 7/7] distro/depends: add install depend for benchmarktool
by Luan Shengde 30 Apr '21

30 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- distro/depends/mysql-8-0-20-dev | 1 + 1 file changed, 1 insertion(+) diff --git a/distro/depends/mysql-8-0-20-dev b/distro/depends/mysql-8-0-20-dev index 23c6ff661..05583bef9 100644 --- a/distro/depends/mysql-8-0-20-dev +++ b/distro/depends/mysql-8-0-20-dev @@ -14,3 +14,4 @@ libmpc libmpc-devel mpfr mpfr-devel +java -- 2.23.0

1 0

[PATCH v4 lkp-tests 6/7] pkg/mysql-8-0-20: add build install benchmarksql
by Luan Shengde 30 Apr '21

30 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- pkg/mysql-8-0-20/PKGBUILD | 51 ++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/pkg/mysql-8-0-20/PKGBUILD b/pkg/mysql-8-0-20/PKGBUILD index 66df6b33f..26247c6f0 100644 --- a/pkg/mysql-8-0-20/PKGBUILD +++ b/pkg/mysql-8-0-20/PKGBUILD @@ -7,13 +7,14 @@ url="http://www.mysql.com" license=('GPLv2') source=("git://$GIT_SERVER/upstream/m/my-sql_with_boostkit_patch/my-sql_with_boostkit_patch.git" "git://$GIT_SERVER/upstream/g/gcc/gcc.git" + "https://mirrors.huaweicloud.com/kunpeng/archive/kunpeng_solution/database/p…" "my.cnf") -md5sums=('SKIP' 'SKIP' '99c4965aa870c475000993f52a0dee66') +md5sums=('SKIP' 'SKIP' 'f33c03fd48ff5993dca6d00b53a46da4' '99c4965aa870c475000993f52a0dee66') build() { - build_gcc build_mysql + build_benchmarksql } build_gcc() @@ -47,9 +48,8 @@ build_gcc() --with-stage1-ldflags='-Wl,-z,relro,-z,now' \ --with-boot-ldflags='-Wl,-z,relro,-z,now' \ --with-multilib-list=lp64 - make -j96 - make install & - wait + make -j96 > /dev/null + make install > /dev/null } } @@ -58,6 +58,11 @@ build_mysql() unset CPPFLAGS unset CFLAGS unset CXXFLAGS + + cd $srcdir/my-sql_with_boostkit_patch && { + grep 3f3d2692fd05a18011d93dcadeef661f386595ac compass-ci.cfg || build_gcc + } + cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver CMAKE=cmake @@ -65,8 +70,36 @@ build_mysql() mkdir -p build cd build && { - $CMAKE .. -DBUILD_CONFIG=mysql_release -DCMAKE_INSTALL_PREFIX=/usr/local/mysql -DMYSQL_DATADIR=/data/mysql/data -DSYSCONFDIR=/etc -DWITH_BOOST=../boost/boost_1_70_0/ - make -j96 + $CMAKE .. -DBUILD_CONFIG=mysql_release -DCMAKE_INSTALL_PREFIX=/usr/local/mysql -DMYSQL_DATADIR=/data/mysql/data -DSYSCONFDIR=/etc -DWITH_BOOST=../boost/boost_1_70_0/ > /dev/null + make -j96 > /dev/null + } +} + +build_benchmarksql() +{ + cd $srcdir && { + cp -r benchmarksql5.0-for-mysql "$HOME/" + chmod -R 777 "$HOME/benchmarksql5.0-for-mysql" + } + + cd "$HOME/benchmarksql5.0-for-mysql/run" && { + chmod 777 *.sh + sed -i \ + -e '/^conn=/ s|mysql://.*:3306|mysql://localhost:3306|' \ + -e '/^user=/cuser=root' \ + -e '/^password=/cpassword=mysql_password' \ + -e '/^warehouses=/cwarehouses=1000' \ + -e '/^loadWorkers=/cloadWorkers=100' \ + -e '/^terminals=/cterminals=200' \ + -e '/^runTxnsPerTerminal=/crunTxnsPerTerminal=0' \ + -e '/^runMins=/crunMins=30' \ + -e '/^limitTxnsPerMin=/climitTxnsPerMin=1000' \ + -e '/^terminalWarehouseFixed=/cterminalWarehouseFixed=true' \ + -e '/^newOrderWeight=/cnewOrderWeight=45' \ + -e '/^paymentWeight=/cpaymentWeight=43' \ + -e '/^orderStatusWeight=/corderStatusWeight=4' \ + -e '/^deliveryWeight=/cdeliveryWeight=4' \ + props.conf } } @@ -75,6 +108,6 @@ package() cp my.cnf /etc/ cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver/build && { - make install - } + make install > /dev/null + } } -- 2.23.0

1 0

[PATCH v4 lkp-tests 5/7] tests: add init database for benchmarksql
by Luan Shengde 30 Apr '21

30 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/mysql-8-0-20 | 66 +++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/tests/mysql-8-0-20 b/tests/mysql-8-0-20 index 990a053d7..4a49e0233 100755 --- a/tests/mysql-8-0-20 +++ b/tests/mysql-8-0-20 @@ -4,69 +4,75 @@ # mysql_host # mysql_port # mysql_db +# benchmark_db # mysql_password + . $LKP_SRC/lib/common.sh : "${mysql_user:=root}" : "${mysql_host:=127.0.0.1}" : "${mysql_port:=3306}" : "${mysql_db:=sysbench_test}" +: "${benchmark_db:=tpcc}" : "${mysql_password:=mysql_password}" create_basedir() { - mkdir -p /data - mkfs.xfs -f /dev/"$data_dev" - mount /dev/"$data_dev" /data + mkdir -p /data + mkfs.xfs -f /dev/"$data_dev" + mount /dev/"$data_dev" /data - mkdir -p /data/mysql/{data,share,tmp,run,log} + mkdir -p /data/mysql/{data,share,tmp,run,log} - useradd mysql - touch /data/mysql/log/mysql.log - chown -R mysql:mysql /data - chown -R mysql:mysql /data/mysql/log/mysql.log + useradd mysql + touch /data/mysql/log/mysql.log + chown -R mysql:mysql /data + chown -R mysql:mysql /data/mysql/log/mysql.log } configure_mysql() { - if [ ! -f $mysqld ] || [ ! -f $mysql ] || [ ! -f $mysql_server ]; then - echo "mysql is not installed!" && exit 1 - fi + if [ ! -f $mysqld ] || [ ! -f $mysql ] || [ ! -f $mysql_server ]; then + echo "mysql is not installed!" && exit 1 + fi - "$mysqld" --user=mysql --datadir=/data/mysql/data --basedir=/data/mysql --initialize --port="$mysql_port" - sleep 10 + "$mysqld" --user=mysql --datadir=/data/mysql/data --basedir=/data/mysql --initialize --port="$mysql_port" + sleep 10 - "$mysql_server" start - sleep 10 + "$mysql_server" start + sleep 10 - "$mysql" -u "$mysql_user" -e "flush privileges; use mysql; alter user 'root'@'localhost' identified by '$mysql_password'; flush privileges;" - "$mysql_server" restart - sleep 10 + "$mysql" -u "$mysql_user" -e "flush privileges; use mysql; alter user 'root'@'localhost' identified by '$mysql_password'; flush privileges;" + sed -i '/^skip-grant-tables/d' /etc/my.cnf + sed -i '/^skip-name-resolve/d' /etc/my.cnf + "$mysql_server" restart + sleep 10 } create_database() { - "$mysql" -u "$mysql_user" -p"$mysql_password" -e "use mysql; update user set host ='%' where user='root'; flush privileges;" - "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $mysql_db;" - "$mysql_server" stop - sleep 10 + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "use mysql; update user set host ='%' where user='root'; flush privileges;" + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $mysql_db;" + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $benchmark_db;" + "$mysql_server" stop + sleep 10 } start_mysql() { - "$mysql_server" start & + "$mysql_server" start & wait } main() { - local mysql="/usr/local/mysql/bin/mysql" - local mysqld="/usr/local/mysql/bin/mysqld" - local mysql_server="/usr/local/mysql/support-files/mysql.server" + local mysql="/usr/local/mysql/bin/mysql" + local mysqld="/usr/local/mysql/bin/mysqld" + local mysql_server="/usr/local/mysql/support-files/mysql.server" - create_basedir - configure_mysql - create_database - start_mysql + create_basedir + configure_mysql + create_database + start_mysql } main -- 2.23.0

1 0

[PATCH v4 lkp-tests 4/7] tests: add script for benchmarksql test tool
by Luan Shengde 30 Apr '21

30 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/benchmark-sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 tests/benchmark-sql diff --git a/tests/benchmark-sql b/tests/benchmark-sql new file mode 100755 index 000000000..6acac2847 --- /dev/null +++ b/tests/benchmark-sql @@ -0,0 +1,15 @@ +#!/bin/sh +# benchmark_pkg_name + +: "${benchmark_pkg_name:=benchmarksql5.0-for-mysql}" + +run_benchmark_mysql() +{ + cd "$HOME/$benchmark_pkg_name/run/" && { + ./runDatabaseBuild.sh props.conf + ./runBenchmark.sh props.conf + ./runDatabaseDestroy.sh props.conf + } +} + +run_benchmark_mysql -- 2.23.0

1 0

[PATCH v4 lkp-tests 3/7] stats: add extract scritp
by Luan Shengde 30 Apr '21

30 Apr '21

add stats script for benchmarksql Signed-off-by: Luan Shengde <shdluan(a)163.com> --- stats/benchmark-sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 stats/benchmark-sql diff --git a/stats/benchmark-sql b/stats/benchmark-sql new file mode 100755 index 000000000..d4794b4f4 --- /dev/null +++ b/stats/benchmark-sql @@ -0,0 +1,16 @@ +#!/usr/bin/env ruby + +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Measured tpmC (NewOrders) = 450.31 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Measured tpmTOTAL = 999.71 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Session Start = 2021-04-29 10:58:15 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Session End = 2021-04-29 11:28:16 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Transaction Count = 30003 + +while (line = STDIN.gets) + case line.chomp! + when /Measured tpmC $NewOrders$ = (\d*\.\d*)/ + puts "Measured_tpmC$NewOrders$: #{$1}" + when /Measured tpmTOTAL = (\d*\.\d*)/ + puts "Measured_tpmTOTAL: #{$1}" + end +end -- 2.23.0

1 0

[PATCH v4 lkp-tests 2/7] jobs: add job yaml
by Luan Shengde 30 Apr '21

30 Apr '21

add job yaml for benchmarksql Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/benchmark-sql.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 jobs/benchmark-sql.yaml diff --git a/jobs/benchmark-sql.yaml b/jobs/benchmark-sql.yaml new file mode 100644 index 000000000..4563b666c --- /dev/null +++ b/jobs/benchmark-sql.yaml @@ -0,0 +1,16 @@ +suite: benchmark-sql + +mysql_user: +mysql_host: localhost +mysql_port: +mysql_db: +benchmark_db: tpcc +mysql_password: +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +benchmark-sql: -- 2.23.0

1 0

[PATCH v4 lkp-tests 1/7] jobs: add job yaml
by Luan Shengde 30 Apr '21

30 Apr '21

add job yaml for sysbench-mysql Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/sysbench-mysql.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 jobs/sysbench-mysql.yaml diff --git a/jobs/sysbench-mysql.yaml b/jobs/sysbench-mysql.yaml new file mode 100644 index 000000000..af017df4b --- /dev/null +++ b/jobs/sysbench-mysql.yaml @@ -0,0 +1,28 @@ +suite: sysbench-mysql + +mysql_user: +mysql_host: localhost +mysql_port: +mysql_db: +mysql_password: +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +sysbench-mysql: + oltp_test_mode: complex + max_requests: 0 + mysql_table_engine: innodb + oltp_tables_count: 1000 + oltp_table_size: 100000 + nr_threads: 256 + runtime: 60 + rand_type: special + rand_spec_pct: 100 + oltp_read_only: false + report_interval: 1 + events: 20000 + mysql_socket: /data/mysql/run/mysql.sock -- 2.23.0

1 0

[PATCH v2 lkp-tests] resize virtual rootfs from 50% to 90% memory, for docker no sapce error
by Zhang Dewan 30 Apr '21

30 Apr '21

Signed-off-by: Zhang Dewan <513619464(a)qq.com> --- lib/bootstrap.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 417c8662a..82a1c17d7 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -57,6 +57,35 @@ mount_tmpfs() mount -t tmpfs -o mode=1777 tmp /tmp } +resize_virtual_rootfs() +{ + # tmpfs and rootfs default use 50% memory: + # + # root@taishan200-2280-2s48p-256g--a13 ~# mount | head -n 1 + # rootfs on / type rootfs (rw,size=133395136k,nr_inodes=2084299) + # + # root@taishan200-2280-2s48p-256g--a13 ~# df -h / + # Filesystem Size Used Avail Use% Mounted on + # rootfs 128G 114G 14G 90% / + # + # this maybe cause the docker no space error. + + if mount | head -n 1 | grep "type rootfs" > /dev/null + then + echo "Set the rootfs to 90% memory" + mount -o remount,rw,size=90% rootfs / + fi + # Test Result: + # root@taishan200-2280-2s64p-256g--a125 ~# free -g + # total used free shared buff/cache available + # Mem: 255Gi 8.5Gi 243Gi 2.7Gi 2.9Gi 224Gi + # Swap: 0B 0B 0B + # root@taishan200-2280-2s64p-256g--a125 ~# df -h / + # Filesystem Size Used Avail Use% Mounted on + # rootfs 230G 2.7G 228G 2% / + +} + get_net_devices() { local net_devices @@ -737,6 +766,7 @@ boot_init() mount_kernel_fs mount_tmpfs + resize_virtual_rootfs redirect_stdout_stderr echo 'Kernel tests: Boot OK!' -- 2.23.0

1 0

[PATCH compass-ci] sparrow/README.md: specify the correct filename in source cmd
by Xu Xijian 30 Apr '21

30 Apr '21

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- sparrow/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparrow/README.md b/sparrow/README.md index 532b7f9..a519497 100644 --- a/sparrow/README.md +++ b/sparrow/README.md @@ -61,7 +61,7 @@ 本文以/c/lkp-tests/jobs/目录下已有的通用测试用例host-info.yaml为例 - 使环境变量生效 ```bash - source /etc/profile.d/compass-ci + source /etc/profile.d/compass.sh ``` - 使用[submit命令](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manu… -- 2.23.0

1 0

[PATCH v3 lkp-tests 7/7] distro/depends: add depend for benchmarktool
by Luan Shengde 30 Apr '21

30 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- distro/depends/mysql-8-0-20-dev | 1 + 1 file changed, 1 insertion(+) diff --git a/distro/depends/mysql-8-0-20-dev b/distro/depends/mysql-8-0-20-dev index 23c6ff661..05583bef9 100644 --- a/distro/depends/mysql-8-0-20-dev +++ b/distro/depends/mysql-8-0-20-dev @@ -14,3 +14,4 @@ libmpc libmpc-devel mpfr mpfr-devel +java -- 2.23.0

1 0

[PATCH v3 lkp-tests 6/7] pkg/mysql-8-0-20: add build install benchmarksql
by Luan Shengde 30 Apr '21

30 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- pkg/mysql-8-0-20/PKGBUILD | 51 ++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/pkg/mysql-8-0-20/PKGBUILD b/pkg/mysql-8-0-20/PKGBUILD index 66df6b33f..26247c6f0 100644 --- a/pkg/mysql-8-0-20/PKGBUILD +++ b/pkg/mysql-8-0-20/PKGBUILD @@ -7,13 +7,14 @@ url="http://www.mysql.com" license=('GPLv2') source=("git://$GIT_SERVER/upstream/m/my-sql_with_boostkit_patch/my-sql_with_boostkit_patch.git" "git://$GIT_SERVER/upstream/g/gcc/gcc.git" + "https://mirrors.huaweicloud.com/kunpeng/archive/kunpeng_solution/database/p…" "my.cnf") -md5sums=('SKIP' 'SKIP' '99c4965aa870c475000993f52a0dee66') +md5sums=('SKIP' 'SKIP' 'f33c03fd48ff5993dca6d00b53a46da4' '99c4965aa870c475000993f52a0dee66') build() { - build_gcc build_mysql + build_benchmarksql } build_gcc() @@ -47,9 +48,8 @@ build_gcc() --with-stage1-ldflags='-Wl,-z,relro,-z,now' \ --with-boot-ldflags='-Wl,-z,relro,-z,now' \ --with-multilib-list=lp64 - make -j96 - make install & - wait + make -j96 > /dev/null + make install > /dev/null } } @@ -58,6 +58,11 @@ build_mysql() unset CPPFLAGS unset CFLAGS unset CXXFLAGS + + cd $srcdir/my-sql_with_boostkit_patch && { + grep 3f3d2692fd05a18011d93dcadeef661f386595ac compass-ci.cfg || build_gcc + } + cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver CMAKE=cmake @@ -65,8 +70,36 @@ build_mysql() mkdir -p build cd build && { - $CMAKE .. -DBUILD_CONFIG=mysql_release -DCMAKE_INSTALL_PREFIX=/usr/local/mysql -DMYSQL_DATADIR=/data/mysql/data -DSYSCONFDIR=/etc -DWITH_BOOST=../boost/boost_1_70_0/ - make -j96 + $CMAKE .. -DBUILD_CONFIG=mysql_release -DCMAKE_INSTALL_PREFIX=/usr/local/mysql -DMYSQL_DATADIR=/data/mysql/data -DSYSCONFDIR=/etc -DWITH_BOOST=../boost/boost_1_70_0/ > /dev/null + make -j96 > /dev/null + } +} + +build_benchmarksql() +{ + cd $srcdir && { + cp -r benchmarksql5.0-for-mysql "$HOME/" + chmod -R 777 "$HOME/benchmarksql5.0-for-mysql" + } + + cd "$HOME/benchmarksql5.0-for-mysql/run" && { + chmod 777 *.sh + sed -i \ + -e '/^conn=/ s|mysql://.*:3306|mysql://localhost:3306|' \ + -e '/^user=/cuser=root' \ + -e '/^password=/cpassword=mysql_password' \ + -e '/^warehouses=/cwarehouses=1000' \ + -e '/^loadWorkers=/cloadWorkers=100' \ + -e '/^terminals=/cterminals=200' \ + -e '/^runTxnsPerTerminal=/crunTxnsPerTerminal=0' \ + -e '/^runMins=/crunMins=30' \ + -e '/^limitTxnsPerMin=/climitTxnsPerMin=1000' \ + -e '/^terminalWarehouseFixed=/cterminalWarehouseFixed=true' \ + -e '/^newOrderWeight=/cnewOrderWeight=45' \ + -e '/^paymentWeight=/cpaymentWeight=43' \ + -e '/^orderStatusWeight=/corderStatusWeight=4' \ + -e '/^deliveryWeight=/cdeliveryWeight=4' \ + props.conf } } @@ -75,6 +108,6 @@ package() cp my.cnf /etc/ cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver/build && { - make install - } + make install > /dev/null + } } -- 2.23.0

1 0

[PATCH v3 lkp-tests 5/7] tests: add init database for benchmarksql
by Luan Shengde 30 Apr '21

30 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/mysql-8-0-20 | 66 +++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/tests/mysql-8-0-20 b/tests/mysql-8-0-20 index 990a053d7..4a49e0233 100755 --- a/tests/mysql-8-0-20 +++ b/tests/mysql-8-0-20 @@ -4,69 +4,75 @@ # mysql_host # mysql_port # mysql_db +# benchmark_db # mysql_password + . $LKP_SRC/lib/common.sh : "${mysql_user:=root}" : "${mysql_host:=127.0.0.1}" : "${mysql_port:=3306}" : "${mysql_db:=sysbench_test}" +: "${benchmark_db:=tpcc}" : "${mysql_password:=mysql_password}" create_basedir() { - mkdir -p /data - mkfs.xfs -f /dev/"$data_dev" - mount /dev/"$data_dev" /data + mkdir -p /data + mkfs.xfs -f /dev/"$data_dev" + mount /dev/"$data_dev" /data - mkdir -p /data/mysql/{data,share,tmp,run,log} + mkdir -p /data/mysql/{data,share,tmp,run,log} - useradd mysql - touch /data/mysql/log/mysql.log - chown -R mysql:mysql /data - chown -R mysql:mysql /data/mysql/log/mysql.log + useradd mysql + touch /data/mysql/log/mysql.log + chown -R mysql:mysql /data + chown -R mysql:mysql /data/mysql/log/mysql.log } configure_mysql() { - if [ ! -f $mysqld ] || [ ! -f $mysql ] || [ ! -f $mysql_server ]; then - echo "mysql is not installed!" && exit 1 - fi + if [ ! -f $mysqld ] || [ ! -f $mysql ] || [ ! -f $mysql_server ]; then + echo "mysql is not installed!" && exit 1 + fi - "$mysqld" --user=mysql --datadir=/data/mysql/data --basedir=/data/mysql --initialize --port="$mysql_port" - sleep 10 + "$mysqld" --user=mysql --datadir=/data/mysql/data --basedir=/data/mysql --initialize --port="$mysql_port" + sleep 10 - "$mysql_server" start - sleep 10 + "$mysql_server" start + sleep 10 - "$mysql" -u "$mysql_user" -e "flush privileges; use mysql; alter user 'root'@'localhost' identified by '$mysql_password'; flush privileges;" - "$mysql_server" restart - sleep 10 + "$mysql" -u "$mysql_user" -e "flush privileges; use mysql; alter user 'root'@'localhost' identified by '$mysql_password'; flush privileges;" + sed -i '/^skip-grant-tables/d' /etc/my.cnf + sed -i '/^skip-name-resolve/d' /etc/my.cnf + "$mysql_server" restart + sleep 10 } create_database() { - "$mysql" -u "$mysql_user" -p"$mysql_password" -e "use mysql; update user set host ='%' where user='root'; flush privileges;" - "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $mysql_db;" - "$mysql_server" stop - sleep 10 + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "use mysql; update user set host ='%' where user='root'; flush privileges;" + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $mysql_db;" + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $benchmark_db;" + "$mysql_server" stop + sleep 10 } start_mysql() { - "$mysql_server" start & + "$mysql_server" start & wait } main() { - local mysql="/usr/local/mysql/bin/mysql" - local mysqld="/usr/local/mysql/bin/mysqld" - local mysql_server="/usr/local/mysql/support-files/mysql.server" + local mysql="/usr/local/mysql/bin/mysql" + local mysqld="/usr/local/mysql/bin/mysqld" + local mysql_server="/usr/local/mysql/support-files/mysql.server" - create_basedir - configure_mysql - create_database - start_mysql + create_basedir + configure_mysql + create_database + start_mysql } main -- 2.23.0

1 0

[PATCH v3 lkp-tests 4/7] tests: add script for benchmarksql test tool
by Luan Shengde 30 Apr '21

30 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/benchmark-sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 tests/benchmark-sql diff --git a/tests/benchmark-sql b/tests/benchmark-sql new file mode 100755 index 000000000..2ec6a7317 --- /dev/null +++ b/tests/benchmark-sql @@ -0,0 +1,17 @@ +#!/bin/sh +# benchmark_pkg_name + +: "${benchmark_pkg_name:=benchmarksql5.0-for-mysql}" + +run_benchmark_mysql() +{ + scripts=(runDatabaseBuild.sh runBenchmark.sh runDatabaseDestroy.sh) + + cd "$HOME/$benchmark_pkg_name/run/" && { + ./runDatabaseBuild.sh props.conf + ./runBenchmark.sh props.conf + ./runDatabaseDestroy.sh props.conf + } +} + +run_benchmark_mysql -- 2.23.0

1 0

[PATCH v3 lkp-tests 3/7] stats: add extract scritp
by Luan Shengde 30 Apr '21

30 Apr '21

add stats script for benchmarksql Signed-off-by: Luan Shengde <shdluan(a)163.com> --- stats/benchmark-sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 stats/benchmark-sql diff --git a/stats/benchmark-sql b/stats/benchmark-sql new file mode 100755 index 000000000..d4794b4f4 --- /dev/null +++ b/stats/benchmark-sql @@ -0,0 +1,16 @@ +#!/usr/bin/env ruby + +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Measured tpmC (NewOrders) = 450.31 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Measured tpmTOTAL = 999.71 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Session Start = 2021-04-29 10:58:15 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Session End = 2021-04-29 11:28:16 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Transaction Count = 30003 + +while (line = STDIN.gets) + case line.chomp! + when /Measured tpmC $NewOrders$ = (\d*\.\d*)/ + puts "Measured_tpmC$NewOrders$: #{$1}" + when /Measured tpmTOTAL = (\d*\.\d*)/ + puts "Measured_tpmTOTAL: #{$1}" + end +end -- 2.23.0

1 0

[PATCH v3 lkp-tests 2/7] jobs: add job yaml
by Luan Shengde 30 Apr '21

30 Apr '21

add job yaml for benchmarksql Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/benchmark-sql.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 jobs/benchmark-sql.yaml diff --git a/jobs/benchmark-sql.yaml b/jobs/benchmark-sql.yaml new file mode 100644 index 000000000..152253c4e --- /dev/null +++ b/jobs/benchmark-sql.yaml @@ -0,0 +1,20 @@ +suite: benchmark-sql + +os: openeuler +os_version: 20.03 +os_mount: initramfs + +mysql_user: +mysql_host: localhost +mysql_port: +mysql_db: +benchmark_db: tpcc +mysql_password: +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +benchmark-sql: -- 2.23.0

1 0

[PATCH v3 lkp-tests 1/7] jobs: add job yaml
by Luan Shengde 30 Apr '21

30 Apr '21

add job yaml for sysbench-mysql Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/sysbench-mysql.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 jobs/sysbench-mysql.yaml diff --git a/jobs/sysbench-mysql.yaml b/jobs/sysbench-mysql.yaml new file mode 100644 index 000000000..6c6606ea2 --- /dev/null +++ b/jobs/sysbench-mysql.yaml @@ -0,0 +1,33 @@ +suite: sysbench-mysql + +os: openeuler +os_version: 20.03 +os_mount: initramfs + +mysql_user: +mysql_host: localhost +mysql_port: +mysql_db: +benchmark_db: tpcc +mysql_password: +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +sysbench-mysql: + oltp_test_mode: complex + max_requests: 0 + mysql_table_engine: innodb + oltp_tables_count: 1000 + oltp_table_size: 100000 + nr_threads: 256 + runtime: 60 + rand_type: special + rand_spec_pct: 100 + oltp_read_only: false + report_interval: 1 + events: 20000 + mysql_socket: /data/mysql/run/mysql.sock -- 2.23.0

1 0

[PATCH v2 compass-ci] lib/mail_job_result: optimize mail content
by Lu Weitao 30 Apr '21

30 Apr '21

optimize mail content limit max lines(100) of compare result Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/mail_job_result.rb | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/mail_job_result.rb b/lib/mail_job_result.rb index 744b7d5c..7937de66 100644 --- a/lib/mail_job_result.rb +++ b/lib/mail_job_result.rb @@ -24,10 +24,10 @@ class MailJobResult def compose_mail set_submitter_info - context = get_compare_result(@job) + context = keep_100_lines(get_compare_result(@job)) return nil unless context - subject = "[Compass-CI] job: #{@job_id} compare result" + subject = "[Compass-CI] #{@job['commit_title'] || @job_id} comparsion" signature = "Regards\nCompass-CI\nhttps://gitee.com/openeuler/compass-ci" data = <<~BODY @@ -38,10 +38,17 @@ class MailJobResult Hi, Thanks for your participation in Kunpeng and software ecosystem! - Your Job: #{@job_id} had finished. - Bellow are comparsion result of the base_commit and current_commit: - \tbase_commit: #{@job['base_commit']} - \tcurrent_commit: #{@job['upstream_commit']} + Your job id: #{@job_id} + Bellow are comparsion result of the base commit and your commit: + \tbase commit: #{@job['base_commit']} + \tcommit_link: #{@job['upstream_url']}/commit/#{@job['base_commit']} + + \tyour commit: #{@job['upstream_commit']} + \tbranch: #{@job['upstream_branch']} + \tcommit_link: #{@job['upstream_url']}/commit/#{@job['upstream_commit']} + + compare command: + \tcompare upstream_commit=#{@job['base_commit']} upstream_commit=#{@job['upstream_commit']} --min_samples #{@job['nr_run']} #{context.to_s} \n\n#{signature}" BODY @@ -86,3 +93,7 @@ def get_compare_result(job) m_titles = [base_commit.slice(0, 16), commit_id.slice(0, 24)] compare_matrixes(matrices_list, suite_list, nil, m_titles, options: options) end + +def keep_100_lines(context) + context.split("\n")[0,99].join("\n") +end -- 2.23.0

1 0

[PATCH compass-ci] lib/mail_job_result: optimize mail content
by Lu Weitao 30 Apr '21

30 Apr '21

optimize mail content limit max lines(100) of compare result Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/mail_job_result.rb | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/mail_job_result.rb b/lib/mail_job_result.rb index 744b7d5c..0bf52b6e 100644 --- a/lib/mail_job_result.rb +++ b/lib/mail_job_result.rb @@ -24,10 +24,10 @@ class MailJobResult def compose_mail set_submitter_info - context = get_compare_result(@job) + context = keep_100_lines(get_compare_result(@job)) return nil unless context - subject = "[Compass-CI] job: #{@job_id} compare result" + subject = "[Compass-CI] commit: #{@job['upstream_commit']} comparsion result" signature = "Regards\nCompass-CI\nhttps://gitee.com/openeuler/compass-ci" data = <<~BODY @@ -38,10 +38,15 @@ class MailJobResult Hi, Thanks for your participation in Kunpeng and software ecosystem! - Your Job: #{@job_id} had finished. - Bellow are comparsion result of the base_commit and current_commit: - \tbase_commit: #{@job['base_commit']} - \tcurrent_commit: #{@job['upstream_commit']} + Bellow are comparsion result of the base commit and your commit: + \tbase commit: #{@job['base_commit']} + \tcommit_link: #{@job['upstream_url']}/commit/#{@job['base_commit']} + + \tyour commit: #{@job['upstream_commit']} + \tbranch: #{@job['upstream_branch']} + \tcommit_link: #{@job['upstream_url']}/commit/#{@job['upstream_commit']} + + compare upstream_commit=#{@job['base_commit']} upstream_commit=#{@job['upstream_commit']} --min_samples #{@job['nr_run']} #{context.to_s} \n\n#{signature}" BODY @@ -86,3 +91,15 @@ def get_compare_result(job) m_titles = [base_commit.slice(0, 16), commit_id.slice(0, 24)] compare_matrixes(matrices_list, suite_list, nil, m_titles, options: options) end + +def keep_100_lines(context) + num = 0 + new_context = '' + context.each_line do |line| + new_context += line + num += 1 + break if num > 99 + end + + new_context +end -- 2.23.0

2 3

Re: [PATCH lkp-tests 1/6] add job yaml for sysbenc
by Li Ping 30 Apr '21

30 Apr '21

sysbench-mysql Reply-To: In-Reply-To: <20210429133100.902627-1-shdluan(a)163.com> On Thu, Apr 29, 2021 at 09:30:55PM +0800, Luan Shengde wrote: >Signed-off-by: Luan Shengde <shdluan(a)163.com> >--- > jobs/sysbench-mysql.yaml | 33 +++++++++++++++++++++++++++++++++ > 1 file changed, 33 insertions(+) > create mode 100644 jobs/sysbench-mysql.yaml > >diff --git a/jobs/sysbench-mysql.yaml b/jobs/sysbench-mysql.yaml >new file mode 100644 >index 000000000..6c6606ea2 >--- /dev/null >+++ b/jobs/sysbench-mysql.yaml >@@ -0,0 +1,33 @@ >+suite: sysbench-mysql >+ >+os: openeuler >+os_version: 20.03 >+os_mount: initramfs >+ >+mysql_user: >+mysql_host: localhost >+mysql_port: >+mysql_db: >+benchmark_db: tpcc >+mysql_password: >+data_dev: sdb >+ >+cci-makepkg: >+ benchmark: mysql-8-0-20 >+ >+mysql-8-0-20: >+ >+sysbench-mysql: >+ oltp_test_mode: complex >+ max_requests: 0 >+ mysql_table_engine: innodb >+ oltp_tables_count: 1000 >+ oltp_table_size: 100000 >+ nr_threads: 256 >+ runtime: 60 >+ rand_type: special >+ rand_spec_pct: 100 >+ oltp_read_only: false >+ report_interval: 1 >+ events: 20000 >+ mysql_socket: /data/mysql/run/mysql.sock >-- >2.23.0 >

2 1

Re: [PATCH lkp-tests 2/6] add job yaml for benmarksql
by Li Ping 30 Apr '21

30 Apr '21

benchmark-sql Reply-To: In-Reply-To: <20210429133100.902627-2-shdluan(a)163.com> On Thu, Apr 29, 2021 at 09:30:56PM +0800, Luan Shengde wrote: >Signed-off-by: Luan Shengde <shdluan(a)163.com> >--- > jobs/benchmark-sql.yaml | 20 ++++++++++++++++++++ > 1 file changed, 20 insertions(+) > create mode 100644 jobs/benchmark-sql.yaml > >diff --git a/jobs/benchmark-sql.yaml b/jobs/benchmark-sql.yaml >new file mode 100644 >index 000000000..152253c4e >--- /dev/null >+++ b/jobs/benchmark-sql.yaml >@@ -0,0 +1,20 @@ >+suite: benchmark-sql >+ >+os: openeuler >+os_version: 20.03 >+os_mount: initramfs >+ >+mysql_user: >+mysql_host: localhost >+mysql_port: >+mysql_db: >+benchmark_db: tpcc >+mysql_password: >+data_dev: sdb >+ >+cci-makepkg: >+ benchmark: mysql-8-0-20 >+ >+mysql-8-0-20: >+ >+benchmark-sql: >-- >2.23.0 >

2 1

[PATCH lkp-tests] install-iso-sync-rootfs: delete cgz content for firstboot reboot
by Yu Chuan 30 Apr '21

30 Apr '21

[Why] In install-iso-firstboot step, we put the reboot-after-systemd-logind.cgz to rootfs, so we need delete all file in that cgz before we archive the result rootfs. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- distro/depends/install-iso-sync-rootfs | 1 + tests/install-iso-sync-rootfs | 28 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/distro/depends/install-iso-sync-rootfs b/distro/depends/install-iso-sync-rootfs index 2abb6b833291..41c472eda77e 100644 --- a/distro/depends/install-iso-sync-rootfs +++ b/distro/depends/install-iso-sync-rootfs @@ -1,2 +1,3 @@ nfs-common lvm2 +wget diff --git a/tests/install-iso-sync-rootfs b/tests/install-iso-sync-rootfs index b74c7bed2044..c2b64a2b44d8 100755 --- a/tests/install-iso-sync-rootfs +++ b/tests/install-iso-sync-rootfs @@ -53,6 +53,31 @@ prepare_exclude_file() done } +delete_firstboot_cgz() +{ + local cgz_url="http://${INITRD_HTTP_HOST}:${INITRD_HTTP_PORT}/initrd/custom_bootstrap/reboot-after-systemd-logind.cgz" + + local tmp_dir="/tmp/$(basename $cgz_url)_${ROOTFS_SUFFIX}" + rm -rf $tmp_dir && mkdir -p $tmp_dir && cd $tmp_dir + + wget $cgz_url || { + log_info "wget cgz failed, skip delete it: $cgz_url" + return 0 + } + + gzip -dc reboot-after-systemd-logind.cgz | cpio -id + + [ -d "custom_bootstrap" ] || { + log_info "cannot find custom_bootstrap dir after unzip" + return 0 + } + + for i in $(find custom_bootstrap/* -not -type d) + do + rm -f ${lv_rootfs_mount_point}/${i#custom_bootstrap/} + done +} + sync_rootfs_to_nfs_server() { # mount nfs rootfs @@ -70,6 +95,9 @@ sync_rootfs_to_nfs_server() mkdir -p ${lv_rootfs_mount_point} mount /dev/mapper/${VG_NAME}-${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX} ${lv_rootfs_mount_point} || die "mount lv rootfs failed" + # delete cgz for firstboot reboot + delete_firstboot_cgz + # sync rootfs local exclude_file=/tmp/exclude_file_${ROOTFS_SUFFIX} -- 2.23.0

1 0

[PATCH v4 lkp-tests] add email feedback for rpmbuild-pkg
by Li Ping 30 Apr '21

30 Apr '21

tests/rpmbuild-pkg: add a new input parameter whether_email [usage]: rpmbuild-pkg: whether_email: 1 when job failed, send an rpmbuild failed email. when job run success, send an rpmbuild success email. Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/email.sh | 35 +++++++++++++++++++++++++++++++++++ tests/rpmbuild-pkg | 24 ++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/lib/email.sh b/lib/email.sh index b5e56df79..a6e0bc3ba 100755 --- a/lib/email.sh +++ b/lib/email.sh @@ -96,6 +96,41 @@ Compass-CI " } +rpmbuild_report() +{ + email_content="To: $author_email +Subject: [NOTIFY Compass-CI] rpmbuild report + +Dear $author_name: + + $rpmbuild_result + +Regards +Compass-CI +" +} + +rpmbuild_success_content() +{ + rpmbuild_result="Your RPM Package is successfully built. + +You can click the follow link to obtain your RPM Package: +http://api.compass-ci.openeuler.org:20012/rpm/pub/${os}-${os_version}/${compat_os}/${os_arch}/Packages + +And you can click the following link to view RPM Package build details: +http://api.compass-ci.openeuler.org:${SRV_HTTP_RESULT_PORT:-20007}$result_root" + + rpmbuild_report +} + +rpmbuild_failed_content() +{ + rpmbuild_result="Sorry to inform you that RPM Package built failed, you can click the following link to view details. +http://api.compass-ci.openeuler.org:${SRV_HTTP_RESULT_PORT:-20007}$result_root" + + rpmbuild_report +} + send_email() { local subject=$1 diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index 3dd75b896..5ad5caf56 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -5,6 +5,7 @@ # - repo_addr # - custom_repo_name # - custom_repo_addr +# - whether_email # Git repo jobs only use the first two parameters, SRPMs jobs use all # user can add custom_repo_name and custom_repo_addr to setup # local repo, value can be an array @@ -12,6 +13,7 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh . $LKP_SRC/lib/rpm.sh +. $LKP_SRC/lib/email.sh : "${compat_os:=budding-openeuler}" @@ -19,6 +21,13 @@ dest_dir="/rpm/upload/${os}-${os_version}/${compat_os}" +die() +{ + rpmbuild_email "rpmbuild_failed" + echo "$@" 1>&2 + exit 99 +} + from_git() { package_name=${upstream_repo##*/} @@ -29,6 +38,20 @@ from_git() download_upstream_repo } +git_pretty_format() +{ + curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' -d '{"git_repo": "'${upstream_dir}/${upstream_repo}.git'", "git_command": ["git-log","-n1","--pretty=format:'$1'", "'${upstream_commit}'"]}' +} + +rpmbuild_email() +{ + [ -n "$whether_email" ] && { + author_email=$(git_pretty_format "%ae") + author_name=$(git_pretty_format "%an") + send_email $1 + } +} + from_srpm() { [ -n "$repo_name" ] || die "repo_name is empty" @@ -143,3 +166,4 @@ add_repo build_rpm upload_rpm_pkg update_repo_mq +rpmbuild_email "rpmbuild_success" -- 2.23.0

1 0

[PATCH lkp-tests] alter the tmpfs use 90% memory
by Zhang Dewan 30 Apr '21

30 Apr '21

Signed-off-by: Zhang Dewan <513619464(a)qq.com> --- lib/bootstrap.sh | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 417c8662a..124b872bd 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -57,6 +57,27 @@ mount_tmpfs() mount -t tmpfs -o mode=1777 tmp /tmp } +resize_virtual_rootfs() +{ + # tmpfs and rootfs default use 50% memory: + # + # root@taishan200-2280-2s48p-256g--a13 ~# mount | head -n 1 + # rootfs on / type rootfs (rw,size=133395136k,nr_inodes=2084299) + # + # root@taishan200-2280-2s48p-256g--a13 ~# df -h / + # Filesystem Size Used Avail Use% Mounted on + # rootfs 128G 114G 14G 90% / + # + # this maybe cause the docker no space error. + + if mount | head -n 1 | grep "type rootfs" > /dev/null + then + echo "Set the rootfs to 90% memory" + mount -o remount,rw,size=90% rootfs / + fi + +} + get_net_devices() { local net_devices @@ -88,10 +109,6 @@ net_devices_link() test_ping() { - has_cmd ping || { - echo "command not found: ping, skip ping test." - return 0 - } ping -c 1 -w 1 $LKP_SERVER > /dev/null } @@ -411,7 +428,6 @@ install_rpms() [ -d /opt/rpms ] || return echo "install $(ls /opt/rpms/*.rpm)" rpm -ivh --force --ignoresize /opt/rpms/*.rpm - rm -f /opt/rpms/*.rpm } try_get_and_set_distro() @@ -737,6 +753,7 @@ boot_init() mount_kernel_fs mount_tmpfs + resize_virtual_rootfs redirect_stdout_stderr echo 'Kernel tests: Boot OK!' -- 2.23.0

2 2

[PATCH v2 lkp-tests] tests/self-test: send result email to commiter and author
by Wu Zhende 30 Apr '21

30 Apr '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/email.sh | 8 ++++++-- tests/self-test | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/email.sh b/lib/email.sh index b5e56df79..99863aa1b 100755 --- a/lib/email.sh +++ b/lib/email.sh @@ -82,10 +82,14 @@ Compass-CI selftest_content() { - email_content="To: $my_email + email_content="To: $recipient_email Subject: [NOTIFY Compass-CI] Self-test report -Dear $my_name: +Dear $author_name and $commiter: + + commit_id: $commit_id + + commit_subject: $commit_subject Your self-test groud_id=selftest job result is: diff --git a/tests/self-test b/tests/self-test index fb3e87b17..11786ac3f 100755 --- a/tests/self-test +++ b/tests/self-test @@ -28,5 +28,23 @@ submit_job() log_info "selftest job result: $report_content" } +get_commit_info() +{ + cd /c/compass-ci || exit + + commit_id=$(git log -1 --format="%H") + commit_subject=$(git log -1 --format="%s") + + author_name=$(git log -1 --format="%an") + author_email=$(git log -1 --format="%ae") + + commiter=$(git log -1 --format="%cn") + commiter_email=$(git log -1 --format="%ce") + + recipient_email="$author_email;$commiter_email" + +} + +get_commit_info submit_job send_email selftest -- 2.23.0

1 0

[PATCH lkp-tests] tsets/self-test: send result email to commiter and author
by Wu Zhende 30 Apr '21

30 Apr '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/email.sh | 8 ++++++-- tests/self-test | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/lib/email.sh b/lib/email.sh index b5e56df79..99863aa1b 100755 --- a/lib/email.sh +++ b/lib/email.sh @@ -82,10 +82,14 @@ Compass-CI selftest_content() { - email_content="To: $my_email + email_content="To: $recipient_email Subject: [NOTIFY Compass-CI] Self-test report -Dear $my_name: +Dear $author_name and $commiter: + + commit_id: $commit_id + + commit_subject: $commit_subject Your self-test groud_id=selftest job result is: diff --git a/tests/self-test b/tests/self-test index fb3e87b17..11786ac3f 100755 --- a/tests/self-test +++ b/tests/self-test @@ -28,5 +28,23 @@ submit_job() log_info "selftest job result: $report_content" } +get_commit_info() +{ + cd /c/compass-ci || exit + + commit_id=$(git log -1 --format="%H") + commit_subject=$(git log -1 --format="%s") + + author_name=$(git log -1 --format="%an") + author_email=$(git log -1 --format="%ae") + + commiter=$(git log -1 --format="%cn") + commiter_email=$(git log -1 --format="%ce") + + recipient_email="$author_email;$commiter_email" + +} + +get_commit_info submit_job send_email selftest -- 2.23.0

1 0

[PATCH v2 lkp-tests 6/6] pkg/mysql-8-0-20: add build install benchmarksql
by Luan Shengde 29 Apr '21

29 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- pkg/mysql-8-0-20/PKGBUILD | 42 +++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/pkg/mysql-8-0-20/PKGBUILD b/pkg/mysql-8-0-20/PKGBUILD index 66df6b33f..de56cf918 100644 --- a/pkg/mysql-8-0-20/PKGBUILD +++ b/pkg/mysql-8-0-20/PKGBUILD @@ -7,13 +7,14 @@ url="http://www.mysql.com" license=('GPLv2') source=("git://$GIT_SERVER/upstream/m/my-sql_with_boostkit_patch/my-sql_with_boostkit_patch.git" "git://$GIT_SERVER/upstream/g/gcc/gcc.git" + "https://mirrors.huaweicloud.com/kunpeng/archive/kunpeng_solution/database/p…" "my.cnf") -md5sums=('SKIP' 'SKIP' '99c4965aa870c475000993f52a0dee66') +md5sums=('SKIP' 'SKIP' 'f33c03fd48ff5993dca6d00b53a46da4' '99c4965aa870c475000993f52a0dee66') build() { - build_gcc build_mysql + build_benchmarksql } build_gcc() @@ -58,6 +59,11 @@ build_mysql() unset CPPFLAGS unset CFLAGS unset CXXFLAGS + + cd $srcdir/my-sql_with_boostkit_patch && { + grep 3f3d2692fd05a18011d93dcadeef661f386595ac compass-ci.cfg || build_gcc + } + cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver CMAKE=cmake @@ -70,11 +76,39 @@ build_mysql() } } +build_benchmarksql() +{ + cd $srcdir && { + cp -r benchmarksql5.0-for-mysql "$HOME/" + chmod -R 777 "$HOME/benchmarksql5.0-for-mysql" + } + + cd "$HOME/benchmarksql5.0-for-mysql/run" && { + chmod 777 *.sh + sed -i \ + -e '/^conn=/ s|mysql://.*:3306|mysql://localhost:3306|' \ + -e '/^user=/cuser=root' \ + -e '/^password=/cpassword=mysql_password' \ + -e '/^warehouses=/cwarehouses=1000' \ + -e '/^loadWorkers=/cloadWorkers=100' \ + -e '/^terminals=/cterminals=200' \ + -e '/^runTxnsPerTerminal=/crunTxnsPerTerminal=0' \ + -e '/^runMins=/crunMins=30' \ + -e '/^limitTxnsPerMin=/climitTxnsPerMin=1000' \ + -e '/^terminalWarehouseFixed=/cterminalWarehouseFixed=true' \ + -e '/^newOrderWeight=/cnewOrderWeight=45' \ + -e '/^paymentWeight=/cpaymentWeight=43' \ + -e '/^orderStatusWeight=/corderStatusWeight=4' \ + -e '/^deliveryWeight=/cdeliveryWeight=4' \ + props.conf + } +} + package() { cp my.cnf /etc/ cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver/build && { - make install - } + make install + } } -- 2.23.0

1 0

[PATCH v2 lkp-tests 5/6] tests: add init database for benchmark
by Luan Shengde 29 Apr '21

29 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/mysql-8-0-20 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/mysql-8-0-20 b/tests/mysql-8-0-20 index 990a053d7..e982eb92e 100755 --- a/tests/mysql-8-0-20 +++ b/tests/mysql-8-0-20 @@ -4,13 +4,16 @@ # mysql_host # mysql_port # mysql_db +# benchmark_db # mysql_password + . $LKP_SRC/lib/common.sh : "${mysql_user:=root}" : "${mysql_host:=127.0.0.1}" : "${mysql_port:=3306}" : "${mysql_db:=sysbench_test}" +: "${benchmark_db:=tpcc}" : "${mysql_password:=mysql_password}" create_basedir() @@ -40,6 +43,8 @@ configure_mysql() sleep 10 "$mysql" -u "$mysql_user" -e "flush privileges; use mysql; alter user 'root'@'localhost' identified by '$mysql_password'; flush privileges;" + sed -i '/^skip-grant-tables/d' /etc/my.cnf + sed -i '/^skip-name-resolve/d' /etc/my.cnf "$mysql_server" restart sleep 10 } @@ -48,6 +53,7 @@ create_database() { "$mysql" -u "$mysql_user" -p"$mysql_password" -e "use mysql; update user set host ='%' where user='root'; flush privileges;" "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $mysql_db;" + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $benchmark_db;" "$mysql_server" stop sleep 10 } -- 2.23.0

1 0

[PATCH v2 lkp-tests 4/6] add script for benchmarksql
by Luan Shengde 29 Apr '21

29 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/benchmark-sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 tests/benchmark-sql diff --git a/tests/benchmark-sql b/tests/benchmark-sql new file mode 100755 index 000000000..2ec6a7317 --- /dev/null +++ b/tests/benchmark-sql @@ -0,0 +1,17 @@ +#!/bin/sh +# benchmark_pkg_name + +: "${benchmark_pkg_name:=benchmarksql5.0-for-mysql}" + +run_benchmark_mysql() +{ + scripts=(runDatabaseBuild.sh runBenchmark.sh runDatabaseDestroy.sh) + + cd "$HOME/$benchmark_pkg_name/run/" && { + ./runDatabaseBuild.sh props.conf + ./runBenchmark.sh props.conf + ./runDatabaseDestroy.sh props.conf + } +} + +run_benchmark_mysql -- 2.23.0

1 0

[PATCH v2 lkp-tests 3/6] stats: add stats file for benchmarksql
by Luan Shengde 29 Apr '21

29 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- stats/benchmark-sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 stats/benchmark-sql diff --git a/stats/benchmark-sql b/stats/benchmark-sql new file mode 100755 index 000000000..d4794b4f4 --- /dev/null +++ b/stats/benchmark-sql @@ -0,0 +1,16 @@ +#!/usr/bin/env ruby + +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Measured tpmC (NewOrders) = 450.31 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Measured tpmTOTAL = 999.71 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Session Start = 2021-04-29 10:58:15 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Session End = 2021-04-29 11:28:16 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Transaction Count = 30003 + +while (line = STDIN.gets) + case line.chomp! + when /Measured tpmC $NewOrders$ = (\d*\.\d*)/ + puts "Measured_tpmC$NewOrders$: #{$1}" + when /Measured tpmTOTAL = (\d*\.\d*)/ + puts "Measured_tpmTOTAL: #{$1}" + end +end -- 2.23.0

1 0

[PATCH v2 lkp-tests 2/6] add job yaml for benchmarksql
by Luan Shengde 29 Apr '21

29 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/benchmark-sql.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 jobs/benchmark-sql.yaml diff --git a/jobs/benchmark-sql.yaml b/jobs/benchmark-sql.yaml new file mode 100644 index 000000000..152253c4e --- /dev/null +++ b/jobs/benchmark-sql.yaml @@ -0,0 +1,20 @@ +suite: benchmark-sql + +os: openeuler +os_version: 20.03 +os_mount: initramfs + +mysql_user: +mysql_host: localhost +mysql_port: +mysql_db: +benchmark_db: tpcc +mysql_password: +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +benchmark-sql: -- 2.23.0

1 0

[PATCH v2 lkp-tests 1/6] add job yaml for sysbench
by Luan Shengde 29 Apr '21

29 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/sysbench-mysql.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 jobs/sysbench-mysql.yaml diff --git a/jobs/sysbench-mysql.yaml b/jobs/sysbench-mysql.yaml new file mode 100644 index 000000000..6c6606ea2 --- /dev/null +++ b/jobs/sysbench-mysql.yaml @@ -0,0 +1,33 @@ +suite: sysbench-mysql + +os: openeuler +os_version: 20.03 +os_mount: initramfs + +mysql_user: +mysql_host: localhost +mysql_port: +mysql_db: +benchmark_db: tpcc +mysql_password: +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +sysbench-mysql: + oltp_test_mode: complex + max_requests: 0 + mysql_table_engine: innodb + oltp_tables_count: 1000 + oltp_table_size: 100000 + nr_threads: 256 + runtime: 60 + rand_type: special + rand_spec_pct: 100 + oltp_read_only: false + report_interval: 1 + events: 20000 + mysql_socket: /data/mysql/run/mysql.sock -- 2.23.0

1 0

[PATCH lkp-tests 6/6] pkg/mysql-8-0-20: add build install benchmarksql
by Luan Shengde 29 Apr '21

29 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- pkg/mysql-8-0-20/PKGBUILD | 42 +++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/pkg/mysql-8-0-20/PKGBUILD b/pkg/mysql-8-0-20/PKGBUILD index 66df6b33f..cf7a96d6b 100644 --- a/pkg/mysql-8-0-20/PKGBUILD +++ b/pkg/mysql-8-0-20/PKGBUILD @@ -7,13 +7,14 @@ url="http://www.mysql.com" license=('GPLv2') source=("git://$GIT_SERVER/upstream/m/my-sql_with_boostkit_patch/my-sql_with_boostkit_patch.git" "git://$GIT_SERVER/upstream/g/gcc/gcc.git" + "https://mirrors.huaweicloud.com/kunpeng/archive/kunpeng_solution/database/p…" "my.cnf") -md5sums=('SKIP' 'SKIP' '99c4965aa870c475000993f52a0dee66') +md5sums=('SKIP' 'SKIP' 'f33c03fd48ff5993dca6d00b53a46da4' '99c4965aa870c475000993f52a0dee66') build() { - build_gcc build_mysql + build_benchmarksql } build_gcc() @@ -58,6 +59,11 @@ build_mysql() unset CPPFLAGS unset CFLAGS unset CXXFLAGS + + cd $srcdir/my-sql_with_boostkit_patch && { + grep 3f3d2692fd05a18011d93dcadeef661f386595ac compass-ci.cfg || build_gcc + } + cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver CMAKE=cmake @@ -70,11 +76,39 @@ build_mysql() } } +build_benchmarksql() +{ + cd $srcdir && { + cp -r benchmarksql5.0-for-mysql "$HOME/" + chmod -R 777 "$HOME/benchmarksql5.0-for-mysql" + } + + cd "$HOME/benchmarksql5.0-for-mysql/run" && { + chmod 777 *.sh + sed -i \ + -e '/^conn=/ s|mysql://.*:3306|mysql://localhost:3306|' \ + -e '/^user=/cuser=root' \ + -e '/^password=/cpassword=mysql_password' \ + -e '/^warehouses=/cwarehouses=10' \ + -e '/^loadWorkers=/cloadWorkers=10' \ + -e '/^terminals=/cterminals=10' \ + -e '/^runTxnsPerTerminal=/crunTxnsPerTerminal=0' \ + -e '/^runMins=/crunMins=30' \ + -e '/^limitTxnsPerMin=/climitTxnsPerMin=1000' \ + -e '/^terminalWarehouseFixed=/cterminalWarehouseFixed=true' \ + -e '/^newOrderWeight=/cnewOrderWeight=45' \ + -e '/^paymentWeight=/cpaymentWeight=43' \ + -e '/^orderStatusWeight=/corderStatusWeight=4' \ + -e '/^deliveryWeight=/cdeliveryWeight=4' \ + props.conf + } +} + package() { cp my.cnf /etc/ cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver/build && { - make install - } + make install + } } -- 2.23.0

1 1

[PATCH lkp-tests 5/6] tests: add init database for benchmark
by Luan Shengde 29 Apr '21

29 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/mysql-8-0-20 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/mysql-8-0-20 b/tests/mysql-8-0-20 index 990a053d7..e982eb92e 100755 --- a/tests/mysql-8-0-20 +++ b/tests/mysql-8-0-20 @@ -4,13 +4,16 @@ # mysql_host # mysql_port # mysql_db +# benchmark_db # mysql_password + . $LKP_SRC/lib/common.sh : "${mysql_user:=root}" : "${mysql_host:=127.0.0.1}" : "${mysql_port:=3306}" : "${mysql_db:=sysbench_test}" +: "${benchmark_db:=tpcc}" : "${mysql_password:=mysql_password}" create_basedir() @@ -40,6 +43,8 @@ configure_mysql() sleep 10 "$mysql" -u "$mysql_user" -e "flush privileges; use mysql; alter user 'root'@'localhost' identified by '$mysql_password'; flush privileges;" + sed -i '/^skip-grant-tables/d' /etc/my.cnf + sed -i '/^skip-name-resolve/d' /etc/my.cnf "$mysql_server" restart sleep 10 } @@ -48,6 +53,7 @@ create_database() { "$mysql" -u "$mysql_user" -p"$mysql_password" -e "use mysql; update user set host ='%' where user='root'; flush privileges;" "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $mysql_db;" + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $benchmark_db;" "$mysql_server" stop sleep 10 } -- 2.23.0

1 1

[PATCH lkp-tests 4/6] tests: add script for benchmarksql
by Luan Shengde 29 Apr '21

29 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/benchmark-sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 tests/benchmark-sql diff --git a/tests/benchmark-sql b/tests/benchmark-sql new file mode 100755 index 000000000..2ec6a7317 --- /dev/null +++ b/tests/benchmark-sql @@ -0,0 +1,17 @@ +#!/bin/sh +# benchmark_pkg_name + +: "${benchmark_pkg_name:=benchmarksql5.0-for-mysql}" + +run_benchmark_mysql() +{ + scripts=(runDatabaseBuild.sh runBenchmark.sh runDatabaseDestroy.sh) + + cd "$HOME/$benchmark_pkg_name/run/" && { + ./runDatabaseBuild.sh props.conf + ./runBenchmark.sh props.conf + ./runDatabaseDestroy.sh props.conf + } +} + +run_benchmark_mysql -- 2.23.0

1 1

[PATCH lkp-tests 3/6] add stats file
by Luan Shengde 29 Apr '21

29 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- stats/benchmark-sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 stats/benchmark-sql diff --git a/stats/benchmark-sql b/stats/benchmark-sql new file mode 100755 index 000000000..d4794b4f4 --- /dev/null +++ b/stats/benchmark-sql @@ -0,0 +1,16 @@ +#!/usr/bin/env ruby + +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Measured tpmC (NewOrders) = 450.31 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Measured tpmTOTAL = 999.71 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Session Start = 2021-04-29 10:58:15 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Session End = 2021-04-29 11:28:16 +# 11:28:16,052 [Thread-2] INFO jTPCC : Term-00, Transaction Count = 30003 + +while (line = STDIN.gets) + case line.chomp! + when /Measured tpmC $NewOrders$ = (\d*\.\d*)/ + puts "Measured_tpmC$NewOrders$: #{$1}" + when /Measured tpmTOTAL = (\d*\.\d*)/ + puts "Measured_tpmTOTAL: #{$1}" + end +end -- 2.23.0

1 1

[PATCH lkp-tests 2/6] add job yaml for benmarksql
by Luan Shengde 29 Apr '21

29 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/benchmark-sql.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 jobs/benchmark-sql.yaml diff --git a/jobs/benchmark-sql.yaml b/jobs/benchmark-sql.yaml new file mode 100644 index 000000000..152253c4e --- /dev/null +++ b/jobs/benchmark-sql.yaml @@ -0,0 +1,20 @@ +suite: benchmark-sql + +os: openeuler +os_version: 20.03 +os_mount: initramfs + +mysql_user: +mysql_host: localhost +mysql_port: +mysql_db: +benchmark_db: tpcc +mysql_password: +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +benchmark-sql: -- 2.23.0

1 0

[PATCH lkp-tests 1/6] add job yaml for sysbenc
by Luan Shengde 29 Apr '21

29 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/sysbench-mysql.yaml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 jobs/sysbench-mysql.yaml diff --git a/jobs/sysbench-mysql.yaml b/jobs/sysbench-mysql.yaml new file mode 100644 index 000000000..6c6606ea2 --- /dev/null +++ b/jobs/sysbench-mysql.yaml @@ -0,0 +1,33 @@ +suite: sysbench-mysql + +os: openeuler +os_version: 20.03 +os_mount: initramfs + +mysql_user: +mysql_host: localhost +mysql_port: +mysql_db: +benchmark_db: tpcc +mysql_password: +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +sysbench-mysql: + oltp_test_mode: complex + max_requests: 0 + mysql_table_engine: innodb + oltp_tables_count: 1000 + oltp_table_size: 100000 + nr_threads: 256 + runtime: 60 + rand_type: special + rand_spec_pct: 100 + oltp_read_only: false + report_interval: 1 + events: 20000 + mysql_socket: /data/mysql/run/mysql.sock -- 2.23.0

1 0

[PATCH lab-z9] add host-info
by Zhang Dewan 29 Apr '21

29 Apr '21

Signed-off-by: Zhang Dewan <513619464(a)qq.com> --- hosts/taishan200-2280-2s64p-256g--a111 | 19 ++++++++++++++++ hosts/taishan200-2280-2s64p-256g--a134 | 19 ++++++++++++++++ hosts/taishan200-2280-2s64p-256g--a135 | 21 +++++++++++++++++ hosts/taishan200-2280-2s64p-256g--a136 | 21 +++++++++++++++++ hosts/taishan200-2280-2s64p-256g--a46 | 20 +++++++++++++++++ hosts/taishan200-2280-2s64p-256g--a82 | 31 ++++++++++++++++++++++++++ 6 files changed, 131 insertions(+) create mode 100644 hosts/taishan200-2280-2s64p-256g--a111 create mode 100644 hosts/taishan200-2280-2s64p-256g--a134 create mode 100644 hosts/taishan200-2280-2s64p-256g--a135 create mode 100644 hosts/taishan200-2280-2s64p-256g--a136 create mode 100644 hosts/taishan200-2280-2s64p-256g--a46 create mode 100644 hosts/taishan200-2280-2s64p-256g--a82 diff --git a/hosts/taishan200-2280-2s64p-256g--a111 b/hosts/taishan200-2280-2s64p-256g--a111 new file mode 100644 index 0000000..cc94ef3 --- /dev/null +++ b/hosts/taishan200-2280-2s64p-256g--a111 @@ -0,0 +1,19 @@ +memory: 128G +nr_hdd_partitions: 2 +hdd_partitions: + - /dev/disk/by-id/scsi-350000399886b6fa5 + - /dev/disk/by-id/scsi-350000399886b69c9 +mac_addr: + - 44:67:47:85:f1:1d + - 44:67:47:85:f1:1e + - 44:67:47:85:f1:1f + - 44:67:47:85:f1:20 + - 44:a1:91:9f:9a:da + - 44:a1:91:9f:9a:db + - 44:a1:91:9f:9a:dc + - 44:a1:91:9f:9a:dd +arch: aarch64 +nr_node: 4 +nr_cpu: 128 +model_name: Kunpeng-920 +ipmi_ip: 9.3.11.4 diff --git a/hosts/taishan200-2280-2s64p-256g--a134 b/hosts/taishan200-2280-2s64p-256g--a134 new file mode 100644 index 0000000..bf223d2 --- /dev/null +++ b/hosts/taishan200-2280-2s64p-256g--a134 @@ -0,0 +1,19 @@ +memory: 128G +nr_hdd_partitions: 2 +hdd_partitions: + - /dev/disk/by-id/scsi-35000c500bd685123 + - /dev/disk/by-id/scsi-35000c500bd6a1bef +mac_addr: + - cc:05:77:fe:d0:44 + - cc:05:77:fe:d0:45 + - cc:05:77:fe:d0:46 + - cc:05:77:fe:d0:47 + - 44:a1:91:9f:ae:ba + - 44:a1:91:9f:ae:bb + - 44:a1:91:9f:ae:bc + - 44:a1:91:9f:ae:bd +arch: aarch64 +nr_node: 4 +nr_cpu: 128 +model_name: Kunpeng-920 +ipmi_ip: 9.3.14.3 diff --git a/hosts/taishan200-2280-2s64p-256g--a135 b/hosts/taishan200-2280-2s64p-256g--a135 new file mode 100644 index 0000000..1a64502 --- /dev/null +++ b/hosts/taishan200-2280-2s64p-256g--a135 @@ -0,0 +1,21 @@ +memory: 256G +nr_hdd_partitions: 4 +hdd_partitions: + - /dev/disk/by-id/scsi-350000399b88bf94d + - /dev/disk/by-id/scsi-350000399b883bbf5 + - /dev/disk/by-id/scsi-350000399b883b77d + - /dev/disk/by-id/scsi-350000399b88bf9ed +mac_addr: + - 44:67:47:86:1a:28 + - 44:67:47:86:1a:29 + - 44:67:47:86:1a:2a + - 44:67:47:86:1a:2b + - 44:67:47:86:1a:2c + - 44:67:47:86:1a:2d + - 44:67:47:86:1a:2e + - 44:67:47:86:1a:2f +arch: aarch64 +nr_node: 4 +nr_cpu: 96 +model_name: Kunpeng-920 +ipmi_ip: 9.3.14.4 diff --git a/hosts/taishan200-2280-2s64p-256g--a136 b/hosts/taishan200-2280-2s64p-256g--a136 new file mode 100644 index 0000000..6623ab5 --- /dev/null +++ b/hosts/taishan200-2280-2s64p-256g--a136 @@ -0,0 +1,21 @@ +memory: 256G +nr_hdd_partitions: 4 +hdd_partitions: + - /dev/disk/by-id/scsi-35000c500c278b1f7 + - /dev/disk/by-id/scsi-35000c500c27e7efb + - /dev/disk/by-id/scsi-35000c500c27f345f + - /dev/disk/by-id/scsi-35000c500c27d013b +mac_addr: + - 44:67:47:c9:df:f2 + - 44:67:47:c9:df:f3 + - 44:67:47:c9:df:f4 + - 44:67:47:c9:df:f5 + - 44:67:47:c9:df:f6 + - 44:67:47:c9:df:f7 + - 44:67:47:c9:df:f8 + - 44:67:47:c9:df:f9 +arch: aarch64 +nr_node: 4 +nr_cpu: 96 +model_name: Kunpeng-920 +ipmi_ip: 9.3.14.5 diff --git a/hosts/taishan200-2280-2s64p-256g--a46 b/hosts/taishan200-2280-2s64p-256g--a46 new file mode 100644 index 0000000..ba03287 --- /dev/null +++ b/hosts/taishan200-2280-2s64p-256g--a46 @@ -0,0 +1,20 @@ +memory: 128G +nr_hdd_partitions: 2 +hdd_partitions: + - /dev/disk/by-id/scsi-35000039988694285 + - /dev/disk/by-id/scsi-350000399886a564d + - /dev/disk/by-id/scsi-350000399886a564d-part1 +mac_addr: + - 44:67:47:85:f8:4d + - 44:67:47:85:f8:4e + - 44:67:47:85:f8:4f + - 44:67:47:85:f8:50 + - cc:05:77:ee:b0:4f + - cc:05:77:ee:b0:50 + - cc:05:77:ee:b0:51 + - cc:05:77:ee:b0:52 +arch: aarch64 +nr_node: 4 +nr_cpu: 128 +model_name: Kunpeng-920 +ipmi_ip: 9.3.2.11 diff --git a/hosts/taishan200-2280-2s64p-256g--a82 b/hosts/taishan200-2280-2s64p-256g--a82 new file mode 100644 index 0000000..02e8376 --- /dev/null +++ b/hosts/taishan200-2280-2s64p-256g--a82 @@ -0,0 +1,31 @@ +memory: 256G +nr_hdd_partitions: 1 +nr_ssd_partitions: 1 +hdd_partitions: + - /dev/disk/by-id/scsi-3500003999812edd1 + - /dev/disk/by-id/scsi-3500003999812edd1-part1 + - /dev/disk/by-id/scsi-3500003999812edd1-part2 + - /dev/disk/by-id/scsi-3500003999812edd1-part3 +ssd_partitions: + - /dev/disk/by-id/ata-SAMSUNG_MZ7LH480HAHQ-00005_S45PNA2MB43947 + - /dev/disk/by-id/ata-SAMSUNG_MZ7LH480HAHQ-00005_S45PNA2MB43947-part1 + - /dev/disk/by-id/ata-SAMSUNG_MZ7LH480HAHQ-00005_S45PNA2MB43947-part2 + - /dev/disk/by-id/ata-SAMSUNG_MZ7LH480HAHQ-00005_S45PNA2MB43947-part3 +mac_addr: + - 84:46:fe:26:d0:59 + - 84:46:fe:26:d0:5a + - 84:46:fe:26:d0:5b + - 84:46:fe:26:d0:5c + - ac:b3:b5:1e:bd:fb + - ac:b3:b5:1e:bd:fc + - ac:b3:b5:1e:bd:fd + - ac:b3:b5:1e:bd:fe + - ac:b3:b5:1e:bc:ab + - ac:b3:b5:1e:bc:ac + - ac:b3:b5:1e:bc:ad + - ac:b3:b5:1e:bc:ae +arch: aarch64 +nr_node: 4 +nr_cpu: 128 +model_name: Kunpeng-920 +ipmi_ip: 9.3.6.11 -- 2.23.0

1 0

git count
by Wu Fengguang 29 Apr '21

29 Apr '21

Liu Yinsi is our code star,thanks Liu Yinsi! lys@crystal ~/todo/bin% ./git-count 20 Liu Yinsi 13 Yu Chuan 12 Li Yuanchao 11 Lu Weitao 9 Cao Xueliang 9 Luan Shengde 8 Bai Jing 8 Liu Shaofei 8 Wu Zhende 3 Xu Xijian 3 Wang Chenglong 3 Lin Jiaxin 3 Li Ping 2 Wei Jihui 2 Wu Fengguang 2 Wang Yong 1 Cui Yili 0 Du Kaitian 0 Zhang Yale 0 Zhang Dewan

1 0

help star
by Wu Fengguang 29 Apr '21

29 Apr '21

Wei Jihui is our help star again, thanks Wei Jihui! 12.0 Wei Jihui 10.5 Liu Yinsi 9.2 Wu Zhende 8.2 Yu Chuan 8.1 Li Yuanchao 7.0 Lu Weitao 6.4 Zhang Dewan 6.1 Cao Xueliang 5.2 Luan Shengde 4.2 Lin Jiaxin 3.6 Xu Xijian 2.6 Wu Fengguang 2.0 Wang Chenglong 1.4 Liu Shaofei 1.3 Wang Yong 1.2 Bai Jing 0.8 Li Ping 0.0 Du Kaitian 0.0 Zhang Yale 0.0 Cui Yili

1 0

review star
by Wu Fengguang 29 Apr '21

29 Apr '21

: wqUser-Agent: Mutt/1.10.1 (2018-07-13) User-Agent: Mutt/1.10.1 (2018-07-13) Xu Xijian is our review star, thanks Xu Xijian! lys@crystal ~/todo/bin% ./review-star Review email records in the last 7 days total review paragraph point people 24 18 22 16 Xu Xijian 48 17 25 11 Wei Jihui 39 9 18 8 Luan Shengde 143 12 14 7 Cao Xueliang 46 7 6 6 Liu Shaofei 7 3 2 4 Zhang Dewan 11 3 5 1 Li Ping 53 2 2 1 Liu Yinsi 19 2 2 1 Wang Yong 5 0 0 0 Cui Yili 27 0 0 0 Li Yuanchao 48 2 9 0 Yu Chuan 24 0 0 0 Wu Zhende 20 0 0 0 Bai Jing 7 1 1 0 Lin Jiaxin 18 0 0 0 Lu Weitao 42 0 0 0 Wang Chenglong 8 0 0 0 Wu Fengguang

1 0

[PATCH lkp-tests] bin/run-ipconfig: support the configuration of multiple network cards
by Cui Yili 29 Apr '21

29 Apr '21

[why] rewrite the configure_networking function to support the configuration of multiple network cards. Signed-off-by: Cui Yili <2268260388(a)qq.com> --- bin/run-ipconfig | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/bin/run-ipconfig b/bin/run-ipconfig index 0f80feafe..ceff60f6f 100755 --- a/bin/run-ipconfig +++ b/bin/run-ipconfig @@ -6,6 +6,43 @@ export PATH="$PATH:/usr/lib/klibc/bin" IP=$DIRECT_IP +# rewrite the configure_networking function +# to support the configuration of multiple network cards +configure_networking() +{ + for device in /sys/class/net/* ; do + if [ -f "$device/address" ]; then + DEVICE=${device##*/} + + [ "${DEVICE}" = "lo" ] && continue + [ "${DEVICE}" = "docker0" ] && continue + + for ROUNDTTT in 2 3 4 6 9 16 25 36 64 100; do + + [ -e /run/net-"${DEVICE}".conf ] && break + case ${IP} in + none|off) + # Do nothing + ;; + ""|on|any) + # Bring up device + ipconfig -t ${ROUNDTTT} "${DEVICE}" + ;; + dhcp|bootp|rarp|both) + ipconfig -t ${ROUNDTTT} -c "${IP}" -d "${DEVICE}" + ;; + *) + ipconfig -t ${ROUNDTTT} -d "$IP" + ;; + esac + done + + . "/run/net-${DEVICE}.conf" + fi + done + +} + read_ip() { for x in $(cat /proc/cmdline) -- 2.23.0

1 0

[PATCH compass-ci] sbin/auto_submit: add commit_title
by Li Yuanchao 29 Apr '21

29 Apr '21

So it would be easy for user to find the commit. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sbin/auto_submit b/sbin/auto_submit index 595053e..08193e5 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -81,7 +81,9 @@ class AutoSubmit base_commit = %x(git -C #{mirror_dir} merge-base master #{branch}).chomp argvs_for_base.push("upstream_commit=#{base_commit}") system(argvs_for_base.join(' ')) - return "author_email=#{author_email} committer_email=#{committer_email} base_commit=#{base_commit}" + commit_title = %x(git -C #{mirror_dir} log --format=%s -1 #{commit_id}).chomp + + return "author_email=#{author_email} committer_email=#{committer_email} base_commit=#{base_commit} commit_title=#{commit_title}" end def submit_specific_branch(submit_argv, newrefs_info, branches) -- 2.23.0

2 2

[PATCH v3 lkp-tests] add email feedback for rpmbuild-pkg
by Li Ping 29 Apr '21

29 Apr '21

tests/rpmbuild-pkg: add a new input parameter whether_email [usage]: rpmbuild-pkg: whether_email: 1 when job failed, send an rpmbuild failed email. when job run success, send an rpmbuild success email. Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/email.sh | 37 +++++++++++++++++++++++++++++++++++++ tests/rpmbuild-pkg | 15 +++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/lib/email.sh b/lib/email.sh index b5e56df79..0082e852d 100755 --- a/lib/email.sh +++ b/lib/email.sh @@ -96,6 +96,43 @@ Compass-CI " } +rpmbuild_report() +{ + local author_email=$(git_pretty_format "%ae") + local author_name=$(git_pretty_format "%an") + email_content="To: $author_email +Subject: [NOTIFY Compass-CI] rpmbuild report + +Dear $author_name: + + $rpmbuild_result + +Regards +Compass-CI +" +} + +rpmbuild_success_content() +{ + rpmbuild_result="Your RPM Package is successfully built. + +You can click the follow link to obtain your RPM Package: +http://api.compass-ci.openeuler.org:20012/rpm/pub/${os}-${os_version}/${compat_os}/${os_arch}/Packages + +And you can click the following link to view RPM Package build details: +http://api.compass-ci.openeuler.org:${SRV_HTTP_RESULT_PORT:-20007}$result_root" + + rpmbuild_report +} + +rpmbuild_failed_content() +{ + rpmbuild_result="Sorry to inform you that RPM Package built failed, you can click the following link to view details. +http://api.compass-ci.openeuler.org:${SRV_HTTP_RESULT_PORT:-20007}$result_root" + + rpmbuild_report +} + send_email() { local subject=$1 diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index 3dd75b896..970418848 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -5,6 +5,7 @@ # - repo_addr # - custom_repo_name # - custom_repo_addr +# - whether_email # Git repo jobs only use the first two parameters, SRPMs jobs use all # user can add custom_repo_name and custom_repo_addr to setup # local repo, value can be an array @@ -12,6 +13,7 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh . $LKP_SRC/lib/rpm.sh +. $LKP_SRC/lib/email.sh : "${compat_os:=budding-openeuler}" @@ -19,6 +21,13 @@ dest_dir="/rpm/upload/${os}-${os_version}/${compat_os}" +die() +{ + [ -n "$whether_email" ] && send_email "rpmbuild_failed" + echo "$@" 1>&2 + exit 99 +} + from_git() { package_name=${upstream_repo##*/} @@ -29,6 +38,11 @@ from_git() download_upstream_repo } +git_pretty_format() +{ + curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' -d '{"git_repo": "'${upstream_dir}/${upstream_repo}.git'", "git_command": ["git-log","-n1","--pretty=format:'$1'", "'${upstream_commit}'"]}' +} + from_srpm() { [ -n "$repo_name" ] || die "repo_name is empty" @@ -143,3 +157,4 @@ add_repo build_rpm upload_rpm_pkg update_repo_mq +[ -n "$whether_email" ] && send_email "rpmbuild_success" -- 2.23.0

1 0

[PATCH compass-ci] lib/job.cr: change borrow runtime 10 days to 30 days
by Cao Xueliang 29 Apr '21

29 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/lib/job.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index bb4ef96..98b2c3d 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -439,12 +439,12 @@ class Job # only job.yaml for borrowing machine has the key: ssh_pub_key return unless @hash.has_key?("ssh_pub_key") - # the maxmum borrowing time is limited no more than 10 days. + # the maxmum borrowing time is limited no more than 30 days. # case the runtime/sleep value count beyond the limit, # it will throw error message and prevent the submit for borrowing machine. # runtime value is converted to second. - max_run_time = 10 * 24 * 3600 - error_msg = "\nMachine borrow time(runtime/sleep) cannot exceed 10 days. Consider re-borrow.\n" + max_run_time = 30 * 24 * 3600 + error_msg = "\nMachine borrow time(runtime/sleep) cannot exceed 30 days. Consider re-borrow.\n" if @hash["pp"]["sleep"].as_i? sleep_run_time = @hash["pp"]["sleep"] -- 2.23.0

1 0

[PATCH compass-ci] sparrow/8-testbox: add multi-qemu
by Liu Yinsi 29 Apr '21

29 Apr '21

run qemu testboxes for user locally deploy ci env, if the host is virtual, not run qemu. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/8-testbox/multi-qemu | 7 +++++++ sparrow/install-tiny | 1 + 2 files changed, 8 insertions(+) create mode 100755 sparrow/8-testbox/multi-qemu diff --git a/sparrow/8-testbox/multi-qemu b/sparrow/8-testbox/multi-qemu new file mode 100755 index 00000000..8a995cdb --- /dev/null +++ b/sparrow/8-testbox/multi-qemu @@ -0,0 +1,7 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +dmidecode -s system-product-name | grep -iq "virtual" && exit + +nohup $CCI_SRC/providers/multi-qemu -n vm-2p8g -c 10 -q vm-2p8g.$(arch) & diff --git a/sparrow/install-tiny b/sparrow/install-tiny index 40358a4c..a81f2d5c 100755 --- a/sparrow/install-tiny +++ b/sparrow/install-tiny @@ -24,3 +24,4 @@ cd $CCI_SRC/sparrow || exit 5-build/register-account 7-systemd/systemd-setup 8-testbox/multi-docker +8-testbox/multi-qemu -- 2.23.0

2 2

[PATCH lkp-tests] jobs: add borrow-30d.yaml
by Liu Yinsi 29 Apr '21

29 Apr '21

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- jobs/borrow-30d.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 jobs/borrow-30d.yaml diff --git a/jobs/borrow-30d.yaml b/jobs/borrow-30d.yaml new file mode 100644 index 000000000..e24fd6480 --- /dev/null +++ b/jobs/borrow-30d.yaml @@ -0,0 +1,14 @@ +suite: borrow +testcase: borrow + +runtime: 30d +ssh_pub_key: <%= + begin + File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp + rescue + nil + end + %> +sshd: +# sleep at the bottom +sleep: -- 2.23.0

1 0

[PATCH compass-ci] sparrow/0-package: add gem dep nokogiri for libvirt
by Xu Xijian 29 Apr '21

29 Apr '21

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- sparrow/0-package/common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sparrow/0-package/common b/sparrow/0-package/common index 82c09ff..a827a65 100755 --- a/sparrow/0-package/common +++ b/sparrow/0-package/common @@ -8,7 +8,8 @@ public_network_ok && { gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ - gem install --no-user-install git activesupport rest-client cucumber json faye-websocket elasticsearch bunny + gem install --no-user-install git activesupport rest-client cucumber json faye-websocket elasticsearch bunny \ + nokogiri } grep -q '^lkp:' /etc/passwd || useradd -u 1090 lkp -- 2.23.0

1 0

[PATCH v2 compass-ci] sparrow/0-package: add deps libxml2-devel libxslt-devel
by Xu Xijian 29 Apr '21

29 Apr '21

These 2 deps are for nokogiri, and nokogiri is dep of libvirt. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- sparrow/0-package/os/centos | 2 ++ sparrow/0-package/os/openEuler | 2 ++ 2 files changed, 4 insertions(+) diff --git a/sparrow/0-package/os/centos b/sparrow/0-package/os/centos index d4d1045..3f7dcac 100755 --- a/sparrow/0-package/os/centos +++ b/sparrow/0-package/os/centos @@ -26,6 +26,8 @@ pkgs=( curl sudo iputils + libxml2-devel + libxslt-devel ) yum install -y --skip-broken "${pkgs[@]}" diff --git a/sparrow/0-package/os/openEuler b/sparrow/0-package/os/openEuler index 5829a12..5153239 100755 --- a/sparrow/0-package/os/openEuler +++ b/sparrow/0-package/os/openEuler @@ -29,6 +29,8 @@ pkgs=( curl sudo iputils + libxml2-devel + libxslt-devel ) public_network_ok && { -- 2.23.0

1 0

[PATCH compass-ci 4/4] lib/mail_job_result.rb: fix incorrect logic
by Lu Weitao 29 Apr '21

29 Apr '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/mail_job_result.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mail_job_result.rb b/lib/mail_job_result.rb index 389ad26a..744b7d5c 100644 --- a/lib/mail_job_result.rb +++ b/lib/mail_job_result.rb @@ -77,7 +77,7 @@ def get_compare_result(job) commit_id = job['upstream_commit'] return nil unless base_commit && commit_id - condition_list = [{'base_commit' => base_commit}, {'upstream_commit' => commit_id}] + condition_list = [{'upstream_commit' => base_commit}, {'upstream_commit' => commit_id}] options = { :min_samples => min_samples, :no_print => true} matrices_list, suite_list = create_matrices_list(condition_list, options[:min_samples]) -- 2.23.0

1 0

[PATCH compass-ci 3/4] lib/es_query.rb: support unmatch fields
by Lu Weitao 29 Apr '21

29 Apr '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/es_query.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 8c5a28fe..9d596bb2 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -15,17 +15,20 @@ class ESQuery end # Example @items: { key1 => value1, key2 => [value2, value3, ..], ...} + # Example @unmatch_items: { key1 => value1, key2 => [value2, value3, ..], ...} # means to query: key1 == value1 && (key2 in [value2, value3, ..]) - def multi_field_query(items, size: 10_000, desc_keyword: nil) + def multi_field_query(items, unmatch_items: {}, size: 10_000, desc_keyword: nil) unless items warn 'empty filter!' exit end query_fields = build_multi_field_subquery_body items + unmatch_fields = build_multi_field_subquery_body unmatch_items query = { query: { bool: { - must: query_fields + must: query_fields, + must_not: unmatch_fields } }, size: size } -- 2.23.0

1 0

[PATCH compass-ci 2/4] lib/compare_matrixes.rb cancel remove 0 changed metric
by Lu Weitao 29 Apr '21

29 Apr '21

[Why] some user are still need check 0 changed metric Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/compare_matrixes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compare_matrixes.rb b/lib/compare_matrixes.rb index da2c2b92..cff73cbe 100644 --- a/lib/compare_matrixes.rb +++ b/lib/compare_matrixes.rb @@ -191,7 +191,7 @@ def remove_unchanged_field(matrixes_values, suite_list) # matrixes_values.each_key do |success| matrixes_values[success].delete_if do |field| - if suite_list.any? { |suite| field.start_with?(suite) } + if success matrixes_values[success][field][:changed] = true end -- 2.23.0

1 0

[PATCH compass-ci 1/4] sbin/compare: cancel fill 0 to matrix
by Lu Weitao 29 Apr '21

29 Apr '21

[Why] matrix => { iperf.xxx => [100, 80, 0], ... } like above, if a stat_metric fill 0, the avg and stddev will be incorrect Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/compare_matrixes.rb | 13 +++++++++---- lib/matrix2.rb | 13 +------------ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/lib/compare_matrixes.rb b/lib/compare_matrixes.rb index fef5714a..da2c2b92 100644 --- a/lib/compare_matrixes.rb +++ b/lib/compare_matrixes.rb @@ -129,10 +129,15 @@ def get_values_by_field(matrixes_list, field, matrixes_size, success, options) # values = {} matrixes_list.length.times do |index| - value_list = fill_missing_with_zeros( - matrixes_list[index][field], - matrixes_size[index] - ) + value_list = matrixes_list[index][field] + unless value_list + if success + value_list = [0] unless value_list + else + value_list = [1] unless value_list + end + end + values[index] = get_values(value_list, success) next if index.zero? diff --git a/lib/matrix2.rb b/lib/matrix2.rb index 2fd63243..82ed9a7a 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -62,16 +62,6 @@ def create_stats(result_root) # stats end -def samples_fill_missing_zeros(value, size) - value.concat([0] * (size - value.size)) -end - -def matrix_fill_miss_zeros(matrix, col_size) - matrix.each_value do |value| - samples_fill_missing_zeros(value, col_size) - end -end - # input: job_list # return: matrix of Hash(String, Array(Number)) # Eg: matrix: { @@ -94,8 +84,7 @@ def create_matrix(job_list) matrix[key] << value end end - col_size = job_list.size - matrix_fill_miss_zeros(matrix, col_size) + return matrix, suites end -- 2.23.0

1 0

[PATCH compass-ci] sparrow/0-package: add deps libxml2-devel libxslt-devel
by Xu Xijian 29 Apr '21

29 Apr '21

These 2 deps are for nokogiri, and nokogiri is dep of libvirt. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- sparrow/0-package/os/openEuler | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparrow/0-package/os/openEuler b/sparrow/0-package/os/openEuler index 5829a12..5153239 100755 --- a/sparrow/0-package/os/openEuler +++ b/sparrow/0-package/os/openEuler @@ -29,6 +29,8 @@ pkgs=( curl sudo iputils + libxml2-devel + libxslt-devel ) public_network_ok && { -- 2.23.0

1 0

[PATCH v2 compass-ci 3/3] sbin/dashboard: add a new script to operate kibana dashboard
by Wu Zhende 29 Apr '21

29 Apr '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- sbin/dashboard | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100755 sbin/dashboard diff --git a/sbin/dashboard b/sbin/dashboard new file mode 100755 index 0000000..c15bd75 --- /dev/null +++ b/sbin/dashboard @@ -0,0 +1,67 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'optparse' +require_relative '../lib/constants.rb' +require_relative '../lib/kibana_dashboard.rb' + +opt_type = nil +opt_host = 'kibana' +opt_hash = {} + +options = OptionParser.new do |opts| + opts.banner = 'Usage: dashboard [options] file/id file/id ...' + opts.separator ' importing or exporting kibana dashboards' + opts.separator ' use "dashboard -e" to export kibana dashboards' + opts.separator ' use "dashboard -e -l" to export logging-kibana dashboards' + opts.separator ' use "dashboard -i" to import kibana dashboards' + opts.separator ' use "dashboard -e -h xx.xx.xx.xx -p xx" to assign kibana' + opts.separator '' + opts.separator 'options:' + + opts.on('-e', '--export', 'export kibana dashboards') do + opt_type = 'export' + end + + opts.on('-i', '--import', 'import kibana dashboards') do + opt_type = 'import' + end + + opts.on('-l', '--logging', 'sends requests to logging-kibana, the default value is kibana') do + opt_host = 'logging-kibana' + end + + opts.on('-h host', '--host host', 'set host') do |host| + opt_hash['host'] = host + end + + opts.on('-p port', '--port port', 'set port') do |port| + opt_hash['port'] = port + end +end + +options.parse!(ARGV) + +unless opt_type + puts(options) + exit +end + +parameters = ARGV + +hash = {} +hash['host'] = opt_host == 'kibana' ? KIBANA_HOST : LOGGING_KIBANA_HOST +hash['port'] = opt_host == 'kibana' ? KIBANA_PORT : LOGGING_KIBANA_PORT +hash['user'] = opt_host == 'kibana' ? ES_USER : LOGGING_ES_USER +hash['password'] = opt_host == 'kibana' ? ES_PASSWORD : LOGGING_ES_PASSWORD +hash.merge!(opt_hash) + +kibana_dashboard = KibanaDashboard.new(hash['host'], hash['port'], hash['user'], hash['password']) + +if opt_type == 'export' + kibana_dashboard.export(parameters) +elsif opt_type == 'import' + kibana_dashboard.import(parameters) +end -- 2.23.0

1 0

[PATCH v2 compass-ci 2/3] lib/kibana_dashboard.rb: add new class
by Wu Zhende 29 Apr '21

29 Apr '21

Add a public class to operate the kibana dashboard Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/constants.rb | 7 ++++ lib/kibana_dashboard.rb | 72 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 lib/kibana_dashboard.rb diff --git a/lib/constants.rb b/lib/constants.rb index c71c3de..ca66718 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -2,6 +2,7 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../container/defconfig.rb' config = cci_defaults @@ -30,6 +31,12 @@ ES_HOSTS = [{ password: ES_PASSWORD }] +KIBANA_HOST ||= config['KIBANA_HOST'] || '172.17.0.1' +KIBANA_PORT ||= config['KIBANA_PORT'] || '20017' + +LOGGING_KIBANA_HOST ||= config['LOGGING_KIBANA_HOST'] || '172.17.0.1' +LOGGING_KIBANA_PORT ||= config['LOGGING_KIBANA_PORT'] || '20000' + SEND_MAIL_HOST ||= config['SEND_MAIL_HOST'] || '172.17.0.1' SEND_MAIL_PORT ||= config['SEND_MAIL_PORT'] || 10001 diff --git a/lib/kibana_dashboard.rb b/lib/kibana_dashboard.rb new file mode 100644 index 0000000..d8cffea --- /dev/null +++ b/lib/kibana_dashboard.rb @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +CCI_SRC = ENV['CCI_SRC'] || File.dirname(File.dirname(File.realpath($PROGRAM_NAME))) + +require_relative 'constants.rb' + +# operat the kibana dashboards +class KibanaDashboard + def initialize(host = KIBANA_HOST, port = KIBANA_PORT, user = ES_USER, password = ES_PASSWORD) + @port = port + @profix = "curl -u #{user}:#{password}" + @profix += " http://#{host}:#{port}/api/kibana/dashboards/" + end + + def get_default_ids(dashboard_ids) + return dashboard_ids unless dashboard_ids.empty? + + return %w[data resource all] if @port == KIBANA_PORT + return %w[data all] if @port == LOGGING_KIBANA_PORT + + dashboard_ids + end + + def get_default_files(files) + return files unless files.empty? + + files = [] + tmp = [] + profix = nil + + if @port == KIBANA_PORT + profix = CCI_SRC + '/container/kibana/' + tmp = %w[data.json resource.json all.json] + elsif @port == LOGGING_KIBANA_PORT + profix = CCI_SRC + '/container/logging-kibana/' + tmp = %w[data.json all.json] + end + + tmp.each do |f| + files << profix + f + end + + files + end + + def export(dashboard_ids) + dashboard_ids = get_default_ids(dashboard_ids) + raise 'empty dashboard ids' if dashboard_ids.empty? + + profix = @profix + 'export?dashboard=' + dashboard_ids.each do |id| + cmd = profix + "#{id} >> #{id}.json" + puts cmd + system cmd + end + end + + def import(files) + files = get_default_files(files) + raise 'empty json files' if files.empty? + + profix = @profix + 'import' + profix += " -H 'Content-Type: application/json' -H 'kbn-xsrf: reporting'" + files.each do |f| + cmd = profix + " -d @#{f}" + puts cmd + system cmd + end + end +end -- 2.23.0

1 0

[PATCH v2 compass-ci 1/3] scheduler-https/start: fix start error
by Wu Zhende 29 Apr '21

29 Apr '21

Missing ENV key: "NODE_PORT" Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/scheduler-https/start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/scheduler-https/start b/container/scheduler-https/start index 5e79d2b..59e7278 100755 --- a/container/scheduler-https/start +++ b/container/scheduler-https/start @@ -69,6 +69,7 @@ cmd = %W[ ] + env + %W[ -e LKP_SRC=#{DEFAULT_LKP} -e CCI_SRC=#{DEFAULT_CCI} + -e NODE_PORT=#{SCHED_HTTPS_PORT} -v #{ENV['LKP_SRC']}:#{DEFAULT_LKP} -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} -v #{CCI_REPOS}/lab-#{LAB}:#{CCI_REPOS}/lab-#{LAB} -- 2.23.0

1 0

[PATCH compass-ci 2/2] dracut-initrd: allow different dir layout for local boot type
by Yu Chuan 29 Apr '21

29 Apr '21

[Why] The current way to get timestamp is strongly dependent on the directory layout, but our os_version is a symbolic link, which corresponds to a source folder structure that is not fixed. But its suffix format is the same, so we get timestamp by intercepting the suffix. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- container/dracut-initrd/bin/set-local-sysroot.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/container/dracut-initrd/bin/set-local-sysroot.sh b/container/dracut-initrd/bin/set-local-sysroot.sh index c6aa8d114d68..0a1125eaace7 100644 --- a/container/dracut-initrd/bin/set-local-sysroot.sh +++ b/container/dracut-initrd/bin/set-local-sysroot.sh @@ -14,7 +14,11 @@ analyse_kernel_cmdline_params() { # example: $nfs_server_ip:/os/${os}/${os_arch}/${os_version}-snapshots/20210310005959 rootfs_src=$(echo $"$rootfs" | sed 's/\///') - timestamp="$(basename $rootfs_src)" + + # adapt $nfs_server_ip:/os/${os}/${os_arch}/${os_version}-2021-03-10-00-59-59 + timestamp="$(echo ${rootfs_src//-/} | grep -oE '[0-9]{14}$')" + [ -n "$timestamp" ] || reboot_with_msg "cannot find right timestamp" + os="$(echo $rootfs_src | awk -F '/|-' '{print $2}')" os_arch="$(echo $rootfs_src | awk -F '/|-' '{print $3}')" os_version="$(echo $rootfs_src | awk -F '/|-' '{print $4}')" -- 2.23.0

2 1

[PATCH compass-ci 1/2] dracut-initrd: optimize the log
by Yu Chuan 29 Apr '21

29 Apr '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- .../dracut-initrd/bin/set-local-sysroot.sh | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/container/dracut-initrd/bin/set-local-sysroot.sh b/container/dracut-initrd/bin/set-local-sysroot.sh index 1a2edafaf967..c6aa8d114d68 100644 --- a/container/dracut-initrd/bin/set-local-sysroot.sh +++ b/container/dracut-initrd/bin/set-local-sysroot.sh @@ -1,5 +1,11 @@ #!/bin/sh +reboot_with_msg() +{ + echo "[compass-ci reboot] $1" + reboot +} + analyse_kernel_cmdline_params() { rootfs="$(getarg root=)" @@ -25,28 +31,22 @@ sync_src_lv() { # need create volume group, usually in first use of this machine. $pv_device e.g. /dev/sda pv_device="$(getarg pv_device=)" [ -n "$pv_device" ] && { - [ -b "$pv_device" ] || { - echo "warn dracut: FATAL: device not found: $pv_device, reboot" - reboot - } + [ -b "$pv_device" ] || reboot_with_msg "warn dracut: FATAL: device not found: $pv_device" # ensure the physical disk has been initialized as physical volume real_pv_device="$(lvm pvs | grep -w $pv_device | awk '{print $1}')" [ "$real_pv_device" = "$pv_device" ] || { - lvm pvcreate -y "$pv_device" || reboot + lvm pvcreate -y "$pv_device" || reboot_with_msg "create pv failed: $pv_device" } # ensure the volume group $vg_name exists real_vg_name="$(lvm pvs | grep -w $vg_name | awk '{print $2}')" [ "$real_vg_name" = "$vg_name" ] || { - lvm vgcreate -y "$vg_name" "$pv_device" || reboot + lvm vgcreate -y "$vg_name" "$pv_device" || reboot_with_msg "create vg failed: $vg_name" } } - lvm vgs "$vg_name" || { - echo "warn dracut: FATAL: vg os not found, reboot" - reboot - } + lvm vgs "$vg_name" || reboot_with_msg "warn dracut: FATAL: vg os not found" # create logical volume src_lv_devname="$(basename $src_lv)" @@ -94,10 +94,7 @@ if [ -z "$use_root_partition" ]; then sync_src_lv "$src_lv" else src_lv="$use_root_partition" - [ -e "$src_lv" ] || { - echo "warn dracut: FATAL: no src_lv with local mount, reboot" - reboot - } + [ -e "$src_lv" ] || reboot_with_msg "warn dracut: FATAL: no src_lv with local mount" fi save_root_partition="$(getarg save_root_partition=)" -- 2.23.0

2 1

[PATCH v2 lkp-tests] add email feedback for rpmbuild-pkg
by Li Ping 29 Apr '21

29 Apr '21

tests/rpmbuild-pkg: add a new input parameter email [usage]: rpmbuild-pkg: whether_email: 1 when job failed, send an rpmbuild failed email. when job run success, send an rpmbuild success email. Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/email.sh | 39 +++++++++++++++++++++++++++++++++++++++ tests/rpmbuild-pkg | 15 +++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/lib/email.sh b/lib/email.sh index b5e56df79..e43f390ae 100755 --- a/lib/email.sh +++ b/lib/email.sh @@ -96,6 +96,45 @@ Compass-CI " } +rpmbuild_success_content() +{ + local author_email=$(git_pretty_format "%ae") + local author_name=$(git_pretty_format "%an") + email_content="To: $author_email +Subject: [NOTIFY Compass-CI] rpmbuild report + +Dear $author_name: + + Your RPM Package is successfully built. + + You can click the follow link to obtain your RPM Package: + http://api.compass-ci.openeuler.org:20012/rpm/pub/${os}-${os_version}/${com… + + And you can click the following link to view RPM Package build details: + http://api.compass-ci.openeuler.org:${SRV_HTTP_RESULT_PORT:-20007}$result_r… + +Regards +Compass-CI +" +} + +rpmbuild_failed_content() +{ + local author_email=$(git_pretty_format "%ae") + local author_name=$(git_pretty_format "%an") + email_content="To: $author_email +Subject: [NOTIFY Compass-CI] rpmbuild report + +Dear $author_name: + + Sorry to inform you that RPM Package built failed, you can click the following link to view details. + http://api.compass-ci.openeuler.org:${SRV_HTTP_RESULT_PORT:-20007}$result_r… + +Regards +Compass-CI +" +} + send_email() { local subject=$1 diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index 3dd75b896..970418848 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -5,6 +5,7 @@ # - repo_addr # - custom_repo_name # - custom_repo_addr +# - whether_email # Git repo jobs only use the first two parameters, SRPMs jobs use all # user can add custom_repo_name and custom_repo_addr to setup # local repo, value can be an array @@ -12,6 +13,7 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh . $LKP_SRC/lib/rpm.sh +. $LKP_SRC/lib/email.sh : "${compat_os:=budding-openeuler}" @@ -19,6 +21,13 @@ dest_dir="/rpm/upload/${os}-${os_version}/${compat_os}" +die() +{ + [ -n "$whether_email" ] && send_email "rpmbuild_failed" + echo "$@" 1>&2 + exit 99 +} + from_git() { package_name=${upstream_repo##*/} @@ -29,6 +38,11 @@ from_git() download_upstream_repo } +git_pretty_format() +{ + curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' -d '{"git_repo": "'${upstream_dir}/${upstream_repo}.git'", "git_command": ["git-log","-n1","--pretty=format:'$1'", "'${upstream_commit}'"]}' +} + from_srpm() { [ -n "$repo_name" ] || die "repo_name is empty" @@ -143,3 +157,4 @@ add_repo build_rpm upload_rpm_pkg update_repo_mq +[ -n "$whether_email" ] && send_email "rpmbuild_success" -- 2.23.0

2 1

[PATCH compass-ci] extract-stats: delete end with ".message" error id
by Cao Xueliang 29 Apr '21

29 Apr '21

we add the .message in error id, the error id can't be the bisect error id, so we delete it when select bisect error id. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/stats_worker.cr | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index f385fee..a244d65 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -137,6 +137,11 @@ class StatsWorker end error_ids.concat(content.as_h.keys) end - error_ids + + error_ids.each do |error_id| + error_ids.delete(error_id) if error_id.ends_with?(".message") + end + + return error_ids end end -- 2.23.0

1 0

[PATCH compass-ci] lib/git_mirror.rb: fix that no need to_json for log
by Li Yuanchao 28 Apr '21

28 Apr '21

When in ruby, log for kibana no need to_json with hash. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 71d6956..a25207a 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -563,7 +563,7 @@ class MirrorMain @log.info({ msg: 'new repo', repo: git_repo - }.to_json) + }) end def git_fail_log(git_repo, possible_new_refs) @@ -571,7 +571,7 @@ class MirrorMain @log.info({ msg: msg, repo: git_repo - }.to_json) + }) end def new_refs_log(git_repo, nr_new_branch) @@ -579,7 +579,7 @@ class MirrorMain msg: 'new refs', repo: git_repo, nr_new_branch: nr_new_branch - }.to_json) + }) end def last_commit_new?(git_repo) -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] sparrow/3-code: delete special symbols in vars values
by Liu Yinsi 28 Apr '21

28 Apr '21

[why] special symbols will be parsed in shell command "docker build -t --build-arg $PASSWORD ...", caused failed to build images, so delete. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index fe86b6cf..014221ee 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -190,7 +190,7 @@ auto-submit: EOF [[ -f /etc/compass-ci/passwd.yaml ]] || { -passwd=$(< /dev/urandom tr -dc "a-zA-Z0-9-=+@#_:/[](){}$%&.^*\!" | head -c 10;echo) +passwd=$(< /dev/urandom tr -dc "a-zA-Z0-9" | head -c 10; echo) cat > /etc/compass-ci/passwd.yaml <<EOF ES_USER: root -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] sparrow/1-storage: set permission /etc/compass-ci/password.yaml 660
by Liu Yinsi 28 Apr '21

28 Apr '21

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/1-storage/permission | 1 + sparrow/1-storage/tiny | 1 + 2 files changed, 2 insertions(+) diff --git a/sparrow/1-storage/permission b/sparrow/1-storage/permission index 3c229c5a..3a8bfdaa 100755 --- a/sparrow/1-storage/permission +++ b/sparrow/1-storage/permission @@ -44,5 +44,6 @@ chgrp team /srv/cci/serial/logs chgrp team /srv/dc chmod g+ws /srv/dc +chmod 660 /etc/compass-ci/password.yaml find /tftpboot -type d | xargs chmod 775 diff --git a/sparrow/1-storage/tiny b/sparrow/1-storage/tiny index c7636b02..1dbe8a32 100755 --- a/sparrow/1-storage/tiny +++ b/sparrow/1-storage/tiny @@ -22,6 +22,7 @@ dirs=( /c /c/git-repos /etc/qemu + /etc/compass-ci/password.yaml /srv/cci/Maildir/.compass-ci/new /srv/cci/Maildir/.compass-ci/cur /srv/cci/Maildir/.compass-ci/sent -- 2.23.0

1 0

[PATCH compass-ci] container: add build depends
by Liu Yinsi 28 Apr '21

28 Apr '21

delimiter's and updaterepo's base image is scheduler-dev. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/delimiter/build-depends | 1 + container/updaterepo/build-depends | 1 + 2 files changed, 2 insertions(+) create mode 100755 container/updaterepo/build-depends diff --git a/container/delimiter/build-depends b/container/delimiter/build-depends index 14bc599b..2ec2b37a 100755 --- a/container/delimiter/build-depends +++ b/container/delimiter/build-depends @@ -1 +1,2 @@ registry +scheduler-dev diff --git a/container/updaterepo/build-depends b/container/updaterepo/build-depends new file mode 100755 index 00000000..5dd403e1 --- /dev/null +++ b/container/updaterepo/build-depends @@ -0,0 +1 @@ +scheduler-dev -- 2.23.0

1 0

[PATCH lkp-tests] lib/bootstrap.sh: remove packages after installation
by Wei Jihui 28 Apr '21

28 Apr '21

[why] /opt/rpms is for save rpm package which the task depends. but if task is cci-depends, it mv $downloaded_rpms to /opt/rpms, then pack /opt/rpms/*.rpm, deps cgz will obtain useless rpm from the task depends. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 58f0531a7..417c8662a 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -411,6 +411,7 @@ install_rpms() [ -d /opt/rpms ] || return echo "install $(ls /opt/rpms/*.rpm)" rpm -ivh --force --ignoresize /opt/rpms/*.rpm + rm -f /opt/rpms/*.rpm } try_get_and_set_distro() -- 2.23.0

1 0

[PATCH lkp-tests] add email feedback for rpmbuild-pkg
by Li Ping 28 Apr '21

28 Apr '21

tests/rpmbuild-pkg: add a new input parameter email [usage]: rpmbuild-pkg: email: 1 when job failed, send an rpmbuild failed email. when job run success, send an rpmbuild success email. Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/email.sh | 41 +++++++++++++++++++++++++++++++++++++++++ tests/rpmbuild-pkg | 10 ++++++++++ 2 files changed, 51 insertions(+) diff --git a/lib/email.sh b/lib/email.sh index b5e56df79..613d9fd70 100755 --- a/lib/email.sh +++ b/lib/email.sh @@ -96,6 +96,47 @@ Compass-CI " } +git_pretty_format() +{ + curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' -d '{"git_repo": "'${upstream_dir}/${upstream_repo}.git'", "git_command": ["git-log","-n1","--pretty=format:'$1'", "'${upstream_commit}'"]}' +} + +rpmbuild_success_content() +{ + local author_email=$(git_pretty_format "%ae") + local author_name=$(git_pretty_format "%an") + email_content="To: $author_email +Subject: [NOTIFY Compass-CI] rpmbuild report + +Dear $author_name: + + Your RPM Package is successfully built. + + You can click the following link to view details: + http://api.compass-ci.openeuler.org:20012/rpm/pub/${os}-${os_version}/${com… + +Regards +Compass-CI +" +} + +rpmbuild_failed_content() +{ + local author_email=$(git_pretty_format "%ae") + local author_name=$(git_pretty_format "%an") + email_content="To: $author_email +Subject: [NOTIFY Compass-CI] rpmbuild report + +Dear $author_name: + + Sorry to inform you that RPM Package built failed, you can click the following link to view details. + http://api.compass-ci.openeuler.org:${SRV_HTTP_RESULT_PORT:-20007}$result_r… + +Regards +Compass-CI +" +} + send_email() { local subject=$1 diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index 3dd75b896..8edcb9d1a 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -5,6 +5,7 @@ # - repo_addr # - custom_repo_name # - custom_repo_addr +# - email # Git repo jobs only use the first two parameters, SRPMs jobs use all # user can add custom_repo_name and custom_repo_addr to setup # local repo, value can be an array @@ -12,6 +13,7 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh . $LKP_SRC/lib/rpm.sh +. $LKP_SRC/lib/email.sh : "${compat_os:=budding-openeuler}" @@ -19,6 +21,13 @@ dest_dir="/rpm/upload/${os}-${os_version}/${compat_os}" +die() +{ + [ -n "$email" ] && send_email "rpmbuild_failed" + echo "$@" 1>&2 + exit 99 +} + from_git() { package_name=${upstream_repo##*/} @@ -143,3 +152,4 @@ add_repo build_rpm upload_rpm_pkg update_repo_mq +[ -n "$email" ] && send_email "rpmbuild_success" -- 2.23.0

1 0

[PATCH v3 compass-ci 2/2] kibana/dashboard_import: import kibana dashboards
by Wu Zhende 28 Apr '21

28 Apr '21

Use this script can import kibana dashboards by json files. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana/dashboard_import | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 container/kibana/dashboard_import diff --git a/container/kibana/dashboard_import b/container/kibana/dashboard_import new file mode 100755 index 0000000..e20e8ff --- /dev/null +++ b/container/kibana/dashboard_import @@ -0,0 +1,40 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require 'pathname' +require_relative '../defconfig.rb' + +names = Set.new %w[ + KIBANA_HOST + KIBANA_PORT + ES_USER + ES_PASSWORD +] + +defaults = relevant_defaults(names) +KIBANA_HOST = defaults['KIBANA_HOST'] || '172.17.0.1' +KIBANA_PORT = defaults['KIBANA_PORT'] || '20017' + +service_authentication = relevant_service_authentication(names) + +files = ARGV +if files.empty? + profile = Pathname.new(File.dirname(__FILE__)).realpath + files << profile + 'data.json' + files << profile + 'resource.json' + files << profile + 'all.json' +end + +cmd = "curl -XPOST -u #{service_authentication['ES_USER']}:#{service_authentication['ES_PASSWORD']}" +cmd += " http://#{KIBANA_HOST}:#{KIBANA_PORT}/api/kibana/dashboards/import" +cmd += " -H 'Content-Type: application/json' -H 'kbn-xsrf: reporting'" + +files.each do |f| + new_cmd = cmd + new_cmd += " -d @#{f}" + puts new_cmd + system new_cmd +end -- 2.23.0

2 1

[PATCH compass-ci] scheduler-https/start: fix start error
by Wu Zhende 28 Apr '21

28 Apr '21

Missing ENV key: "NODE_PORT" Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/scheduler-https/start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/scheduler-https/start b/container/scheduler-https/start index 5e79d2b..59e7278 100755 --- a/container/scheduler-https/start +++ b/container/scheduler-https/start @@ -69,6 +69,7 @@ cmd = %W[ ] + env + %W[ -e LKP_SRC=#{DEFAULT_LKP} -e CCI_SRC=#{DEFAULT_CCI} + -e NODE_PORT=#{SCHED_HTTPS_PORT} -v #{ENV['LKP_SRC']}:#{DEFAULT_LKP} -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} -v #{CCI_REPOS}/lab-#{LAB}:#{CCI_REPOS}/lab-#{LAB} -- 2.23.0

1 0

[PATCH v3 compass-ci 1/2] kibana/dashboard_export: exporting kibana dashboards
by Wu Zhende 28 Apr '21

28 Apr '21

Use this script can export kibana dashboards by ids. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana/dashboard_export | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 container/kibana/dashboard_export diff --git a/container/kibana/dashboard_export b/container/kibana/dashboard_export new file mode 100755 index 0000000..1532aae --- /dev/null +++ b/container/kibana/dashboard_export @@ -0,0 +1,33 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig.rb' + +names = Set.new %w[ + KIBANA_HOST + KIBANA_PORT + ES_USER + ES_PASSWORD +] + +defaults = relevant_defaults(names) +KIBANA_HOST = defaults['KIBANA_HOST'] || '172.17.0.1' +KIBANA_PORT = defaults['KIBANA_PORT'] || '20017' + +service_authentication = relevant_service_authentication(names) + +dashboard_ids = ARGV +dashboard_ids = %w[data resource all] if dashboard_ids.empty? + +cmd = "curl -u #{service_authentication['ES_USER']}:#{service_authentication['ES_PASSWORD']}" +cmd += " http://#{KIBANA_HOST}:#{KIBANA_PORT}/api/kibana/dashboards/export?dashboard=" + +dashboard_ids.each do |id| + new_cmd = cmd + new_cmd += "#{id} >> #{id}.json" + puts new_cmd + system new_cmd +end -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] kibana/dashboard_import: import kibana dashboards
by Wu Zhende 28 Apr '21

28 Apr '21

Use this script can import kibana dashboards by json files. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana/dashboard_import | 44 +++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 container/kibana/dashboard_import diff --git a/container/kibana/dashboard_import b/container/kibana/dashboard_import new file mode 100755 index 0000000..a6129a5 --- /dev/null +++ b/container/kibana/dashboard_import @@ -0,0 +1,44 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require 'pathname' +require_relative '../defconfig.rb' + +names = Set.new %w[ + KIBANA_HOST + KIBANA_PORT + ES_USER + ES_PASSWORD +] + +defaults = relevant_defaults(names) +KIBANA_HOST = defaults['KIBANA_HOST'] || '172.17.0.1' +KIBANA_PORT = defaults['KIBANA_PORT'] || '20017' + +service_authentication = relevant_service_authentication(names) + +files = [] +ARGV.each do |f| + files >> f +end + +if files.empty? + profile = Pathname.new(File.dirname(__FILE__)).realpath + files << profile + 'data.json' + files << profile + 'resource.json' + files << profile + 'all.json' +end + +cmd = "curl -XPOST -u #{service_authentication['ES_USER']}:#{service_authentication['ES_PASSWORD']}" +cmd += " http://#{KIBANA_HOST}:#{KIBANA_PORT}/api/kibana/dashboards/import" +cmd += " -H 'Content-Type: application/json' -H 'kbn-xsrf: reporting'" + +files.each do |f| + new_cmd = cmd + new_cmd += " -d @#{f}" + puts new_cmd + system new_cmd +end -- 2.23.0

2 2

[PATCH v2 compass-ci 1/2] kibana/dashboard_export: exporting kibana dashboards
by Wu Zhende 28 Apr '21

28 Apr '21

Use this script can export kibana dashboards by ids. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana/dashboard_export | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 container/kibana/dashboard_export diff --git a/container/kibana/dashboard_export b/container/kibana/dashboard_export new file mode 100755 index 0000000..d6e0054 --- /dev/null +++ b/container/kibana/dashboard_export @@ -0,0 +1,37 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig.rb' + +names = Set.new %w[ + KIBANA_HOST + KIBANA_PORT + ES_USER + ES_PASSWORD +] + +defaults = relevant_defaults(names) +KIBANA_HOST = defaults['KIBANA_HOST'] || '172.17.0.1' +KIBANA_PORT = defaults['KIBANA_PORT'] || '20017' + +service_authentication = relevant_service_authentication(names) + +dashboard_ids = [] +ARGV.each do |i| + dashboard_ids << i +end + +dashboard_ids = %w[data resource all] if dashboard_ids.empty? + +cmd = "curl -u #{service_authentication['ES_USER']}:#{service_authentication['ES_PASSWORD']}" +cmd += " http://#{KIBANA_HOST}:#{KIBANA_PORT}/api/kibana/dashboards/export?dashboard=" + +dashboard_ids.each do |id| + new_cmd = cmd + new_cmd += "#{id} >> #{id}.json" + puts new_cmd + system new_cmd +end -- 2.23.0

2 1

[PATCH lkp-tests] tests/rpmbuild-pkg: change the way obtaining spec file
by Li Ping 28 Apr '21

28 Apr '21

[why] tests/rpmbuild-pkg: package_name=${upstream_repo##*/} when the package name is inconsistent with the repo name, it will cause the error like: + grep http -rl /root/rpmbuild/SPECS/zstd-test.spec grep: /root/rpmbuild/SPECS/zstd-test.spec: No such file or directory + spectool -g -R /root/rpmbuild/SPECS/zstd-test.spec Can't open '/root/rpmbuild/SPECS/zstd-test.spec': No such file or directory at /usr/bin/spectool line 314. Signed-off-by: Li Ping <1477412247(a)qq.com> --- tests/rpmbuild-pkg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index 5c694b9d4..3dd75b896 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -49,7 +49,7 @@ init_workspace() download_upstream_repo() { - local url="git://${GIT_SERVER}/openeuler/${upstream_repo}" + local url="git://${GIT_SERVER}/${upstream_dir}/${upstream_repo}" git clone "$url" || die "clone git repo ${package_name} failed: $url" cd "$package_name" || exit @@ -75,14 +75,14 @@ build_rpm() [ -n "$package_name" ] && { # HTTP is proxy cache friendly - sed -i 's/^$Source[^ ]*:[ \t]*$https/\1http/g' "$(grep http -rl "$spec_dir/${package_name}.spec")" + sed -i 's/^$Source[^ ]*:[ \t]*$https/\1http/g' "$(grep http -rl "$spec_dir/"*.spec)" } # Install build depends yum-builddep -y "$spec_dir"/*.spec || die "failed to solve dependencies" # Download tar.gz to default path ${HOME}/rpmbuild/SOURCE [ -n "$package_name" ] && { - spectool -g -R "$spec_dir/${package_name}.spec" || die "failed to download source file" + spectool -g -R "$spec_dir/"*.spec || die "failed to download source file" } # Building rpm or srpm packages rpmbuild -ba "$spec_dir"/*.spec || die "failed to build rpms" -- 2.23.0

1 0

[PATCH v6 compass-ci] src/lib/web_backend.rb: filter git_repos
by Li Yuanchao 28 Apr '21

28 Apr '21

To make sure git_repos displayed on the web are right. examples of git_repo: a/AvxToNeon/AvxToNeon 0/caterqiu/001 aur-a/a2png/a2png m/my-sql_with_boostkit_patch/my-sql_with_boostkit_patch Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 9e899d9..e305ac6 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -400,6 +400,14 @@ def query_repos(must, from: 0, size: 1) repos = [] result['hits']['hits'].each do |r| r = r['_source'] + + # What does this regular expression want: + # 1. a three-segment structure "xxx/xxx/xxx" + # 2. there can be '-' or '_' in every segment, but can't be at first or last of the segment. + # 3. the first segment can have lowercase of letters or numbers in it. + # 4. the other two segments can have letters(lowercase or uppercase) or numbers in it. + next unless r['git_repo'] =~ %r{^([a-z0-9]([a-z0-9\-_]*[a-z0-9])*(/[a-zA-Z0-9][\w\-]*[a-zA-Z0-9]){2})$} + repos << { git_url: get_repo_url(r['url']), git_repo: r['git_repo'] -- 2.23.0

1 0

[PATCH lkp-tests] jobs/send-email.yaml: change the default value.
by Li Ping 28 Apr '21

28 Apr '21

the default value of email subject should not be job_failed, and there is already a send-email-on-fail.yaml. Signed-off-by: Li Ping <1477412247(a)qq.com> --- jobs/send-email.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/send-email.yaml b/jobs/send-email.yaml index 8f8b48b96..2689634c2 100644 --- a/jobs/send-email.yaml +++ b/jobs/send-email.yaml @@ -3,4 +3,4 @@ # values of $subject are limited to job_failed/job_debug/borrow_success/borrow_failed. send_email: - subject: job_failed + subject: -- 2.23.0

1 0

[PATCH compass-ci 2/2] kibana/dashboard_import: import kibana dashboards
by Wu Zhende 28 Apr '21

28 Apr '21

Use this script can import kibana dashboards by json files. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana/dashboard_import | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 container/kibana/dashboard_import diff --git a/container/kibana/dashboard_import b/container/kibana/dashboard_import new file mode 100755 index 0000000..670fa3f --- /dev/null +++ b/container/kibana/dashboard_import @@ -0,0 +1,45 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require 'pathname' +require_relative '../defconfig.rb' + +names = Set.new %w[ + KIBANA_HOST + KIBANA_PORT + ES_USER + ES_PASSWORD +] + +defaults = relevant_defaults(names) +KIBANA_HOST = defaults['KIBANA_HOST'] || '172.17.0.1' +KIBANA_PORT = defaults['KIBANA_PORT'] || '20017' + +service_authentication = relevant_service_authentication(names) + +files = [] +ARGV.each do |f| + files >> f +end + +if files.empty? + profile = Pathname.new(File.dirname(__FILE__)).realpath + files << profile + 'data.json' + files << profile + 'resource.json' + files << profile + 'all.json' +end +puts files + +cmd = "curl -XPOST -u #{service_authentication['ES_USER']}:#{service_authentication['ES_PASSWORD']}" +cmd += " http://#{KIBANA_HOST}:#{KIBANA_PORT}/api/kibana/dashboards/import" +cmd += " -H 'Content-Type: application/json' -H 'kbn-xsrf: reporting'" + +files.each do |f| + new_cmd = cmd + new_cmd += " -d @#{f}" + puts new_cmd + system new_cmd +end -- 2.23.0

2 1

[PATCH compass-ci] src/lib/web_backend.rb: modify parameter list for web compare
by Li Yuanchao 28 Apr '21

28 Apr '21

The parameter list for web compare need to change, as tbox_groups have $user at the back. Suite and tbox_group whose count < 2 need not to show, cause have no one to compare. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index da3c3c5..a9ab091 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -45,16 +45,11 @@ def es_count(query) ES_CLIENT.count(index: 'jobs*', body: query)['count'] end -# "vm-2p8g--212" remove "--212" -# "vm-2p8g--zzz" remove "--zzz" -# "vm-git-bisect" don't remove "-bisect" +# delete $user after '--' or '.' or '~' def filter_tbox_group(es_result) result = Set.new es_result.each do |r| - if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-(([a-zA-Z]+)|(\d+))$)/ - index = r.index('--') || r.rindex('-') - r = r[0, index] - end + r = r.gsub(/(--|\.|~).*$/, '') result.add r end result.to_a @@ -70,10 +65,10 @@ def all_suite size: 0 } es_result = es_query(query)['aggregations']['all_suite']['buckets'] - es_result.sort_by! { |h| h['doc_count'] } - es_result.reverse!.map! { |x| x['key'] } - - es_result + result_second = [] + es_result.each { |e| result_second << e if e['doc_count'] > 1 } + result_second.sort_by! { |h| h['doc_count'] } + result_second.reverse!.map! { |x| x['key'] } end def all_tbox_group @@ -86,10 +81,12 @@ def all_tbox_group size: 0 } es_result = es_query(query)['aggregations']['all_tbox_group']['buckets'] - es_result.sort_by! { |h| h['doc_count'] } - es_result.reverse!.map! { |x| x['key'] } + result_second = [] + es_result.each { |e| result_second << e if e['doc_count'] > 1 } + result_second.sort_by! { |h| h['doc_count'] } + result_second.reverse!.map! { |x| x['key'] } - filter_tbox_group(es_result) + filter_tbox_group(result_second) end def compare_candidates_body -- 2.23.0

2 1

[PATCH v2 compass-ci] sbin/es-jobs-mapping.sh: add field
by Lu Weitao 28 Apr '21

28 Apr '21

add nr_run(integer), author_email(keyword) Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- sbin/es-jobs-mapping.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sbin/es-jobs-mapping.sh b/sbin/es-jobs-mapping.sh index 67b1ed14..0c2fc62d 100755 --- a/sbin/es-jobs-mapping.sh +++ b/sbin/es-jobs-mapping.sh @@ -148,6 +148,12 @@ else "upstream_commit": { "type": "keyword" }, + "base_commit": { + "type": "keyword" + }, + "nr_run": { + "type": "integer" + }, "my_email": { "type": "keyword" }, -- 2.23.0

2 2

[PATCH compass-ci 1/2] kibana/dashboard_export: exporting kibana dashboards
by Wu Zhende 28 Apr '21

28 Apr '21

Use this script can export kibana dashboards by ids. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana/dashboard_export | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 container/kibana/dashboard_export diff --git a/container/kibana/dashboard_export b/container/kibana/dashboard_export new file mode 100755 index 0000000..d6e0054 --- /dev/null +++ b/container/kibana/dashboard_export @@ -0,0 +1,37 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig.rb' + +names = Set.new %w[ + KIBANA_HOST + KIBANA_PORT + ES_USER + ES_PASSWORD +] + +defaults = relevant_defaults(names) +KIBANA_HOST = defaults['KIBANA_HOST'] || '172.17.0.1' +KIBANA_PORT = defaults['KIBANA_PORT'] || '20017' + +service_authentication = relevant_service_authentication(names) + +dashboard_ids = [] +ARGV.each do |i| + dashboard_ids << i +end + +dashboard_ids = %w[data resource all] if dashboard_ids.empty? + +cmd = "curl -u #{service_authentication['ES_USER']}:#{service_authentication['ES_PASSWORD']}" +cmd += " http://#{KIBANA_HOST}:#{KIBANA_PORT}/api/kibana/dashboards/export?dashboard=" + +dashboard_ids.each do |id| + new_cmd = cmd + new_cmd += "#{id} >> #{id}.json" + puts new_cmd + system new_cmd +end -- 2.23.0

1 0

[PATCH v2 lkp-tests 3/3] iso2rootfs: some optimize
by Yu Chuan 27 Apr '21

27 Apr '21

1. change dailybuild result rootfs dir name why: - make the dailybuild rootfs directory more meaningful. how: - before: /srv/os/${iso_os}/${iso_arch}/${iso_version}-${TIMESTAMP} - after : /srv/os/${iso_os}/${iso_arch}/${iso_version}-dailybuild-${TIMESTAMP} 2. change sha256sum file why: - unify the file name, then easier to statistics. how: - file name: - before: ${ROOTFS_DES_DIR}/${iso_version}-latest.sha256sum - after : ${ROOTFS_DES_DIR}/SHA256SUM Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/iso2rootfs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/iso2rootfs b/tests/iso2rootfs index 06c8565110ab..ca972b2399f6 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -318,6 +318,8 @@ download_compass_ci() config_rootfs_dir() { ROOTFS_DES_DIR=${ROOTFS_LOCAL_PATH}/${iso_version}-$(date "+%Y-%m-%d-%H-%M-%S") + [ -n "${dailybuild_iso_url_file}" ] && ROOTFS_DES_DIR=${ROOTFS_LOCAL_PATH}/${iso_version}-dailybuild-$(date "+%Y-%m-%d-%H-%M-%S") + [ -d "${ROOTFS_DES_DIR}" ] && ROOTFS_DES_DIR="${ROOTFS_DES_DIR}-${HOSTNAME##*--}" mkdir -p "$ROOTFS_DES_DIR" @@ -592,7 +594,7 @@ update_dailybuild_soft_link() [ -n "${dailybuild_iso_url_file}" ] || return 0 mv "${CHECKSUM_FILE_CACHE}.tmp" "${CHECKSUM_FILE_CACHE}" - cp "${CHECKSUM_FILE_CACHE}" "${ROOTFS_DES_DIR}" + cp "${CHECKSUM_FILE_CACHE}" "${ROOTFS_DES_DIR}/SHA256SUM" local soft_link="${iso_version}-dailybuild" cd "${ROOTFS_LOCAL_PATH}" && -- 2.23.0

1 0

[PATCH v2 lkp-tests 2/3] install-iso: fix some bugs
by Yu Chuan 27 Apr '21

27 Apr '21

[What Bugs] 1. optimize name: reboot.cgz => reboot-after-systemd-logind.cgz 2. nfs mount result dir: - before: ${os}/${os_arch}/${os_version}-iso-${ROOTFS_SUFFIX} - after : ${iso_os}/${iso_arch}/${iso_version}-iso-${ROOTFS_SUFFIX} 3. exit directory before umount Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 6 +++--- tests/install-iso-sync-rootfs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index c51b3784b5aa..481b99e35df4 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -196,8 +196,8 @@ submit_install_iso_firstboot_yaml() custom_ipxe="#!ipxe\\\\n" custom_ipxe="${custom_ipxe}\\\\ndhcp" custom_ipxe="${custom_ipxe}\\\\ninitrd http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/${iso_os}/${iso_arch}/${iso_version}/initrd.lkp" - custom_ipxe="${custom_ipxe}\\\\ninitrd http://${INITRD_HTTP_HOST}:${INITRD_HTTP_PORT}/initrd/custom_bootstrap/reboot.cgz" - custom_ipxe="${custom_ipxe}\\\\nkernel http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/${iso_os}/${iso_arch}/${iso_version}/boot/vmlinuz ip=dhcp root=/dev/mapper/os-${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX} initrd=initrd.lkp initrd=reboot.cgz local rw" + custom_ipxe="${custom_ipxe}\\\\ninitrd http://${INITRD_HTTP_HOST}:${INITRD_HTTP_PORT}/initrd/custom_bootstrap/reboot-after-systemd-logind.cgz" + custom_ipxe="${custom_ipxe}\\\\nkernel http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/${iso_os}/${iso_arch}/${iso_version}/boot/vmlinuz ip=dhcp root=/dev/mapper/os-${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX} initrd=initrd.lkp initrd=reboot-after-systemd-logind.cgz local rw" custom_ipxe="${custom_ipxe}\\\\nboot\\\\n" sed -i "/custom_ipxe:/ccustom_ipxe: \"${custom_ipxe}\"" $firstboot_yaml @@ -225,7 +225,7 @@ submit_install_iso_sync_rootfs_yaml() install-iso-sync-rootfs.NFS_ROOT_DIR=${NFS_ROOT_DIR} \ install-iso-sync-rootfs.ROOTFS_SUFFIX=${ROOTFS_SUFFIX} \ install-iso-sync-rootfs.VG_NAME=${VG_NAME} \ - testbox=$testbox || die "submit ${sync_rootfs_yaml} failed" + testbox=$testbox || die "submit job failed: ${sync_rootfs_yaml}" } main() diff --git a/tests/install-iso-sync-rootfs b/tests/install-iso-sync-rootfs index 34096229ccfb..b74c7bed2044 100755 --- a/tests/install-iso-sync-rootfs +++ b/tests/install-iso-sync-rootfs @@ -59,7 +59,7 @@ sync_rootfs_to_nfs_server() local nfs_rootfs_mount_point=/tmp/nfs_rootfs_${ROOTFS_SUFFIX} mkdir -p ${nfs_rootfs_mount_point} - mount -t nfs ${NFS_SERVER_HOST}:${NFS_ROOT_DIR}/${os}/${os_arch}/${os_version}-iso-${ROOTFS_SUFFIX} ${nfs_rootfs_mount_point} || die "mount nfs rootfs failed" + mount -t nfs ${NFS_SERVER_HOST}:${NFS_ROOT_DIR}/${iso_os}/${iso_arch}/${iso_version}-iso-${ROOTFS_SUFFIX} ${nfs_rootfs_mount_point} || die "mount nfs rootfs failed" # activate the lvm vgchange -ay @@ -78,7 +78,7 @@ sync_rootfs_to_nfs_server() cd ${nfs_rootfs_mount_point} && mkdir -p "${SKIP_SYNC_DIRS[@]}" # umount nfs rootfs - umount $nfs_rootfs_mount_point || log_warn "umount failed: nfs_rootfs_mount_point" + cd / && umount $nfs_rootfs_mount_point || log_warn "umount failed: nfs_rootfs_mount_point" # umount local logical volume umount $lv_rootfs_mount_point || log_warn "umount failed: $lv_rootfs_mount_point" -- 2.23.0

1 0

[PATCH v2 lkp-tests 1/3] install-iso-pre: change the logic of ks generate and upload
by Yu Chuan 27 Apr '21

27 Apr '21

[Why] 1. 'key difference' of ks for different physical machine: - Choice Install Source - Choice Install Partition - Choice working Nic 2. 'other ks content' for different os is different. so the change is as following: before: 'other ks content' is all same. after : add a httpd dir, which stores the 'other ks content' for different os, then they only need generate the 'key difference' related to current physical machine, 'other ks content' can download from httpd. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 36 ++---------------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index c89895a994cc..c51b3784b5aa 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -112,41 +112,9 @@ generate_ks() bootloader --location=none logvol / --vgname=${VG_NAME} --name=${LV_NAME} --useexisting - # Root password - rootpw --iscrypted ${secrets_ROOTFS_ROOT_PASSWD} - - # Reboot after installation - reboot --eject - - # Use text mode install - text - - # Keyboard layouts - keyboard --vckeymap=us --xlayouts='us' - - # System language - lang en_US.UTF-8 - - # Do not configure the X Window System - skipx - - # System services - services --enabled="chronyd" - - # System timezone - timezone Asia/Shanghai --isUtc - - %packages - @^minimal-environment - - %end - - %anaconda - pwpolicy root --minlen=8 --minquality=1 --notstrict --nochanges --notempty - pwpolicy user --minlen=8 --minquality=1 --notstrict --nochanges --emptyok - pwpolicy luks --minlen=8 --minquality=1 --notstrict --nochanges --notempty - %end EOF + + curl -s http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/install/ks/template/${iso_os}-${iso_arch}-${iso_version}.ks >> $ks_file } upload_ks() -- 2.23.0

1 0

[PATCH compass-ci] lib/mail_bisect_result: use assistant client send account mail
by Cao Xueliang 27 Apr '21

27 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/mail_bisect_result.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/mail_bisect_result.rb b/lib/mail_bisect_result.rb index 38828a3..1112f3b 100644 --- a/lib/mail_bisect_result.rb +++ b/lib/mail_bisect_result.rb @@ -84,12 +84,7 @@ class MailBisectResult 'my_name' => @git_commit.author_name, 'my_commit_url' => "#{@git_commit.url}/commit/#{@commit_id}" } - - account_info = ESQuery.new(index: 'accounts').query_by_id(@to) - return if account_info - - apply_account = AutoAssignAccount.new(user_info) - apply_account.send_account + AssistantClient.new.send_account_mail(user_info) end def rm_work_dir -- 2.23.0

1 0

[PATCH lkp-tests] lib/email.sh: rename $job_state
by Liu Yinsi 27 Apr '21

27 Apr '21

job_state => report_content See [PATCH lkp-tests 1/3] tests/self-test: send mail when failed to submit job Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- lib/email.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/email.sh b/lib/email.sh index 24ef7bb64..b5e56df79 100755 --- a/lib/email.sh +++ b/lib/email.sh @@ -89,7 +89,7 @@ Dear $my_name: Your self-test groud_id=selftest job result is: - $job_state + $report_content Regards Compass-CI -- 2.23.0

1 0

[PATCH compass-ci] container/assistant: add send_account_mail function
by Cao Xueliang 27 Apr '21

27 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assistant/routes.rb | 12 +++++++++++ .../assistant/views/send_account_mail.rb | 20 +++++++++++++++++++ lib/assign_account_client.rb | 13 +++++++----- lib/assistant_client.rb | 8 ++++++++ 4 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 container/assistant/views/send_account_mail.rb diff --git a/container/assistant/routes.rb b/container/assistant/routes.rb index 232ec8c..9a96fc0 100755 --- a/container/assistant/routes.rb +++ b/container/assistant/routes.rb @@ -9,6 +9,7 @@ require 'open3' require_relative './views/locate_files' require_relative './views/get_mail_list' +require_relative './views/send_account_mail' configure do set :bind, '0.0.0.0' @@ -37,3 +38,14 @@ get '/get_mail_list/:type' do return [200, result.to_json] end + +post '/send_account_mail' do + begin + data = JSON.parse(request.body.read) + result = send_account_mail(data) + rescue StandardError => e + return [400, e.backtrace.inspect] + end + + return [200, result.to_json] +end diff --git a/container/assistant/views/send_account_mail.rb b/container/assistant/views/send_account_mail.rb new file mode 100644 index 0000000..b96d082 --- /dev/null +++ b/container/assistant/views/send_account_mail.rb @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# # frozen_string_literal: true +# +require_relative "#{ENV['CCI_SRC']}/lib/es_query" +require_relative "#{ENV['CCI_SRC']}/lib/assign_account_client" + +def send_account_mail(user_info) + account_info = ESQuery.new(index: 'accounts').query_by_id(user_info['my_email']) + return {'status'=> true} if account_info + + begin + apply_account = AutoAssignAccount.new(user_info) + apply_account.send_account + rescue + return {'status' => false} + end + + return {'status' => true} +end diff --git a/lib/assign_account_client.rb b/lib/assign_account_client.rb index f551a9d..5aad955 100644 --- a/lib/assign_account_client.rb +++ b/lib/assign_account_client.rb @@ -4,13 +4,13 @@ # frozen_string_literal: true require 'json' -require 'mail' require 'set' require 'optparse' require 'rest-client' require_relative '../container/defconfig' require_relative 'es_client' require_relative '../container/mail-robot/lib/assign-account-email' +require_relative "#{ENV['CCI_SRC']}/lib/parse_mail_list" names = Set.new %w[ JUMPER_HOST @@ -32,6 +32,7 @@ class AutoAssignAccount @my_info = user_info @my_info_es = {} + @account_info = {} end def update_from_es @@ -59,10 +60,10 @@ class AutoAssignAccount end def update_my_info_from_account_info - account_info = apply_account + @account_info = apply_account - @my_info['my_login_name'] = account_info['my_login_name'] - @my_info['my_ssh_pubkey'] << account_info['my_jumper_pubkey'] unless account_info['my_jumper_pubkey'].nil? + @my_info['my_login_name'] = @account_info['my_login_name'] + @my_info['my_ssh_pubkey'] << @account_info['my_jumper_pubkey'] unless @account_info['my_jumper_pubkey'].nil? end def store_account_info @@ -72,7 +73,9 @@ class AutoAssignAccount def send_mail @my_info['bisect'] = true - message = build_apply_account_email(@my_info) + default_email = parse_mail_list('delimiter') + @my_info['my_email'] = default_email['to'] unless default_email.empty? + message = build_apply_account_email(@my_info, @account_info, false) %x(curl -XPOST "#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text" -d "#{message}") end diff --git a/lib/assistant_client.rb b/lib/assistant_client.rb index c951f2b..6774e9e 100644 --- a/lib/assistant_client.rb +++ b/lib/assistant_client.rb @@ -21,4 +21,12 @@ class AssistantClient return JSON.parse(response.body) end + + def send_account_mail(user_info) + resource = RestClient::Resource.new("http://#{@host}:#{@port}/send_account_mail") + response = resource.post(user_info.to_json) + return nil unless response.code == 200 + + return JSON.parse(response.body) + end end -- 2.23.0

1 0

[PATCH lkp-tests 3/3] jobs: remove multi-docker/qemu
by Liu Yinsi 27 Apr '21

27 Apr '21

multi-qemu and multi-docker will deploy a simply ci env by calling compass-ci/sparrow/install-client, but deploy-cci will deploy a ci env by calling compass-ci/sparrow/install-tiny, they are 2 defferent envs, so remove multi-docker/qemu. testboxes for docker and qemu will be ready by sparrow/8-testbox, 8-testbox already be called by sparrow/install-tiny. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- jobs/self-test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jobs/self-test.yaml b/jobs/self-test.yaml index ad2dcd2a2..b0c1c08c3 100644 --- a/jobs/self-test.yaml +++ b/jobs/self-test.yaml @@ -2,5 +2,4 @@ suite: self-test category: functional deploy-cci: -multi-qemu: -multi-docker: +self-test: -- 2.23.0

1 0

[PATCH lkp-tests 2/3] tests/self-test: fix command not found
by Liu Yinsi 27 Apr '21

27 Apr '21

[error] /lkp/lkp/src/tests/self-test: line 14: submit: command not found /lkp/lkp/src/tests/self-test: line 14: submit: command not found /lkp/lkp/src/tests/self-test: line 14: submit: command not found /lkp/lkp/src/tests/self-test: line 24: es-jobs: command not found Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- tests/self-test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/self-test b/tests/self-test index 6e4ca7ef2..fb3e87b17 100755 --- a/tests/self-test +++ b/tests/self-test @@ -3,6 +3,8 @@ . $LKP_SRC/lib/email.sh . $LKP_SRC/lib/log.sh +source /etc/profile.d/compass.sh + submit_args=( "testbox=dc-8g cci-depends.benchmark=sshd cci-depends.yaml" "testbox=dc-8g borrow-1h.yaml runtime=1m" -- 2.23.0

1 0

[PATCH lkp-tests 1/3] tests/self-test: send mail when failed to submit job
by Liu Yinsi 27 Apr '21

27 Apr '21

When failed to deploy ci env, some services not ready like es, sched, can't query anything from ES, and email content $job_state is null, so send email when failed to submit job. function submit_one() is redundant, so delete. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- tests/self-test | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/self-test b/tests/self-test index cb06b7226..6e4ca7ef2 100755 --- a/tests/self-test +++ b/tests/self-test @@ -1,6 +1,7 @@ #!/bin/bash . $LKP_SRC/lib/email.sh +. $LKP_SRC/lib/log.sh submit_args=( "testbox=dc-8g cci-depends.benchmark=sshd cci-depends.yaml" @@ -8,21 +9,22 @@ submit_args=( "testbox=vm-2p8g borrow-1h.yaml runtime=1m" ) -submit_one() -{ - args="$1" - submit_result=$(submit -m $args group_id=selftest) -} - -submit_all() +submit_job() { for args in "${submit_args[@]}" do - submit_one "$args" + report_content=$(submit -m $args group_id=selftest 2>&1) + echo $report_content | grep -q "\"job_state\":\"submit\"" || + { + log_error "submit $args failed: $report_content" + send_email selftest + exit 1 + } done - job_state=$(es-jobs group_id=selftest | grep job_state) + report_content=$(es-jobs group_id=selftest | grep job_state) + log_info "selftest job result: $report_content" } -submit_all +submit_job send_email selftest -- 2.23.0

1 0

[PATCH lkp-tests] iso2rootfs: some optimize
by Yu Chuan 27 Apr '21

27 Apr '21

1. change dailybuild result rootfs dir name why: - make the dailybuild rootfs directory more meaningful. how: - before: /srv/os/${iso_os}/${iso_arch}/${iso_version}-${TIMESTAMP} - after : /srv/os/${iso_os}/${iso_arch}/${iso_version}-dailybuild-${TIMESTAMP} 2. change sha256sum file why: - unify the file name, then easier to statistics. how: - file name: - before: ${ROOTFS_DES_DIR}/${iso_version}-latest.sha256sum - after : ${ROOTFS_DES_DIR}/iso.sha256sum - file content: - before: 168d2a8c24d20acdf06e67ffee90174b321b52e36afbddfc41602454feb2c09c openEuler-20.03-LTS-aarch64-dvd.iso - after : 168d2a8c24d20acdf06e67ffee90174b321b52e36afbddfc41602454feb2c09c Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/iso2rootfs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/iso2rootfs b/tests/iso2rootfs index 06c8565110ab..e8694a73e807 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -318,6 +318,8 @@ download_compass_ci() config_rootfs_dir() { ROOTFS_DES_DIR=${ROOTFS_LOCAL_PATH}/${iso_version}-$(date "+%Y-%m-%d-%H-%M-%S") + [ -n "${dailybuild_iso_url_file}" ] && ROOTFS_DES_DIR=${ROOTFS_LOCAL_PATH}/${iso_version}-dailybuild-$(date "+%Y-%m-%d-%H-%M-%S") + [ -d "${ROOTFS_DES_DIR}" ] && ROOTFS_DES_DIR="${ROOTFS_DES_DIR}-${HOSTNAME##*--}" mkdir -p "$ROOTFS_DES_DIR" @@ -592,7 +594,7 @@ update_dailybuild_soft_link() [ -n "${dailybuild_iso_url_file}" ] || return 0 mv "${CHECKSUM_FILE_CACHE}.tmp" "${CHECKSUM_FILE_CACHE}" - cp "${CHECKSUM_FILE_CACHE}" "${ROOTFS_DES_DIR}" + echo $SHA256SUM_NET > "${ROOTFS_DES_DIR}/iso.sha256sum" local soft_link="${iso_version}-dailybuild" cd "${ROOTFS_LOCAL_PATH}" && -- 2.23.0

2 2

[PATCH lkp-tests 2/3] install-iso-pre: change the logic of ks generate and upload
by Yu Chuan 27 Apr '21

27 Apr '21

[Why] 1. 'key difference' of ks for different physical machine: - http install source. - install target disk/partition. - working nic. 2. 'other ks content' for different os is different. so the change is as following: before: 'other ks content' is all same. after : add a httpd dir, which stores the 'other ks content' for different os, then they only need generate the 'key difference' related to current physical machine, 'other ks content' can download from httpd. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 43 +++++++++---------------------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index c89895a994cc..f9ff9ebe9e94 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -91,6 +91,13 @@ prepare_lv() create_lv } +append_ks_template() +{ + wget -O ${ks_file}.tmp http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/install/ks/template/${iso_os}-${iso_arch}-${iso_version}.ks + cat ${ks_file}.tmp >> $ks_file + rm -rf ${ks_file}.tmp +} + generate_ks() { log_info "start generate $ks_file" @@ -112,41 +119,9 @@ generate_ks() bootloader --location=none logvol / --vgname=${VG_NAME} --name=${LV_NAME} --useexisting - # Root password - rootpw --iscrypted ${secrets_ROOTFS_ROOT_PASSWD} - - # Reboot after installation - reboot --eject - - # Use text mode install - text - - # Keyboard layouts - keyboard --vckeymap=us --xlayouts='us' - - # System language - lang en_US.UTF-8 - - # Do not configure the X Window System - skipx - - # System services - services --enabled="chronyd" - - # System timezone - timezone Asia/Shanghai --isUtc - - %packages - @^minimal-environment - - %end - - %anaconda - pwpolicy root --minlen=8 --minquality=1 --notstrict --nochanges --notempty - pwpolicy user --minlen=8 --minquality=1 --notstrict --nochanges --emptyok - pwpolicy luks --minlen=8 --minquality=1 --notstrict --nochanges --notempty - %end EOF + + append_ks_template } upload_ks() -- 2.23.0

2 2

[PATCH v2 compass-ci] src/lib/web_backend.rb: show OS by order on web
by Li Yuanchao 27 Apr '21

27 Apr '21

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 576b922..da3c3c5 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -97,10 +97,10 @@ def compare_candidates_body query_conditions: { suite: all_suite, OS: [ - { os: 'openeuler', os_version: ['1.0', '20.03'] }, + { os: 'archlinux', os_version: ['5.5.0-1'] }, { os: 'centos', os_version: ['7.6', '7.8', '8.1'] }, { os: 'debian', os_version: %w[10 sid] }, - { os: 'archlinux', os_version: ['5.5.0-1'] } + { os: 'openeuler', os_version: ['1.0', '20.03'] } ], os_arch: %w[aarch64 x86_64], tbox_group: all_tbox_group -- 2.23.0

1 0

[PATCH lkp-tests 3/3] install-iso: fix some bugs
by Yu Chuan 27 Apr '21

27 Apr '21

[What Bugs] 1. optimize name: reboot.cgz => reboot-after-sshd.cgz 2. nfs mount result dir: - before: ${os}/${os_arch}/${os_version}-iso-${ROOTFS_SUFFIX} - after : ${iso_os}/${iso_arch}/${iso_version}-iso-${ROOTFS_SUFFIX} 3. exit directory before umount Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 6 +++--- tests/install-iso-sync-rootfs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index f9ff9ebe9e94..16639e437d71 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -203,8 +203,8 @@ submit_install_iso_firstboot_yaml() custom_ipxe="#!ipxe\\\\n" custom_ipxe="${custom_ipxe}\\\\ndhcp" custom_ipxe="${custom_ipxe}\\\\ninitrd http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/${iso_os}/${iso_arch}/${iso_version}/initrd.lkp" - custom_ipxe="${custom_ipxe}\\\\ninitrd http://${INITRD_HTTP_HOST}:${INITRD_HTTP_PORT}/initrd/custom_bootstrap/reboot.cgz" - custom_ipxe="${custom_ipxe}\\\\nkernel http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/${iso_os}/${iso_arch}/${iso_version}/boot/vmlinuz ip=dhcp root=/dev/mapper/os-${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX} initrd=initrd.lkp initrd=reboot.cgz local rw" + custom_ipxe="${custom_ipxe}\\\\ninitrd http://${INITRD_HTTP_HOST}:${INITRD_HTTP_PORT}/initrd/custom_bootstrap/reboot-after-sshd.cgz" + custom_ipxe="${custom_ipxe}\\\\nkernel http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/${iso_os}/${iso_arch}/${iso_version}/boot/vmlinuz ip=dhcp root=/dev/mapper/os-${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX} initrd=initrd.lkp initrd=reboot-after-sshd.cgz local rw" custom_ipxe="${custom_ipxe}\\\\nboot\\\\n" sed -i "/custom_ipxe:/ccustom_ipxe: \"${custom_ipxe}\"" $firstboot_yaml @@ -232,7 +232,7 @@ submit_install_iso_sync_rootfs_yaml() install-iso-sync-rootfs.NFS_ROOT_DIR=${NFS_ROOT_DIR} \ install-iso-sync-rootfs.ROOTFS_SUFFIX=${ROOTFS_SUFFIX} \ install-iso-sync-rootfs.VG_NAME=${VG_NAME} \ - testbox=$testbox || die "submit ${sync_rootfs_yaml} failed" + testbox=$testbox || die "submit job failed: ${sync_rootfs_yaml}" } main() diff --git a/tests/install-iso-sync-rootfs b/tests/install-iso-sync-rootfs index 34096229ccfb..b74c7bed2044 100755 --- a/tests/install-iso-sync-rootfs +++ b/tests/install-iso-sync-rootfs @@ -59,7 +59,7 @@ sync_rootfs_to_nfs_server() local nfs_rootfs_mount_point=/tmp/nfs_rootfs_${ROOTFS_SUFFIX} mkdir -p ${nfs_rootfs_mount_point} - mount -t nfs ${NFS_SERVER_HOST}:${NFS_ROOT_DIR}/${os}/${os_arch}/${os_version}-iso-${ROOTFS_SUFFIX} ${nfs_rootfs_mount_point} || die "mount nfs rootfs failed" + mount -t nfs ${NFS_SERVER_HOST}:${NFS_ROOT_DIR}/${iso_os}/${iso_arch}/${iso_version}-iso-${ROOTFS_SUFFIX} ${nfs_rootfs_mount_point} || die "mount nfs rootfs failed" # activate the lvm vgchange -ay @@ -78,7 +78,7 @@ sync_rootfs_to_nfs_server() cd ${nfs_rootfs_mount_point} && mkdir -p "${SKIP_SYNC_DIRS[@]}" # umount nfs rootfs - umount $nfs_rootfs_mount_point || log_warn "umount failed: nfs_rootfs_mount_point" + cd / && umount $nfs_rootfs_mount_point || log_warn "umount failed: nfs_rootfs_mount_point" # umount local logical volume umount $lv_rootfs_mount_point || log_warn "umount failed: $lv_rootfs_mount_point" -- 2.23.0

2 1

[PATCH v5 compass-ci] src/lib/web_backend.rb: filter git_repos
by Li Yuanchao 27 Apr '21

27 Apr '21

To make sure git_repos displayed on the web are right. examples: a/AvxToNeon/AvxToNeon aur-a/a2png/a2png m/my-sql_with_boostkit_patch/my-sql_with_boostkit_patch Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index cbf6e5c..0c1f4fa 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -403,6 +403,8 @@ def query_repos(must, from: 0, size: 1) repos = [] result['hits']['hits'].each do |r| r = r['_source'] + next unless r['git_repo'] =~ %r{^([a-z0-9]([a-z0-9\-_]*[a-z0-9])*(/[a-zA-Z0-9][\w\-]*[a-zA-Z0-9]){2})$} + repos << { git_url: get_repo_url(r['url']), git_repo: r['git_repo'] -- 2.23.0

1 0

[PATCH compass-ci] src/lib/web_backend.rb: show OS by order on web
by Li Yuanchao 27 Apr '21

27 Apr '21

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index dde64c0..03db8f6 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -97,10 +97,10 @@ def compare_candidates_body query_conditions: { suite: all_suite, OS: [ - { os: 'openeuler', os_version: ['1.0', '20.03'] }, + { os: 'archlinux', os_version: ['5.5.0-1'] } { os: 'centos', os_version: ['7.6', '7.8', '8.1'] }, { os: 'debian', os_version: %w[10 sid] }, - { os: 'archlinux', os_version: ['5.5.0-1'] } + { os: 'openeuler', os_version: ['1.0', '20.03'] }, ], os_arch: %w[aarch64 x86_64], tbox_group: all_tbox_group -- 2.23.0

1 0

[PATCH v4 compass-ci] src/lib/web_backend.rb: filter git_repos
by Li Yuanchao 27 Apr '21

27 Apr '21

To make sure git_repos displayed on the web are right. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 91287b4..dde64c0 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -403,6 +403,8 @@ def query_repos(must, from: 0, size: 1) repos = [] result['hits']['hits'].each do |r| r = r['_source'] + next unless r['git_repo'] =~ %r{^([a-z0-9]([a-z0-9\-_]*[a-z0-9])*(/[a-z0-9][a-z0-9\-_]*[a-z0-9]){2})$} + repos << { git_url: get_repo_url(r['url']), git_repo: r['git_repo'] -- 2.23.0

1 0

[PATCH v3 compass-ci] src/lib/web_backend.rb: filter git_repos
by Li Yuanchao 27 Apr '21

27 Apr '21

To make sure git_repos displayed on the web are right. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 91287b4..0bfc6c9 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -403,6 +403,8 @@ def query_repos(must, from: 0, size: 1) repos = [] result['hits']['hits'].each do |r| r = r['_source'] + next unless r['git_repo'] =~ /^([a-z0-9]([a-z0-9\-\_]*[a-z0-9])*(\/[a-z0-9][a-z0-9\-\_]*[a-z0-9]){2})$/ + repos << { git_url: get_repo_url(r['url']), git_repo: r['git_repo'] -- 2.23.0

1 0

[PATCH compass-ci 2/2] sparrow/3-code: delete special symbols in vars values
by Liu Yinsi 27 Apr '21

27 Apr '21

[why] special symbols will be parsed in shell command "docker build -t --build-arg $PASSWORD ...", caused failed to build images, so delete. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index b68d9e67..15d9bc45 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -190,7 +190,7 @@ auto-submit: EOF [[ -f /etc/compass-ci/passwd.yaml ]] || { -passwd=$(< /dev/urandom tr -dc "a-zA-Z0-9-=+@#_:/[](){}$%&.^*\!" | head -c 10;echo) +passwd=$(< /dev/urandom tr -dc "a-zA-Z0-9" | head -c 10; echo) cat > /etc/compass-ci/passwd.yaml <<EOF ES_USER: root -- 2.23.0

2 2

[PATCH compass-ci 1/2] sparrow/1-storage: set permission /etc/compass-ci/password.yaml 660
by Liu Yinsi 27 Apr '21

27 Apr '21

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/1-storage/permission | 1 + sparrow/1-storage/tiny | 1 + 2 files changed, 2 insertions(+) diff --git a/sparrow/1-storage/permission b/sparrow/1-storage/permission index 3c229c5a..3a8bfdaa 100755 --- a/sparrow/1-storage/permission +++ b/sparrow/1-storage/permission @@ -44,5 +44,6 @@ chgrp team /srv/cci/serial/logs chgrp team /srv/dc chmod g+ws /srv/dc +chmod 660 /etc/compass-ci/password.yaml find /tftpboot -type d | xargs chmod 775 diff --git a/sparrow/1-storage/tiny b/sparrow/1-storage/tiny index c7636b02..1dbe8a32 100755 --- a/sparrow/1-storage/tiny +++ b/sparrow/1-storage/tiny @@ -22,6 +22,7 @@ dirs=( /c /c/git-repos /etc/qemu + /etc/compass-ci/password.yaml /srv/cci/Maildir/.compass-ci/new /srv/cci/Maildir/.compass-ci/cur /srv/cci/Maildir/.compass-ci/sent -- 2.23.0

2 2

[PATCH v2 compass-ci] src/lib/web_backend.rb: filter git_repos
by Li Yuanchao 27 Apr '21

27 Apr '21

To make sure git_repos displayed on the web are right. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 91287b4..066b781 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -403,6 +403,8 @@ def query_repos(must, from: 0, size: 1) repos = [] result['hits']['hits'].each do |r| r = r['_source'] + next unless r['git_repo'] =~ /^([a-z0-9])/ + repos << { git_url: get_repo_url(r['url']), git_repo: r['git_repo'] -- 2.23.0

1 0

[PATCH compass-ci] src/lib/web_backend.rb: filter git_repos
by Li Yuanchao 27 Apr '21

27 Apr '21

To make sure git_repos displayed on the web are right. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 91287b4..452e94a 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -403,6 +403,8 @@ def query_repos(must, from: 0, size: 1) repos = [] result['hits']['hits'].each do |r| r = r['_source'] + next unless r['git_repo'] =~ /^([a-z]\/)/ + repos << { git_url: get_repo_url(r['url']), git_repo: r['git_repo'] -- 2.23.0

1 0