- Compass-ci - mailweb.openeuler.org

[PATCH v5 compass-ci] docker2rootfs: add unzip vmlinux function
by Wang Chenglong 20 Jan '21

20 Jan '21

[Why] Unpacked vmlinux into vmlinuz format. Signeo-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker2rootfs/common | 34 ++++++++++++++++++++++++++++- container/docker2rootfs/run | 1 + container/docker2rootfs/setup-image | 2 +- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/container/docker2rootfs/common b/container/docker2rootfs/common index c26bbc8..52ae0c2 100755 --- a/container/docker2rootfs/common +++ b/container/docker2rootfs/common @@ -5,6 +5,8 @@ # This file is called by script run running on host # define functions and set host environment variables +. $CCI_SRC/container/qcow2rootfs/bin/common + get_host_aarch() { aarch=$(arch) @@ -133,7 +135,7 @@ get_rootfs_kernel() local vmlinuz_file local vmlinuz kernel - vmlinuz_file=$(find ./boot -name "vmlinuz-*" | grep -v rescue) && export ROOTFS_VMLINUZ_FILE=$vmlinuz_file + vmlinuz_file=$(find ./boot -name "vmlinu[z|x]-*" | grep -v rescue) && export ROOTFS_VMLINUZ_FILE=$vmlinuz_file vmlinuz=$(basename "$vmlinuz_file") && export ROOTFS_VMLINUZ=$vmlinuz kernel=${vmlinuz:8} && export ROOTFS_KERNEL=$kernel @@ -155,3 +157,33 @@ create_links_vmlinuz_initrd() echo "[INFO] Create links to initrd.lkp and vmlinuz success!" } + +unzip_vmlinuz() { + get_rootfs_kernel + + file ${ROOTFS_VMLINUZ_FILE} | grep -q gzip || return + + get_vmlinuz_start_postion ${ROOTFS_VMLINUZ_FILE} + [ $VMLINUZ_START_POSTION -eq -1 ] && return + + local unzip_str + + if [ $VMLINUZ_START_POSTION -eq 0 ]; then + unzip_str="dd if=$ROOTFS_VMLINUZ bs=1 | zcat > $ROOTFS_VMLINUZ.tmp" + else + unzip_str="dd if=$ROOTFS_VMLINUZ bs=1 skip=$VMLINUZ_START_POSTION | zcat > $ROOTFS_VMLINUZ.tmp" + fi + + echo "Unzipping vmlinuz..." + cd "$ROOTFS_DIR/boot" + $unzip_str + + if [ $? -eq 0 ]; then + echo "[INFO] Unzipping vmlinuz success!" + else + echo "[WARNING] Unzipping vmlinuz failed!" + fi + + mv $ROOTFS_VMLINUZ.tmp $ROOTFS_VMLINUZ + chmod o+r $ROOTFS_VMLINUZ +} diff --git a/container/docker2rootfs/run b/container/docker2rootfs/run index 8a7b315..cdb19bf 100755 --- a/container/docker2rootfs/run +++ b/container/docker2rootfs/run @@ -31,3 +31,4 @@ start=( cp_rootfs $IMAGE_PACK $2 docker rm -f rootfs-docker create_links_vmlinuz_initrd $2 +unzip_vmlinuz diff --git a/container/docker2rootfs/setup-image b/container/docker2rootfs/setup-image index ffe4937..47bfb7e 100755 --- a/container/docker2rootfs/setup-image +++ b/container/docker2rootfs/setup-image @@ -38,7 +38,7 @@ setup_login() pack_cgz() { echo "Packing package. Please wait." - find ./ ! -path "./tmp/${IMAGE_PACK}" | cpio -o -Hnewc | gzip -9 >./tmp/$IMAGE_PACK + find ./ ! -path "./tmp/$IMAGE_PACK" | cpio -o -Hnewc | gzip -9 > "./tmp/$IMAGE_PACK" chmod 644 /tmp/${IMAGE_PACK} } -- 2.23.0

1 1

[PATCH v3 compass-ci 2/2] my-qemu.sh: add `-h|--help`, `-d|--debug` params for my-qemu.sh
by Yu Chuan 20 Jan '21

20 Jan '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- providers/my-qemu.sh | 35 +++++++++++++++++++++++++++++++++++ providers/qemu/kvm.sh | 13 +++++++------ providers/qemu/pxe.sh | 10 ++++++++-- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/providers/my-qemu.sh b/providers/my-qemu.sh index 8845a346e195..fc98ea97360d 100755 --- a/providers/my-qemu.sh +++ b/providers/my-qemu.sh @@ -2,6 +2,41 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +: ${LKP_SRC:="/c/lkp-tests"} + +source ${LKP_SRC}/lib/log.sh + +help() +{ + cat <<-'EOF' + Usage: ./my-qemu.sh [OPTION]... + Ask scheduler service whether there is a job, and start a local vm if there is a job. + Scheduler api: http://${SCHED_HOST}:${SCHED_PORT}/boot.ipxe/mac/${mac} + + Mandatory arguments to long options are mandatory for short options too. + -h, --help display this help and exit + -d, --debug open the local vm serial port in the current shell + EOF + + exit 0 +} + +while true +do + [ $# -eq 0 ] && break + case "$1" in + -h|--help) + help;; + -d|--debug) + export DEBUG=true;; + *) + log_error "Unknown param: $1" + help + exit 1;; + esac + shift +done + [[ $tbox_group ]] || tbox_group=vm-2p8g export hostname=$tbox_group.$USER-$$ diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index 93977ec15438..ec8c00887283 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -122,11 +122,11 @@ set_qemu() print_message() { - echo SCHED_PORT: $SCHED_PORT - echo kernel: $kernel - echo initrds: $initrds - echo append: $append - echo less $log_file + log_info SCHED_PORT: $SCHED_PORT + log_info kernel: $kernel + log_info initrds: $initrds + log_info append: $append + [ "$DEBUG" == "true" ] || log_info less $log_file sleep 5 } @@ -143,9 +143,10 @@ public_option() -k en-us -no-reboot -nographic - -serial file:${log_file} -monitor null ) + + [ "$DEBUG" == "true" ] || kvm=("${kvm[@]}" -serial file:${log_file}) } individual_option() diff --git a/providers/qemu/pxe.sh b/providers/qemu/pxe.sh index fa688331964a..0ba052c8276e 100755 --- a/providers/qemu/pxe.sh +++ b/providers/qemu/pxe.sh @@ -4,6 +4,10 @@ # - nr_cpu # - memory +: ${LKP_SRC:="/c/lkp-tests"} + +source ${LKP_SRC}/lib/log.sh + : ${nr_cpu:=1} : ${memory:=1G} @@ -18,7 +22,7 @@ fi qemu=qemu-system-aarch64 command -v $qemu >/dev/null || qemu=qemu-kvm -echo less $serial_log +[ "$DEBUG" == "true" ] || log_info less $serial_log kvm=( $qemu @@ -33,7 +37,9 @@ kvm=( -k en-us -no-reboot -nographic - -serial file:${serial_log} -monitor null ) + +[ "$DEBUG" == "true" ] || kvm=("${kvm[@]}" -serial file:${serial_log}) + "${kvm[@]}" -- 2.23.0

4 5

[PATCH v4 compass-ci] docker2rootfs: add unzip vmlinux function
by Wang Chenglong 20 Jan '21

20 Jan '21

[Why] Unpacked vmlinux into vmlinuz format. Signeo-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker2rootfs/common | 33 ++++++++++++++++++++++++++++- container/docker2rootfs/run | 1 + container/docker2rootfs/setup-image | 2 +- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/container/docker2rootfs/common b/container/docker2rootfs/common index c26bbc8..e8c56ba 100755 --- a/container/docker2rootfs/common +++ b/container/docker2rootfs/common @@ -5,6 +5,8 @@ # This file is called by script run running on host # define functions and set host environment variables +. $CCI_SRC/container/qcow2rootfs/bin/common + get_host_aarch() { aarch=$(arch) @@ -133,7 +135,7 @@ get_rootfs_kernel() local vmlinuz_file local vmlinuz kernel - vmlinuz_file=$(find ./boot -name "vmlinuz-*" | grep -v rescue) && export ROOTFS_VMLINUZ_FILE=$vmlinuz_file + vmlinuz_file=$(find ./boot -name "vmlinu[z|x]-*" | grep -v rescue) && export ROOTFS_VMLINUZ_FILE=$vmlinuz_file vmlinuz=$(basename "$vmlinuz_file") && export ROOTFS_VMLINUZ=$vmlinuz kernel=${vmlinuz:8} && export ROOTFS_KERNEL=$kernel @@ -155,3 +157,32 @@ create_links_vmlinuz_initrd() echo "[INFO] Create links to initrd.lkp and vmlinuz success!" } + +unzip_vmlinuz() { + get_rootfs_kernel + + file ${ROOTFS_VMLINUZ_FILE} | grep -q gzip || return + + get_vmlinuz_start_postion ${ROOTFS_VMLINUZ_FILE} + [ $VMLINUZ_START_POSTION -eq -1 ] && return + + local unzip_str + + if [ $VMLINUZ_START_POSTION -eq 0 ]; then + unzip_str="dd if=$ROOTFS_VMLINUZ bs=1 | zcat > $ROOTFS_VMLINUZ.tmp" + else + unzip_str="dd if=$ROOTFS_VMLINUZ bs=1 skip=$VMLINUZ_START_POSTION | zcat > $ROOTFS_VMLINUZ.tmp" + fi + + echo "Unzipping vmlinuz..." + cd "$ROOTFS_DIR/boot" + $unzip_str + mv $ROOTFS_VMLINUZ.tmp $ROOTFS_VMLINUZ + chmod o+r $ROOTFS_VMLINUZ + + if [ $? -eq 0 ]; then + echo "[INFO] Unzipping vmlinuz success!" + else + echo "[WARNING] Unzipping vmlinuz failed!" + fi +} diff --git a/container/docker2rootfs/run b/container/docker2rootfs/run index 8a7b315..cdb19bf 100755 --- a/container/docker2rootfs/run +++ b/container/docker2rootfs/run @@ -31,3 +31,4 @@ start=( cp_rootfs $IMAGE_PACK $2 docker rm -f rootfs-docker create_links_vmlinuz_initrd $2 +unzip_vmlinuz diff --git a/container/docker2rootfs/setup-image b/container/docker2rootfs/setup-image index ffe4937..47bfb7e 100755 --- a/container/docker2rootfs/setup-image +++ b/container/docker2rootfs/setup-image @@ -38,7 +38,7 @@ setup_login() pack_cgz() { echo "Packing package. Please wait." - find ./ ! -path "./tmp/${IMAGE_PACK}" | cpio -o -Hnewc | gzip -9 >./tmp/$IMAGE_PACK + find ./ ! -path "./tmp/$IMAGE_PACK" | cpio -o -Hnewc | gzip -9 > "./tmp/$IMAGE_PACK" chmod 644 /tmp/${IMAGE_PACK} } -- 2.23.0

2 4

[PATCH lkp-tests] tests/build-pkg: fix to download *.tar packages failure
by Liu Shaofei 19 Jan '21

19 Jan '21

[why] curl: (56) OpenSSL SSL_read: Connection reset by peer, errno 104 ==> ERROR: Failure while downloading buddy-2.4.tar.gz Aborting... Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- tests/build-pkg | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/build-pkg b/tests/build-pkg index 31762384d..7c65f26bc 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -127,7 +127,6 @@ replace_source() } } } - url=$(echo "$url" | sed 's|https://|http://|g') echo \'$url\' >> $1 done -- 2.23.0

2 1

[PATCH v5 compass-ci] lib/compare_error_messages.rb: add the credibility check of job
by Lin Jiaxin 19 Jan '21

19 Jan '21

defect: may match the wrong result error_id="build-pkg.pixz.c:warning:'subsuf'defined-but-not-used[-Wunused-function]" filename="pixz.c" correct result: gcc -DHAVE_CONFIG_H -I. -I.. -D_FORTIFY_SOURCE=2 -pthread -Wall -Wno-unknown-pragmas -march=native -O2 -pipe -fstack-protector-strong -fno-plt -Wl,-O1, --sort-common,--as-needed,-z,relro,-z,now -MT pixz-pixz.o -MD -MP -MF .deps/pixz-pixz.Tpo -c -o pixz-pixz.o `test -f 'pixz.c' || echo './'`pixz.c ^^^^^^ wrong result: mv -f .deps/pixz.cpu.Tpo .deps/pixz.cpu.Po ^^^^^^ Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- lib/compare_error_messages.rb | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/lib/compare_error_messages.rb b/lib/compare_error_messages.rb index ca4f03a..92533b8 100644 --- a/lib/compare_error_messages.rb +++ b/lib/compare_error_messages.rb @@ -8,6 +8,40 @@ require_relative 'es_query' require_relative 'error_messages' require_relative "#{ENV['LKP_SRC']}/lib/common" +# previous_job_id's error_ids include error_id +# If later_job_id's output include error filename that extract from error_id, +# the later_job_id is credible. +def credible?(previous_job_id, later_job_id, error_id) + es = ESQuery.new + + previous_result_file = File.join('/srv', es.query_by_id(previous_job_id)['result_root'], 'build-pkg') + later_result_file = File.join('/srv', es.query_by_id(later_job_id)['result_root'], 'build-pkg') + + return false if filenames_check(previous_result_file, later_result_file, error_id).value?(false) + + return true +end + +def filenames_check(previous_result_file, later_result_file, error_id) + filenames = Set.new + filenames_check = Hash.new { |h, k| h[k] = false } + + error_lines = ErrorMessages.new(previous_result_file).obtain_error_messages_by_error_id(error_id, true) + error_lines.each do |error_line| + # "src/ssl_sock.c:1454:104: warning: unused parameter 'al' [-Wunused-parameter]" => "src/ssl_sock.c" + filenames << $1 if error_line =~ /(.*)(:\d+){2}: (error|warning):/ + end + + File.open(later_result_file).each_line do |line| + filenames.each do |filename| + filenames_check[filename] + filenames_check[filename] = true if line.include?(filename) + end + end + + return filenames_check +end + def get_compare_result(previous_job_id, later_job_id) es = ESQuery.new -- 2.23.0

1 0

[PATCH compass-ci] container/delimiter: replace gem sources and delete useless packages
by Cao Xueliang 19 Jan '21

19 Jan '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/delimiter/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container/delimiter/Dockerfile b/container/delimiter/Dockerfile index 6ebd937..cc57320 100644 --- a/container/delimiter/Dockerfile +++ b/container/delimiter/Dockerfile @@ -9,5 +9,5 @@ RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweiclou 'g++' 'gcc' 'pcre' 'libevent' 'make' 'git' 'bash' 'grep' 'coreutils' 'curl' 'util-linux' RUN umask 002 && \ - echo ':sources: ["http://rubygems.org"]' >> ~/.gemrc && \ - gem install rest-client activesupport git json yaml threadpool elasticsearch faye-websocket terminal-table mail + gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ + gem install rest-client activesupport git json yaml elasticsearch faye-websocket terminal-table mail io-console -- 2.23.0

1 0

[PATCH v4 compass-ci 2/2] compare_error_messages.rb/error_messages.rb: optimize string catenate
by Lin Jiaxin 19 Jan '21

19 Jan '21

Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- lib/compare_error_messages.rb | 2 +- lib/error_messages.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compare_error_messages.rb b/lib/compare_error_messages.rb index d9171bf..237b2c9 100644 --- a/lib/compare_error_messages.rb +++ b/lib/compare_error_messages.rb @@ -73,7 +73,7 @@ def format_error_messages(error_messages, new_error_ids) new_error_number = 0 error_messages.each do |k, v| - if new_error_ids.include?('build-pkg.' + build_pkg_error_id(k)) + if new_error_ids.include?("build-pkg.#{build_pkg_error_id(k)}") new_error_number += 1 formatted_error_messages = add_sign(formatted_error_messages, '>>', v) else diff --git a/lib/error_messages.rb b/lib/error_messages.rb index d83d890..50f9637 100644 --- a/lib/error_messages.rb +++ b/lib/error_messages.rb @@ -44,7 +44,7 @@ class ErrorMessages error_messages_by_error_id = [] error_messages = obtain_error_messages error_messages.each do |k, v| - if ('build-pkg.' + build_pkg_error_id(k)) == error_id + if "build-pkg.#{build_pkg_error_id(k)}" == error_id error_messages_by_error_id += return_error_line ? [k] : v.to_a end end -- 2.23.0

1 0

[PATCH v4 compass-ci 1/2] lib/compare_error_messages.rb: add the credibility check of job
by Lin Jiaxin 19 Jan '21

19 Jan '21

defect: may match the wrong result error_id="build-pkg.pixz.c:warning:'subsuf'defined-but-not-used[-Wunused-function]" filename="pixz.c" correct result: gcc -DHAVE_CONFIG_H -I. -I.. -D_FORTIFY_SOURCE=2 -pthread -Wall -Wno-unknown-pragmas -march=native -O2 -pipe -fstack-protector-strong -fno-plt -Wl,-O1, --sort-common,--as-needed,-z,relro,-z,now -MT pixz-pixz.o -MD -MP -MF .deps/pixz-pixz.Tpo -c -o pixz-pixz.o `test -f 'pixz.c' || echo './'`pixz.c ^^^^^^ wrong result: mv -f .deps/pixz.cpu.Tpo .deps/pixz.cpu.Po ^^^^^^ Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- lib/compare_error_messages.rb | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/lib/compare_error_messages.rb b/lib/compare_error_messages.rb index 0b51fc3..d9171bf 100644 --- a/lib/compare_error_messages.rb +++ b/lib/compare_error_messages.rb @@ -8,6 +8,40 @@ require_relative 'es_query' require_relative 'error_messages' require_relative "#{ENV['LKP_SRC']}/lib/common" +# previous_job_id's error_ids include error_id +# If later_job_id's output include error filename that extract from error_id, +# the later_job_id is credible. +def credible?(previous_job_id, later_job_id, error_id) + es = ESQuery.new + + previous_result_file = File.join('/srv', es.query_by_id(previous_job_id)['result_root'], 'build-pkg') + later_result_file = File.join('/srv', es.query_by_id(later_job_id)['result_root'], 'build-pkg') + + return false if filenames_check(previous_result_file, later_result_file, error_id).value?(false) + + return true +end + +def filenames_check(previous_result_file, later_result_file, error_id) + filenames = Set.new + filenames_check = Hash.new { |h, k| h[k] = false } + + error_lines = ErrorMessages.new(previous_result_file).obtain_error_messages_by_error_id(error_id, true) + error_lines.each do |error_line| + # "src/ssl_sock.c:1454:104: warning: unused parameter ��al�� [-Wunused-parameter]" => "src/ssl_sock.c" + filenames << $1 if error_line =~ /(.*)(:\d+){2}: (error|warning):/ + end + + File.open(later_result_file).each_line do |line| + filenames.each do |filename| + filenames_check[filename] + filenames_check[filename] = true if line.include?(filename) + end + end + + return filenames_check +end + def get_compare_result(previous_job_id, later_job_id) es = ESQuery.new -- 2.23.0

1 0

[PATCH v4 compass-ci] container/delimiter: replace gem sources and delete useless packages
by Cao Xueliang 19 Jan '21

19 Jan '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/delimiter/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container/delimiter/Dockerfile b/container/delimiter/Dockerfile index c4e5074..cc57320 100644 --- a/container/delimiter/Dockerfile +++ b/container/delimiter/Dockerfile @@ -9,5 +9,5 @@ RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweiclou 'g++' 'gcc' 'pcre' 'libevent' 'make' 'git' 'bash' 'grep' 'coreutils' 'curl' 'util-linux' RUN umask 002 && \ - echo ':sources: ["http://rubygems.org"]' >> ~/.gemrc && \ - gem install rest-client activesupport git json yaml threadpool elasticsearch faye-websocket terminal-table mail io-console + gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ + gem install rest-client activesupport git json yaml elasticsearch faye-websocket terminal-table mail io-console -- 2.23.0

1 0

[PATCH lkp-tests] lib/job-init.sh: delete unused operation
by Xu Xijian 19 Jan '21

19 Jan '21

[ -z "$os_mount" ] has been checked in first branch of if block, here is redundant. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- lib/job-init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/job-init.sh b/lib/job-init.sh index dc4a65d91..1fb898872 100755 --- a/lib/job-init.sh +++ b/lib/job-init.sh @@ -58,7 +58,7 @@ setup_result_service() is_docker || [ -n "$id" ] || { if [ "$os_mount" = 'nfs' ] || [ -z "$os_mount" ]; then supports_netfs 'nfs' && result_service=$RESULT_SERVER:/result && return - elif [ "$os_mount" = 'cifs' ] || [ -z "$os_mount" ]; then + elif [ "$os_mount" = 'cifs' ]; then supports_netfs 'cifs' && result_service=//$RESULT_SERVER/result && return fi } -- 2.23.0

1 0

[PATCH v8 compass-ci] lib/es_jobs.rb: optimize es-jobs output
by Lu Kaiyi 19 Jan '21

19 Jan '21

[why] Reduce the nesting of results and output yaml [example] es-jobs os=archlinux [output] kvcount.arch=aarch64: 43 kvcount.arch=x86_64: 2 kvcount.category=benchmark: 12 kvcount.category=functional: 33 kvcount.job_state=failed: 20 kvcount.job_state=finished: 25 kvcount.os=archlinux: 45 kvcount.suite=cci-depends: 28 kvcount.suite=cci-makepkg: 5 kvcount.suite=iperf: 3 kvcount.suite=iperf-walk-os-test: 9 kvcount.summary.any_error=1: 24 kvcount.summary.any_fail=1: 25 kvcount.summary.any_stderr=1: 44 kvcount.summary.any_warning=1: 23 kvcount.summary.success=1: 1 kvcount.tbox_group=vm-2p16g--wangyong: 1 kvcount.tbox_group=vm-2p16g--wcl: 1 kvcount.tbox_group=vm-2p16g.wangyong: 34 kvcount.tbox_group=vm-2p8g: 9 ... raw.id.[job_state=failed]: ["crystal.122405", "crystal.122406", "crystal.122331", "crystal.122365", "crystal.122404", "crystal.122438", "crystal.122607", "crystal.122370", "crystal.122375", "crystal.122410", "crystal.122900", "crystal.122906", "crystal.122271", "crystal.122451", "crystal.122835", "crystal.122834", "crystal.122232", "crystal.122403", "crystal.122453", "crystal.122886"] ... raw.id.[summary.any_error=1]: ["crystal.122436", "crystal.122476", "crystal.122803", "crystal.122883", "crystal.122910", "crystal.122438", "crystal.122452", "crystal.122895", "crystal.122446", "crystal.122685", "crystal.122687", "crystal.122833", "crystal.122451", "crystal.122690", "crystal.122719", "crystal.122882", "crystal.122890", "crystal.122453", "crystal.122443", "crystal.122435", "crystal.122432", "crystal.122688", "crystal.122689", "crystal.122886"] ... sum.stats.stderr./lkp/lkp/src/monitors/perf-stat:#:main: 12 sum.stats.stderr.Can_not_find_perf_command: 12 sum.stats.stderr./lkp/lkp/src/tests/wrapper:line#:which:command_not_found: 10 ... raw.stats.sched_debug.cfs_rq:/.load.stddev: [524288.0, null, null, null, null, null, null, null, null, 524288.0, 524288.0, null, null, null, null, null, 516608.0, null, 1048576.0, null, null, null, null, null, null, null, 524288.0, null, null, null, null, 2104832.0, 1572864.0, null, null, 2097152.0, null, null, null, null, null, null, null, null, null] ... avg.stats.sched_debug.cfs_rq:/.load.stddev: 1048576.0 avg.stats.softirqs.CPU1.NET_RX: 2.5833333333333335 avg.stats.slabinfo.kmalloc-512.active_objs: 1372.75 ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_jobs.rb | 198 +++++++++++++++++++++++++++---------------------- 1 file changed, 108 insertions(+), 90 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..40b3f74 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -3,137 +3,155 @@ LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' +KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo summary.success + summary.any_fail summary.any_error summary.any_stderr summary.any_warning].freeze + +require 'yaml' require "#{LKP_SRC}/lib/stats" require_relative './es_query' # deal jobs search from es class ESJobs def initialize(es_query, my_refine = [], fields = [], stats_filter = []) - @es_query = es_query - @es = ESQuery.new(ES_HOST, ES_PORT) + @jobs = query_jobs_from_es(es_query) @refine = my_refine @fields = fields @stats_filter = stats_filter - @stats_filter_result = {} @refine_jobs = [] - @jobs = {} - @stats_level = { - 0 => 'stats.success', - 1 => 'stats.unknown', - 2 => 'stats.warning', - 3 => 'stats.has_error' - } - set_defaults - deal_jobs + set_jobs_summary end - def set_defaults - query_result = @es.multi_field_query(@es_query) - query_result['hits']['hits'].each do |job| - @jobs[job['_id']] = job['_source'] - end - - @stats = { - 'stats.count' => Hash.new(0), - 'stats.sum' => Hash.new(0), - 'stats.avg' => Hash.new(0) - } - @result = {} - @fields.each do |field| - @result[field] = [] - end + def query_jobs_from_es(items) + es = ESQuery.new(ES_HOST, ES_PORT) + result = es.multi_field_query items + jobs = result['hits']['hits'] + jobs.map! { |job| job['_source'] } + return jobs end - def add_result_fields(job, level) - return unless @refine.include?(level) || @refine.include?(-1) + # set jobs summary fields information in place + def set_jobs_summary + @jobs.each do |job| + stats = job['stats'] + next unless stats - @refine_jobs << job['id'] - @fields.each do |field| - value = job[field] - if value - value = job['id'] + '.' + value if field == 'job_state' - @result[field] << value + summary_result = '' + if stats.keys.any? { |stat| stat.match(/warn/i) } + job['summary.any_warning'] = 1 + summary_result = 'warning' + end + if stats.keys.any? { |stat| stat.match(/stderr/i) } + job['summary.any_stderr'] = 1 + summary_result = 'stderr' + end + if stats.keys.any? { |stat| stat.match(/error|nr_fail=0/i) } + job['summary.any_error'] = 1 + summary_result = 'error' + end + if stats.keys.any? { |stat| stat.match(/fail/i) } && + stats.keys.all? { |stat| !stat.match(/nr_fail=0/i) } + job['summary.any_fail'] = 1 + summary_result = 'fail' end - next unless job['stats'] - - @result[field] << job['stats'][field] if job['stats'][field] + if summary_result.empty? + job['summary.success'] = 1 + end end end - def deal_jobs - stats_count = Hash.new(0) - stats_jobs = {} - - @jobs.each do |job_id, job| - level = deal_stats(job) - add_result_fields(job, level) + def get_all_metrics(jobs) + metrics = [] + jobs.each do |job| + stats = job['stats'] + next unless stats - stat_key = @stats_level[level] - stat_jobs_key = stat_key + '_jobs' - - stats_count[stat_key] += 1 - stats_jobs[stat_jobs_key] ||= [] - stats_jobs[stat_jobs_key] << job_id + metrics.concat(stats.keys) end + metrics.uniq! + end - @stats['stats.count'].merge!(stats_count) - @stats['stats.count'].merge!(stats_jobs) + def initialize_result_hash(metrics) + result = { + 'kvcount' => {}, + 'raw.id' => {}, + 'sum.stats' => {}, + 'raw.stats' => {}, + 'avg.stats' => {}, + 'max.stats' => {}, + 'min.stats' => {} + } + metrics.each { |metric| result['raw.stats'][metric] = [] } + result end - def deal_stats(job, level = 0) - return 1 unless job['stats'] + def set_default_value(result, stats, metrics) + left_metrics = metrics - stats.keys + left_metrics.each { |metric| result['raw.stats'][metric] << nil } - job['stats'].each do |key, value| - match_stats_filter(key, value, job['id']) - calculate_stat(key, value) - level = get_stat_level(key, level) + stats.each do |key, value| + result['raw.stats'][key] << value end - return level end - def match_stats_filter(key, value, job_id) - @stats_filter.each do |filter| - next unless key.include?(filter) - - key = job_id + '.' + key - @stats_filter_result[key] = value + def kvcount(result, job) + KEYWORD.each do |keyword| + next unless job[keyword] - break + result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0 + result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 + result['raw.id']["[#{keyword}=#{job[keyword]}]"] ||= [] + result['raw.id']["[#{keyword}=#{job[keyword]}]"] << job['id'] end end - def calculate_stat(key, value) - if function_stat?(key) - return unless @fields.include?('stats.sum') - - @stats['stats.sum'][key] += value - else - return unless @fields.include?('stats.avg') - - @stats['stats.avg'][key] = (@stats['stats.avg'][key] + value) / 2 + def stats_count(result) + result['raw.stats'].each do |key, value| + if function_stat?(key) + result['sum.stats'][key] = value.compact.size + else + result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f + result['max.stats'][key] = value.compact.max + result['min.stats'][key] = value.compact.min + end end end - def get_stat_level(stat, level) - return level if level >= 3 - return 3 if stat.match(/error|fail/i) - return 2 if stat.match(/warn/i) + def query_jobs_state(jobs) + metrics = get_all_metrics(jobs) + result = initialize_result_hash(metrics) + jobs.each do |job| + stats = job['stats'] + next unless stats + + set_default_value(result, stats, metrics) + kvcount(result, job) + end - return 0 + stats_count(result) + result end - def output - result = { - 'stats.count' => @stats['stats.count'] - } + def output_yaml(prefix, result) + result.each do |key, value| + if prefix.empty? + prefix_key = "#{key}" + else + prefix_key = "#{prefix}.#{key}" + end - @stats.each do |key, value| - result[key] = value if @fields.include?(key) + if value.is_a? Hash + output_yaml(prefix_key, value) + else + puts "#{prefix_key}: #{value}".gsub('nil', 'null') + end end + end - @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? - @result.merge!(result) - puts JSON.pretty_generate(@result) + def output + @result = query_jobs_state(@jobs) + @result['kvcount'] = @result['kvcount'].to_a.sort.to_h + @result['raw.id'] = @result['raw.id'].to_a.sort.to_h + output_yaml('', @result) end end -- 2.23.0

2 1

[PATCH lkp-tests] jobs: split fault_reproduction.yaml
by Wei Jihui 19 Jan '21

19 Jan '21

[why] user can choose when to sleep ssh: use ssh.yaml is ssh when task normal finish. use ssh-on-fail.yaml is ssh when task is failed. [usage] submit -i ssh.yaml job.yaml Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- jobs/ssh-on-fail.yaml | 5 +++++ jobs/{fault_reproduction.yaml => ssh.yaml} | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 jobs/ssh-on-fail.yaml rename jobs/{fault_reproduction.yaml => ssh.yaml} (80%) diff --git a/jobs/ssh-on-fail.yaml b/jobs/ssh-on-fail.yaml new file mode 100644 index 00000000..3fbc1175 --- /dev/null +++ b/jobs/ssh-on-fail.yaml @@ -0,0 +1,5 @@ +<< : jobs/ssh.yaml + +on_fail: + sshd: + sleep: 6h diff --git a/jobs/fault_reproduction.yaml b/jobs/ssh.yaml similarity index 80% rename from jobs/fault_reproduction.yaml rename to jobs/ssh.yaml index 46a1e042..11e5b175 100644 --- a/jobs/fault_reproduction.yaml +++ b/jobs/ssh.yaml @@ -8,7 +8,3 @@ ssh_pub_key: %> sshd: sleep: 6h - -on_fail: - sshd: - sleep: 6h -- 2.23.0

1 0

[PATCH v3 compass-ci 4/4] container/delimiter: replace gem sources and delete useless packge
by Cao Xueliang 19 Jan '21

19 Jan '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/delimiter/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container/delimiter/Dockerfile b/container/delimiter/Dockerfile index c4e5074..cc57320 100644 --- a/container/delimiter/Dockerfile +++ b/container/delimiter/Dockerfile @@ -9,5 +9,5 @@ RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweiclou 'g++' 'gcc' 'pcre' 'libevent' 'make' 'git' 'bash' 'grep' 'coreutils' 'curl' 'util-linux' RUN umask 002 && \ - echo ':sources: ["http://rubygems.org"]' >> ~/.gemrc && \ - gem install rest-client activesupport git json yaml threadpool elasticsearch faye-websocket terminal-table mail io-console + gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ + gem install rest-client activesupport git json yaml elasticsearch faye-websocket terminal-table mail io-console -- 2.23.0

1 0

[PATCH v3 compass-ci 3/4] delimiter: use submit script in lkp-tests to submit job
by Cao Xueliang 19 Jan '21

19 Jan '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter/constants.rb | 1 + src/delimiter/utils.rb | 17 +++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/delimiter/constants.rb b/src/delimiter/constants.rb index 23b5053..457e33d 100644 --- a/src/delimiter/constants.rb +++ b/src/delimiter/constants.rb @@ -13,3 +13,4 @@ DELIMITER_TASK_QUEUE = 'delimiter' BISECT_RUN_SCRIPT = "#{ENV['CCI_SRC']}/src/delimiter/find-commit/bisect_run_script.rb" # The files which are in this dir can be uploaded by lkp-tests TMP_RESULT_ROOT = ENV['TMP_RESULT_ROOT'] || '/tmp/lkp/result' +PROCESS_JOB_YAML = "/tmp/process.yaml" diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index aaad394..2c6549c 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -61,15 +61,17 @@ module Utils return monitor.run end + def save_job_to_yaml(job, yaml_file) + File.open(yaml_file, 'w') { |f| YAML.dump(job, f) } + end + def submit_job(job) - sched = SchedClient.new - response = sched.submit_job(job.to_json) + save_job_to_yaml(job, PROCESS_JOB_YAML) + response = %x(#{LKP_SRC}/sbin/submit #{PROCESS_JOB_YAML}) puts "submit job response: #{response}" - res_arr = JSON.parse(response) - return nil if res_arr.empty? || !res_arr[0]['message'].empty? + return $1 if response =~ /job id=(.*)/ - # just consider build-pkg job - return res_arr[0]['job_id'] + return nil end # submit the bad job @@ -84,6 +86,8 @@ module Utils extract_finished = monitor_run_stop(query) return nil unless extract_finished.zero? + raise "the job is incredible for bisect: #{new_job_id}" unless credible?(job['bad_job_id'], new_job_id, error_id) + stats = query_stats(new_job_id, 10) raise "es cant query #{new_job_id} stats field!" unless stats @@ -138,6 +142,7 @@ module Utils job['my_token'] = account_info['my_token'] job['bad_job_id'] = job_id + job.delete('tboxgroup') job.delete('subqueue') job.delete('queue') job.delete('id') -- 2.23.0

1 0

[PATCH v3 compass-ci 2/4] src/stats_worker: select error_id which has the compile file
by Cao Xueliang 19 Jan '21

19 Jan '21

We select the error_id which has the compile file for more credible git bisect. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/stats_worker.cr | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index bcf6e52..01bae21 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -82,10 +82,10 @@ class StatsWorker } ) - new_error_ids = check_new_error_ids(error_ids, job_id) - unless new_error_ids.empty? + error_id = select_error_id(check_new_error_ids(error_ids, job_id)) + if error_id STDOUT.puts "send a delimiter task: job_id is #{job_id}" - @tq.add_task(DELIMITER_TASK_QUEUE, JSON.parse({"error_id" => new_error_ids.sample, + @tq.add_task(DELIMITER_TASK_QUEUE, JSON.parse({"error_id" => error_id, "job_id" => job_id, "lab" => LAB}.to_json)) end @@ -93,6 +93,14 @@ class StatsWorker system "echo '#{msg}'" end + def select_error_id(new_error_ids : Array) + new_error_ids.each do |error_id| + return error_id if /(cpp|c|h):(warning|error)/i =~ error_id + end + + return nil + end + def check_new_error_ids(error_ids : Array, job_id : String) new_error_ids = [] of String error_ids.each do |error_id| -- 2.23.0

1 0

[PATCH v3 compass-ci 1/4] lib/git_bisect: use "--first-parent" option when start bisect
by Cao Xueliang 19 Jan '21

19 Jan '21

1. The alpine 3.13 support to use the latest git which has the "--first-parent" option. 2. This option is useful in avoiding false positives when a merged branch contained broken or non-buildable commits, but the merge itself was OK. Ref: https://git-scm.com/docs/git-bisect Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/delimiter/Dockerfile | 2 +- lib/git_bisect.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/container/delimiter/Dockerfile b/container/delimiter/Dockerfile index ab7519d..c4e5074 100644 --- a/container/delimiter/Dockerfile +++ b/container/delimiter/Dockerfile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM alpine:3.11 +FROM alpine:3.13 RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories && \ apk update && \ diff --git a/lib/git_bisect.rb b/lib/git_bisect.rb index 2fee60c..e577afd 100644 --- a/lib/git_bisect.rb +++ b/lib/git_bisect.rb @@ -82,7 +82,7 @@ class GitBisect puts "bad_commit: #{@bad_commit}" puts "good_commit: #{@good_commit}" - result = `git -C #{@work_dir} bisect start #{@bad_commit} #{@good_commit}` + result = `git -C #{@work_dir} bisect start --first-parent #{@bad_commit} #{@good_commit}` temp = result.split(/\n/) if temp[0].include? 'Bisecting' result = `git -C #{@work_dir} bisect run #{BISECT_RUN_SCRIPT} #{@bad_job_id} #{(a)error_id.inspect} #{@work_dir}` -- 2.23.0

1 0

[PATCH compass-ci] doc/manual: optimize borrow-machine.zh.md
by Luan Shengde 19 Jan '21

19 Jan '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- doc/manual/borrow-machine.zh.md | 288 +++++++------------------------- 1 file changed, 59 insertions(+), 229 deletions(-) diff --git a/doc/manual/borrow-machine.zh.md b/doc/manual/borrow-machine.zh.md index 916aea7..e4c15f2 100644 --- a/doc/manual/borrow-machine.zh.md +++ b/doc/manual/borrow-machine.zh.md @@ -1,243 +1,73 @@ -# 前提条件 - -请确认您已按照 [apply-account.md](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc… -- send apply account email. -- receive email from compass-ci-robot(a)qq.com. -- local environment configuration. - - -# 申请测试机（虚拟机） - -1. 生成本地RSA公私钥对 - - ```shell - hi684@account-vm ~% ssh-keygen -t rsa - Generating public/private rsa key pair. - Enter file in which to save the key (/home/hi684/.ssh/id_rsa): - Created directory '/home/hi684/.ssh'. - Enter passphrase (empty for no passphrase): - Enter same passphrase again: - Your identification has been saved in /home/hi684/.ssh/id_rsa. - Your public key has been saved in /home/hi684/.ssh/id_rsa.pub. - The key fingerprint is: - SHA256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx hi684@account-vm - The key's randomart image is: - +---[RSA 2048]----+ - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - +----[SHA256]-----+ - hi684@account-vm ~% ls -hla .ssh - total 16K - drwx------. 2 hi684 hi684 4.0K Nov 26 16:37 . - drwx------. 7 hi684 hi684 4.0K Nov 26 16:37 .. - -rw-------. 1 hi684 hi684 1.8K Nov 26 16:37 id_rsa - -rw-r--r--. 1 hi684 hi684 398 Nov 26 16:37 id_rsa.pub - ``` - -2. 根据需求选择yaml - - 每位用户`/home/${USER}`目录下面都存放了一个 lkp-tests 的文件夹。 - - ```shell - hi684@account-vm ~% cd lkp-tests/jobs - hi684@account-vm ~/lkp-tests/jobs% ls -hl borrow-* - -rw-r--r--. 1 root root 53 Nov 2 14:54 borrow-10d.yaml - -rw-r--r--. 1 root root 64 Nov 2 14:54 borrow-1d.yaml - -rw-r--r--. 1 root root 235 Nov 19 15:27 borrow-1h.yaml - ``` - -3. 提交yaml并连接测试机（虚拟机） - - ```shell - hi684@account-vm ~/lkp-tests/jobs% submit -c -m testbox=vm-2p8g borrow-1h.yaml - submit borrow-1h.yaml, got job_id=z9.170593 - query=>{"job_id":["z9.170593"]} - connect to ws://172.168.131.2:11310/filter - {"job_id":"z9.170593","message":"","job_state":"submit","result_root":"/srv/result/borrow/2020-11-26/vm-2p8g/openeuler-20.03-aarch64/3600/z9.170593"} - {"job_id": "z9.170593", "result_root": "/srv/result/borrow/2020-11-26/vm-2p8g/openeuler-20.03-aarch64/3600/z9.170593", "job_state": "set result root"} - {"job_id": "z9.170593", "job_state": "boot"} - {"job_id": "z9.170593", "job_state": "download"} - {"time":"2020-11-26 14:45:06","mac":"0a-1f-0d-3c-91-5c","ip":"172.18.156.13","job_id":"z9.170593","state":"running","testbox":"vm-2p8g.taishan200-2280-2s64p-256g--a38-12"} - {"job_state":"running","job_id":"z9.170593"} - {"job_id": "z9.170593", "state": "set ssh port", "ssh_port": "51840", "tbox_name": "vm-2p8g.taishan200-2280-2s64p-256g--a38-12"} - Host 172.168.131.2 not found in /home/hi684/.ssh/known_hosts - Warning: Permanently added '[172.168.131.2]:51840' (ECDSA) to the list of known hosts. - Last login: Wed Sep 23 11:10:58 2020 - - - Welcome to 4.19.90-2003.4.0.0036.oe1.aarch64 - - System information as of time: Thu Nov 26 06:44:18 CST 2020 - - System load: 0.83 - Processes: 107 - Memory used: 6.1% - Swap used: 0.0% - Usage On: 89% - IP address: 172.18.156.13 - Users online: 1 - - - - root@vm-2p8g ~# - ``` - - 更多关于`submit命令如何使用`、`testbox都有什么可选项`、`如何borrow指定的操作系统`，请参见文章末尾FAQ。 - -4. 使用完毕退还测试机（虚拟机） - - ```shell - root@vm-2p8g ~# reboot - Connection to 172.168.131.2 closed by remote host. - Connection to 172.168.131.2 closed. - hi684@account-vm ~/lkp-tests/jobs% - ``` - - -# 申请测试机（物理机） - - -1. 生成本地RSA公私钥对 - - ```shell - hi684@account-vm ~% ssh-keygen -t rsa - Generating public/private rsa key pair. - Enter file in which to save the key (/home/hi684/.ssh/id_rsa): - Created directory '/home/hi684/.ssh'. - Enter passphrase (empty for no passphrase): - Enter same passphrase again: - Your identification has been saved in /home/hi684/.ssh/id_rsa. - Your public key has been saved in /home/hi684/.ssh/id_rsa.pub. - The key fingerprint is: - SHA256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx hi684@account-vm - The key's randomart image is: - +---[RSA 2048]----+ - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - |xxxxxxxxxxxxxxxxx| - +----[SHA256]-----+ - hi684@account-vm ~% ls -hla .ssh - total 16K - drwx------. 2 hi684 hi684 4.0K Nov 26 16:37 . - drwx------. 7 hi684 hi684 4.0K Nov 26 16:37 .. - -rw-------. 1 hi684 hi684 1.8K Nov 26 16:37 id_rsa - -rw-r--r--. 1 hi684 hi684 398 Nov 26 16:37 id_rsa.pub - ``` - -2. 根据需求选择yaml - - 每位用户`/home/${USER}`目录下面都存放了一个 lkp-tests 的文件夹。 - - ```shell - hi684@account-vm ~% cd lkp-tests/jobs - hi684@account-vm ~/lkp-tests/jobs% ls -hl borrow-* - -rw-r--r--. 1 root root 53 Nov 2 14:54 borrow-10d.yaml - -rw-r--r--. 1 root root 64 Nov 2 14:54 borrow-1d.yaml - -rw-r--r--. 1 root root 235 Nov 19 15:27 borrow-1h.yaml - ``` - -3. 提交yaml并连接测试机（物理机） - - ```shell - hi684@account-vm ~/lkp-tests/jobs% submit -c -m testbox=taishan200-2280-2s64p-256g borrow-1h.yaml - submit borrow-1h.yaml, got job_id=z9.170594 - query=>{"job_id":["z9.170594"]} - connect to ws://172.168.131.2:11310/filter - {"job_id":"z9.170594","message":"","job_state":"submit","result_root":"/srv/result/borrow/2020-11-26/taishan200-2280-2s64p-256g/openeuler-20.03-aarch64/3600/z9.170594"} - {"job_id": "z9.170594", "result_root": "/srv/result/borrow/2020-11-26/taishan200-2280-2s64p-256g/openeuler-20.03-aarch64/3600/z9.170594", "job_state": "set result root"} - {"job_id": "z9.170594", "job_state": "boot"} - {"job_id": "z9.170594", "job_state": "download"} - {"time":"2020-11-26 14:51:56","mac":"84-46-fe-26-d3-47","ip":"172.168.178.48","job_id":"z9.170594","state":"running","testbox":"taishan200-2280-2s64p-256g--a5"} - {"job_state":"running","job_id":"z9.170594"} - {"job_id": "z9.170594", "state": "set ssh port", "ssh_port": "50420", "tbox_name": "taishan200-2280-2s64p-256g--a5"} - Host 172.168.131.2 not found in /home/hi684/.ssh/known_hosts - Warning: Permanently added '[172.168.131.2]:50420' (ECDSA) to the list of known hosts. - Last login: Wed Sep 23 11:10:58 2020 +# 前置准备 +- 申请account +- 配置默认yaml文件 - Welcome to 4.19.90-2003.4.0.0036.oe1.aarch64 +如果未完成以上步骤，请参考[apply-account.md](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/apply-account.md)进行操作。 - System information as of time: Thu Nov 26 14:51:59 CST 2020 +# 申请测试机 - System load: 1.31 - Processes: 1020 - Memory used: 5.1% - Swap used: 0.0% - Usage On: 3% - IP address: 172.168.178.48 - Users online: 1 - - - - root@taishan200-2280-2s64p-256g--a5 ~# - ``` - - 更多关于`submit命令如何使用`、`testbox都有什么可选项`、`如何borrow指定的操作系统`，请参见文章末尾FAQ。 - -4. 使用完毕退还测试机（物理机） - - ```shell - root@taishan200-2280-2s64p-256g--a5 ~# reboot - Connection to 172.168.131.2 closed by remote host. - Connection to 172.168.131.2 closed. - hi684@account-vm ~/lkp-tests/jobs% - ``` +## 1. 生成本地RSA公/私钥对 + 使用下面命令查看是否已存在公/私钥： + + ls -a ~/.ssh + + 如果没有现成的公/私钥对，请使用下面命令进行生成： + + ssh-keygen -r rsa + +## 2. 选择job yaml + + 在 ~/lkp-tests/jobs 目录下未您提供了众多job yaml模板。 + + 使用以下命令可筛选出借机器的模板： + + cd ~/lkp-tests/jobs + ls -a borrow* + + - 检查您选择的模板中引用的公钥地址和文件名和你本地的公钥一致。 + +## 3. 提交job + + 使用以下命令提交job： + + submit -c -m testbox=vm-2p8g borrow-1h.yaml + + - 使用该命令，您可以实时查看job状态，测试机正常运行后直接登入。 + +## 4. 退还测试机 + + 到期自动退还： + + 借用期限到达后，测试接将自动重启归还。 + + 手动退还： + + 执行‘reboot’命令进行归还。 + + - 所有测试机在执行‘reboot’命令后都会被归还。 + - 机器归还后，如果您还需要继续借用，请重新提交job进行申请。 # FAQ +## 自定义借用时长 + + 在借用机器的模板中，找到’runtime‘字段，根据需求修改借用时长。 + +## submit命令指导 + + [submit命令详解](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manua… + +## 可选的testbox -* 如何自行修改申请时长 - - ```shell - hi684@account-vm ~/lkp-tests/jobs% cat borrow-1h.yaml - suite: borrow - testcase: borrow - - runtime: 1h - ssh_pub_key: <%= - begin - File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp - rescue - nil - end - %> - sshd: - # sleep at the bottom - sleep: - hi684@account-vm ~/lkp-tests/jobs% grep runtime: borrow-1h.yaml - sleep: 1h - # 使用vim来修改你的runtime字段的值 - hi684@account-vm ~/lkp-tests/jobs% vim borrow-1h.yaml - # 修改完毕后重新submit即可 - hi684@account-vm ~/lkp-tests/jobs% submit -c -m testbox=vm-2p8g borrow-1h.yaml - ``` - -* Submit命令指导 - - 参考文档：[submit命令详解](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc… - -* testbox有什么可选项 - - testbox可选项请参考：https://gitee.com/wu_fengguang/lab-z9/tree/master/hosts + 可选的testbox模板，请参考：https://gitee.com/wu_fengguang/lab-z9/tree/master/hosts >![](./../public_sys-resources/icon-note.gif) **说明：** > - > 虚拟机的testbox : vm-xxx - > 物理机的testbox : taishan200-2280-xxx + > - 容器: dc-xxx + > - 虚拟机: vm-xxx + > - 物理机: taishan200-2280-xxx @@ -245,6 +75,6 @@ > - 物理机的testbox若选择以`--axx`结尾的，则表示指定到了具体的某一个物理机。若此物理机任务队列中已经有任务在排队，则需要等待队列中前面的任务执行完毕后，才会轮到你提交的borrow任务。 > - 物理机的testbox若不选择以`-axx`结尾的，表示不指定具体的某一个物理机。则此时集群中的空闲物理机会即时被分配执行你的borrow任务。 -* 如何 borrow 指定的操作系统 +## 如何 borrow 指定的操作系统关于支持的`os`, `os_arch`, `os_version`，参见：[os-os_verison-os_arch.md](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/job/os-os_verison-os_arch.md) -- 2.23.0

1 0

[PATCH lkp-tests] distro/adaptation: add pgbench dependency
by Hu Xuejiao 19 Jan '21

19 Jan '21

Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- distro/adaptation/centos | 1 + 1 file changed, 1 insertion(+) diff --git a/distro/adaptation/centos b/distro/adaptation/centos index 2cfe3877..8fbae666 100644 --- a/distro/adaptation/centos +++ b/distro/adaptation/centos @@ -335,3 +335,4 @@ liboath-dev: liboath-devel lsb-release: redhat-lsb linux-base: openssh-client: openssh-clients +postgresql-server: postgresql -- 2.23.0

1 0

[PATCH compass-ci] scheduler: respond an '{}' when 'vt' is non-existent or empty
by Ren Wen 19 Jan '21

19 Jan '21

In job.yaml: case1: no 'vt' -> {} case2: vt: -> {} case3: vt: domain: <domain.xml> -> {"domain":"<domain.xml>"} Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/scheduler/find_job_boot.cr | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 152d549..f16dc30 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -174,6 +174,9 @@ class Sched _kernel_params = job["kernel_params"]? _kernel_params = _kernel_params.as_a.map(&.to_s).join(" ") if _kernel_params + _vt = job["vt"]? + _vt = ({} of String => String) unless (_vt && _vt != nil) + return { "job_id" => job.id, "kernel_uri" => job.kernel_uri, @@ -181,7 +184,7 @@ class Sched "kernel_params" => _kernel_params, "result_root" => job.result_root, "LKP_SERVER" => job["LKP_SERVER"]?, - "vt" => job["vt"]?, + "vt" => _vt, "RESULT_WEBDAV_PORT" => job["RESULT_WEBDAV_PORT"]? || "3080", }.to_json end -- 2.23.0

1 0

[PATCH v2 compass-ci 3/3] delimiter: use submit script in lkp-tests to submit job
by Cao Xueliang 19 Jan '21

19 Jan '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter/constants.rb | 1 + src/delimiter/utils.rb | 17 +++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/delimiter/constants.rb b/src/delimiter/constants.rb index 23b5053..457e33d 100644 --- a/src/delimiter/constants.rb +++ b/src/delimiter/constants.rb @@ -13,3 +13,4 @@ DELIMITER_TASK_QUEUE = 'delimiter' BISECT_RUN_SCRIPT = "#{ENV['CCI_SRC']}/src/delimiter/find-commit/bisect_run_script.rb" # The files which are in this dir can be uploaded by lkp-tests TMP_RESULT_ROOT = ENV['TMP_RESULT_ROOT'] || '/tmp/lkp/result' +PROCESS_JOB_YAML = "/tmp/process.yaml" diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index aaad394..2c6549c 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -61,15 +61,17 @@ module Utils return monitor.run end + def save_job_to_yaml(job, yaml_file) + File.open(yaml_file, 'w') { |f| YAML.dump(job, f) } + end + def submit_job(job) - sched = SchedClient.new - response = sched.submit_job(job.to_json) + save_job_to_yaml(job, PROCESS_JOB_YAML) + response = %x(#{LKP_SRC}/sbin/submit #{PROCESS_JOB_YAML}) puts "submit job response: #{response}" - res_arr = JSON.parse(response) - return nil if res_arr.empty? || !res_arr[0]['message'].empty? + return $1 if response =~ /job id=(.*)/ - # just consider build-pkg job - return res_arr[0]['job_id'] + return nil end # submit the bad job @@ -84,6 +86,8 @@ module Utils extract_finished = monitor_run_stop(query) return nil unless extract_finished.zero? + raise "the job is incredible for bisect: #{new_job_id}" unless credible?(job['bad_job_id'], new_job_id, error_id) + stats = query_stats(new_job_id, 10) raise "es cant query #{new_job_id} stats field!" unless stats @@ -138,6 +142,7 @@ module Utils job['my_token'] = account_info['my_token'] job['bad_job_id'] = job_id + job.delete('tboxgroup') job.delete('subqueue') job.delete('queue') job.delete('id') -- 2.23.0

1 0

[PATCH v2 compass-ci 2/3] src/stats_worker: select error_id which has the compile file
by Cao Xueliang 19 Jan '21

19 Jan '21

We select the error_id which has the compile file for more credible git bisect. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/stats_worker.cr | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index bcf6e52..01bae21 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -82,10 +82,10 @@ class StatsWorker } ) - new_error_ids = check_new_error_ids(error_ids, job_id) - unless new_error_ids.empty? + error_id = select_error_id(check_new_error_ids(error_ids, job_id)) + if error_id STDOUT.puts "send a delimiter task: job_id is #{job_id}" - @tq.add_task(DELIMITER_TASK_QUEUE, JSON.parse({"error_id" => new_error_ids.sample, + @tq.add_task(DELIMITER_TASK_QUEUE, JSON.parse({"error_id" => error_id, "job_id" => job_id, "lab" => LAB}.to_json)) end @@ -93,6 +93,14 @@ class StatsWorker system "echo '#{msg}'" end + def select_error_id(new_error_ids : Array) + new_error_ids.each do |error_id| + return error_id if /(cpp|c|h):(warning|error)/i =~ error_id + end + + return nil + end + def check_new_error_ids(error_ids : Array, job_id : String) new_error_ids = [] of String error_ids.each do |error_id| -- 2.23.0

1 0

[PATCH v2 compass-ci 1/3] lib/git_bisect: use "--first-parent" option when start bisect
by Cao Xueliang 19 Jan '21

19 Jan '21

1. The alpine 3.13 support to use the latest git which has the "--first-parent" option. 2. This option is useful in avoiding false positives when a merged branch contained broken or non-buildable commits, but the merge itself was OK. Ref: https://git-scm.com/docs/git-bisect Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/delimiter/Dockerfile | 6 +++--- lib/git_bisect.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/container/delimiter/Dockerfile b/container/delimiter/Dockerfile index ab7519d..cc57320 100644 --- a/container/delimiter/Dockerfile +++ b/container/delimiter/Dockerfile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM alpine:3.11 +FROM alpine:3.13 RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories && \ apk update && \ @@ -9,5 +9,5 @@ RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweiclou 'g++' 'gcc' 'pcre' 'libevent' 'make' 'git' 'bash' 'grep' 'coreutils' 'curl' 'util-linux' RUN umask 002 && \ - echo ':sources: ["http://rubygems.org"]' >> ~/.gemrc && \ - gem install rest-client activesupport git json yaml threadpool elasticsearch faye-websocket terminal-table mail io-console + gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ + gem install rest-client activesupport git json yaml elasticsearch faye-websocket terminal-table mail io-console diff --git a/lib/git_bisect.rb b/lib/git_bisect.rb index 2fee60c..e577afd 100644 --- a/lib/git_bisect.rb +++ b/lib/git_bisect.rb @@ -82,7 +82,7 @@ class GitBisect puts "bad_commit: #{@bad_commit}" puts "good_commit: #{@good_commit}" - result = `git -C #{@work_dir} bisect start #{@bad_commit} #{@good_commit}` + result = `git -C #{@work_dir} bisect start --first-parent #{@bad_commit} #{@good_commit}` temp = result.split(/\n/) if temp[0].include? 'Bisecting' result = `git -C #{@work_dir} bisect run #{BISECT_RUN_SCRIPT} #{@bad_job_id} #{(a)error_id.inspect} #{@work_dir}` -- 2.23.0

1 0

[PATCH v2 compass-ci] scheduler: more response of boot.libvirt
by Ren Wen 19 Jan '21

19 Jan '21

To download from libvirt-xml path, adding such response: - SRV_HTTP_HOST - SRV_HTTP_PORT Signed-off-by: Ren Wen <15991987063(a)163.com> --- container/scheduler/start | 3 +++ src/scheduler/constants.cr | 3 +++ src/scheduler/find_job_boot.cr | 2 ++ 3 files changed, 8 insertions(+) diff --git a/container/scheduler/start b/container/scheduler/start index 70a3496..c32a7a8 100755 --- a/container/scheduler/start +++ b/container/scheduler/start @@ -25,6 +25,8 @@ names = Set.new %w[ SSHR_PORT SSHR_PORT_BASE SSHR_PORT_LEN + SRV_HTTP_HOST + SRV_HTTP_PORT lab ] @@ -35,6 +37,7 @@ defaults['SSHR_PORT_LEN'] ||= 2000 defaults['SCHED_PORT'] ||= '3000' defaults['SCHED_HOST'] ||= '172.17.0.1' defaults['MASTER_FLUENTD_PORT'] ||= '24224' +defaults['SRV_HTTP_PORT'] ||= '11300' env = docker_env(defaults) CCI_REPOS = ENV['CCI_REPOS'] || '/c' diff --git a/src/scheduler/constants.cr b/src/scheduler/constants.cr index 1146b21..668da6d 100644 --- a/src/scheduler/constants.cr +++ b/src/scheduler/constants.cr @@ -23,6 +23,9 @@ INITRD_HTTP_PORT = (ENV.has_key?("INITRD_HTTP_PORT") ? ENV["INITRD_HTTP_PORT"] : OS_HTTP_HOST = (ENV.has_key?("OS_HTTP_HOST") ? ENV["OS_HTTP_HOST"] : "172.17.0.1") OS_HTTP_PORT = (ENV.has_key?("OS_HTTP_PORT") ? ENV["OS_HTTP_PORT"] : 8000).to_i32 +SRV_HTTP_HOST = (ENV.has_key?("SRV_HTTP_HOST") ? ENV["SRV_HTTP_HOST"] : "172.17.0.1") +SRV_HTTP_PORT = (ENV.has_key?("SRV_HTTP_PORT") ? ENV["SRV_HTTP_PORT"] : "11300") + CCI_REPOS = (ENV.has_key?("CCI_REPOS") ? ENV["CCI_REPOS"] : "/c") LAB_REPO = "lab-z9" diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 152d549..3f446b2 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -183,6 +183,8 @@ class Sched "LKP_SERVER" => job["LKP_SERVER"]?, "vt" => job["vt"]?, "RESULT_WEBDAV_PORT" => job["RESULT_WEBDAV_PORT"]? || "3080", + "SRV_HTTP_HOST" => SRV_HTTP_HOST, + "SRV_HTTP_PORT" => SRV_HTTP_PORT, }.to_json end -- 2.23.0

1 0

[PATCH compass-ci] src/stats_worker: choice error_id which has the compile file
by Cao Xueliang 19 Jan '21

19 Jan '21

We choice the error_id which has the compile file for more credible git bisect. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/stats_worker.cr | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index bcf6e52..4f3bd81 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -82,10 +82,10 @@ class StatsWorker } ) - new_error_ids = check_new_error_ids(error_ids, job_id) - unless new_error_ids.empty? + error_id = choice_error_id(check_new_error_ids(error_ids, job_id)) + if error_id STDOUT.puts "send a delimiter task: job_id is #{job_id}" - @tq.add_task(DELIMITER_TASK_QUEUE, JSON.parse({"error_id" => new_error_ids.sample, + @tq.add_task(DELIMITER_TASK_QUEUE, JSON.parse({"error_id" => error_id, "job_id" => job_id, "lab" => LAB}.to_json)) end @@ -93,6 +93,14 @@ class StatsWorker system "echo '#{msg}'" end + def choice_error_id(new_error_ids : Array) + new_error_ids.each do |error_id| + return error_id if /(cpp|c|h):(warning|error)/i =~ error_id + end + + return nil + end + def check_new_error_ids(error_ids : Array, job_id : String) new_error_ids = [] of String error_ids.each do |error_id| -- 2.23.0

2 2

[PATCH compass-ci] 如何登陆测试机调测任务.md: change host-info.yaml to spinlock.yaml
by Zhang Yale 19 Jan '21

19 Jan '21

host-info.yaml looks too cryptic, so use spinlock.yaml as example can be more practical and simple. Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- ...03\346\265\213\344\273\273\345\212\241.md" | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git "a/doc/manual/\345\246\202\344\275\225\347\231\273\345\275\225\346\265\213\350\257\225\346\234\272\350\260\203\346\265\213\344\273\273\345\212\241.md" "b/doc/manual/\345\246\202\344\275\225\347\231\273\345\275\225\346\265\213\350\257\225\346\234\272\350\260\203\346\265\213\344\273\273\345\212\241.md" index 7a54505..d423bcd 100644 --- "a/doc/manual/\345\246\202\344\275\225\347\231\273\345\275\225\346\265\213\350\257\225\346\234\272\350\260\203\346\265\213\344\273\273\345\212\241.md" +++ "b/doc/manual/\345\246\202\344\275\225\347\231\273\345\275\225\346\265\213\350\257\225\346\234\272\350\260\203\346\265\213\344\273\273\345\212\241.md" @@ -12,38 +12,43 @@ ### 方法一：在测试机运行脚本之前进入测试机调测任务 -在job.yaml里加上sshd和sleep字段，测试机在运行脚本之前sleep, 并免密登录进去，手动输入命令或脚本进行调试，以host-info.yaml为例： +在job.yaml里加上sshd和sleep字段，测试机在运行脚本之前sleep, 并免密登录进去，手动输入命令或脚本进行调试，以spinlock.yaml为例： ```yaml - suite: host-info - category: functional + suite: spinlock + category: benchmark + + nr_threads: + - 1 sshd: # sleep at the bottom sleep: 1h - - host-info: + spinlock: ``` ### 方法二：在测试任务运行失败时进入测试机调测任务 -在job.yaml里加上on_fail字段，并在on_fail下加上sshd和sleep字段，测试任务失败后，免密登录进去手动调试，以host-info.yaml为例： +在job.yaml里加上on_fail字段，并在on_fail下加上sshd和sleep字段，测试任务失败后，免密登录进去手动调试，以spinlock.yaml为例： ```yaml - suite: host-info - category: functional + suite: spinlock + category: benchmark + + nr_threads: + - 1 - host-info: + spinlock: on_fail: sshd: - sleep: 1000s + sleep: 1h ``` ## 2.2 提交job.yaml ### 方法一：直接提交 - submit host-info.yaml + submit spinlock.yaml 在收到邮件后，按照邮件提示手动免密登录到执行机调测 @@ -51,6 +56,6 @@ ### 方法二：带参数提交 - submit -m -c host-info.yaml + submit -m -c spinlock.yaml 该方法不用查看邮件，可自动免密登录到测试机 -- 2.23.0

1 0