- Compass-ci - mailweb.openeuler.org

[PATCH v4 compass-ci] sbin/run_job: Automatic task execution
by Hu Xuejiao 30 Dec '20

30 Dec '20

[purpose] Verify that a local job can be consumed properly Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sbin/run_job | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100755 sbin/run_job diff --git a/sbin/run_job b/sbin/run_job new file mode 100755 index 0000000..c84df0d --- /dev/null +++ b/sbin/run_job @@ -0,0 +1,74 @@ +#!/bin/bash + +cci_makepkg_file="$LKP_SRC/jobs/cci-makepkg.yaml" +grep "atomic" $cci_makepkg_file || sed -i 's|benchmark:|& atomic|g' $cci_makepkg_file + +submit_one_job() +{ + cmd=( + submit -m $yaml + os=${single_arg[0]} + os_arch=${single_arg[1]} + os_version=${single_arg[2]} + os_mount=${single_arg[3]} + docker_images=${single_arg[4]} + testbox=$testbox + queue="${tbox_group}.${HOSTNAME}" + ) + output=$("${cmd[@]}") + echo "$output" | grep -o -E 'job_state=finished[^\"]*' +} + +submit_job() +{ + yaml_args=("$cci_makepkg_file" "host-info.yaml") + for yaml in "${yaml_args[@]}" + do + submit_one_job & + done +} + +run_qemu() +{ + local nr_vm=4 + local testbox="vm-2p8g" + local tbox_group="vm-2p8g" + local hostname="${tbox_group}.${HOSTNAME}" + local queues="${tbox_group}.${HOSTNAME}" + + $CCI_SRC/providers/multi-qemu -n "$hostname" -c "$nr_vm" -q "$queues" + + qemu_args=( + "openeuler aarch64 20.03 cifs" + "centos aarch64 7.6.1810 initramfs" + ) + for qemu_arg in "${qemu_args[@]}" + do + local single_arg=($(echo $qemu_arg)) + submit_job + done +} + +run_docker() +{ + local nr_container=2 + local tbox_group="dc-8g" + local testbox="dc-8g" + local hostname="${tbox_group}.${HOSTNAME}" + local queues="${tbox_group}.${HOSTNAME}" + + $CCI_SRC/providers/multi-docker -n $hostname -c $nr_container -q $queues + + docker_args=( + "centos aarch64 7 container" + ) + for docker_arg in "${docker_args[@]}" + do + local single_arg=($(echo $docker_arg)) + submit_job + done +} + +run_qemu +run_docker +wait -- 2.23.0

1 0

[PATCH v2 compass-ci] lib/mail_bisect_result: optimize bisect report
by Cao Xueliang 30 Dec '20

30 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- doc/bisect_email.en.md | 28 ++++++++++++++++++++++++++++ lib/constants.rb | 2 ++ lib/mail_bisect_result.rb | 28 ++++++++++++++++++---------- 3 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 doc/bisect_email.en.md diff --git a/doc/bisect_email.en.md b/doc/bisect_email.en.md new file mode 100644 index 0000000..fee7ee1 --- /dev/null +++ b/doc/bisect_email.en.md @@ -0,0 +1,28 @@ +# Who are we? + +Compass-CI runs a number of tests for open source projects for free. +We aim to improve OSS quality in a number of hardware and OS environment. +We are part of the openEuler community. + +# Why do you get the email? + +Whenever you git push, our test robot will pull and test the new code. +When new errors are found, Compass-CI will auto run git bisect to find out the first bad commit, +and send email report to the commit author. + +# About the email + +The first bad commit job result directory: +you can read the below file to understand the content of each file. +- https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/browse-res… + +Notice: +You may need to read the output file of the first bad commit job result directory, +it is the build log about the first bad commit. + +# how to confirm bug fix + +- git push your fix patch to your project. +- download the job.yaml from the first bad commit job directory. +- submit -m job.yaml upstream_commit=$your_fix_commit. +- check result from the new result directory which is in output above command and key is result_root. diff --git a/lib/constants.rb b/lib/constants.rb index f0938f2..c3c7543 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -13,3 +13,5 @@ MAIL_PORT = config['SEND_MAIL_PORT'] || 49000 SRV_HTTP_HOST = config['SRV_HTTP_HOST'] || ENV['SRV_HTTP_HOST'] || '172.17.0.1' SRV_HTTP_PORT = config['SRV_HTTP_PORT'] || ENV['SRV_HTTP_PORT'] || 11300 + +SRV_HTTP_DOMAIN = config['SRV_HTTP_DOMAIN'] || ENV['SRV_HTTP_DOMAIN'] || 'api.compass-ci.openeuler.org' diff --git a/lib/mail_bisect_result.rb b/lib/mail_bisect_result.rb index 5f88c36..f0f0194 100644 --- a/lib/mail_bisect_result.rb +++ b/lib/mail_bisect_result.rb @@ -3,6 +3,8 @@ require 'json' require 'base64' require_relative 'git' +require_relative 'es_query' +require_relative 'constants' require_relative 'mail_client' require_relative 'assign_account_client' @@ -27,10 +29,11 @@ class MailBisectResult def compose_mail subject = "[Compass-CI][#{(a)repo.split('/')[1]}] #{@commit_id[0..9]} #{(a)bisect_error[0].split("\n")[0]}" - prefix_srv = "http://#{ENV['SRV_HTTP_HOST']}:#{ENV['SRV_HTTP_PORT']}" - bisect_job_url = ENV['result_root'] ? "bisect job info: #{prefix_srv}#{ENV['result_root']}\n" : '' - pkgbuild_repo_url = "PKGBUILD info: #{prefix_srv}/#{@pkgbuild_repo}\n" - first_bad_commit_job_url = "first bad commit job info: #{prefix_srv}#{@first_bad_commit_result_root}\n" + prefix_srv = "http://#{SRV_HTTP_DOMAIN}:#{SRV_HTTP_PORT}" + bisect_job_url = ENV['result_root'] ? "bisect job result directory:\n#{prefix_srv}#{ENV['result_root']}\n" : '' + bisect_report_doc = "bisect email doc:\nhttps://gitee.com/wu_fengguang/compass-ci/blob/master/doc/bisect_emai…" + pkgbuild_repo_url = "PKGBUILD:\n#{prefix_srv}/#{@pkgbuild_repo}\n" + first_bad_commit_job_url = "first bad commit job result directory:\n#{prefix_srv}#{@first_bad_commit_result_root}\n" data = <<~BODY To: #{@to} @@ -38,17 +41,19 @@ class MailBisectResult Hi #{(a)git_commit.author_name}, - url: #{(a)git_commit.url} - commit: #{@commit_id} ("#{(a)git_commit.subject}") - compiler: gcc (GCC) 7.3.0 + We found some error/warning(s) and the first bad commit in the below project: + git url: #{(a)git_commit.url} + git commit: #{@commit_id} ("#{(a)git_commit.subject}") - all errors/warnings (new ones prefixed by >>): + All error/warning(s) (new ones prefixed by >>): #{@all_errors} - reference information: + Reference information: + compiler: gcc (GCC) 7.3.0 #{pkgbuild_repo_url} - #{bisect_job_url} #{first_bad_commit_job_url} + #{bisect_job_url} + #{bisect_report_doc} Regards, Compass CI team BODY @@ -67,6 +72,9 @@ class MailBisectResult 'my_commit_url' => "#{@git_commit.url}/commit/#{@commit_id}" } + account_info = ESQuery.new(index: 'accounts').query_by_id(@to) + return if account_info + apply_account = AutoAssignAccount.new(user_info) apply_account.send_account end -- 2.23.0

1 0

[PATCH compass-ci] lib/mail_bisect_result: optimize bisect report
by Cao Xueliang 30 Dec '20

30 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/mail_bisect_result.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/mail_bisect_result.rb b/lib/mail_bisect_result.rb index 5f88c36..4883072 100644 --- a/lib/mail_bisect_result.rb +++ b/lib/mail_bisect_result.rb @@ -3,6 +3,7 @@ require 'json' require 'base64' require_relative 'git' +require_relative 'es_query' require_relative 'mail_client' require_relative 'assign_account_client' @@ -28,9 +29,9 @@ class MailBisectResult def compose_mail subject = "[Compass-CI][#{(a)repo.split('/')[1]}] #{@commit_id[0..9]} #{(a)bisect_error[0].split("\n")[0]}" prefix_srv = "http://#{ENV['SRV_HTTP_HOST']}:#{ENV['SRV_HTTP_PORT']}" - bisect_job_url = ENV['result_root'] ? "bisect job info: #{prefix_srv}#{ENV['result_root']}\n" : '' - pkgbuild_repo_url = "PKGBUILD info: #{prefix_srv}/#{@pkgbuild_repo}\n" - first_bad_commit_job_url = "first bad commit job info: #{prefix_srv}#{@first_bad_commit_result_root}\n" + bisect_job_url = ENV['result_root'] ? "bisect job result directory: #{prefix_srv}#{ENV['result_root']}\n" : '' + pkgbuild_repo_url = "PKGBUILD file directory: #{prefix_srv}/#{@pkgbuild_repo}\n" + first_bad_commit_job_url = "first bad commit job result directory: #{prefix_srv}#{@first_bad_commit_result_root}\n" data = <<~BODY To: #{@to} @@ -46,9 +47,9 @@ class MailBisectResult #{@all_errors} reference information: + #{first_bad_commit_job_url} #{pkgbuild_repo_url} #{bisect_job_url} - #{first_bad_commit_job_url} Regards, Compass CI team BODY @@ -67,6 +68,9 @@ class MailBisectResult 'my_commit_url' => "#{@git_commit.url}/commit/#{@commit_id}" } + account_info = ESQuery.new(index: 'accounts').query_by_id(@to) + return if account_info + apply_account = AutoAssignAccount.new(user_info) apply_account.send_account end -- 2.23.0

1 1

[PATCH v1 compass-ci] lib/mail_bisect_result: optimize bisect report
by Cao Xueliang 30 Dec '20

30 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/constants.rb | 2 ++ lib/mail_bisect_result.rb | 28 ++++++++++++++++++---------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/lib/constants.rb b/lib/constants.rb index f0938f2..c3c7543 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -13,3 +13,5 @@ MAIL_PORT = config['SEND_MAIL_PORT'] || 49000 SRV_HTTP_HOST = config['SRV_HTTP_HOST'] || ENV['SRV_HTTP_HOST'] || '172.17.0.1' SRV_HTTP_PORT = config['SRV_HTTP_PORT'] || ENV['SRV_HTTP_PORT'] || 11300 + +SRV_HTTP_DOMAIN = config['SRV_HTTP_DOMAIN'] || ENV['SRV_HTTP_DOMAIN'] || 'api.compass-ci.openeuler.org' diff --git a/lib/mail_bisect_result.rb b/lib/mail_bisect_result.rb index 5f88c36..bc3c74a 100644 --- a/lib/mail_bisect_result.rb +++ b/lib/mail_bisect_result.rb @@ -3,6 +3,8 @@ require 'json' require 'base64' require_relative 'git' +require_relative 'es_query' +require_relative 'constants' require_relative 'mail_client' require_relative 'assign_account_client' @@ -27,10 +29,11 @@ class MailBisectResult def compose_mail subject = "[Compass-CI][#{(a)repo.split('/')[1]}] #{@commit_id[0..9]} #{(a)bisect_error[0].split("\n")[0]}" - prefix_srv = "http://#{ENV['SRV_HTTP_HOST']}:#{ENV['SRV_HTTP_PORT']}" - bisect_job_url = ENV['result_root'] ? "bisect job info: #{prefix_srv}#{ENV['result_root']}\n" : '' - pkgbuild_repo_url = "PKGBUILD info: #{prefix_srv}/#{@pkgbuild_repo}\n" - first_bad_commit_job_url = "first bad commit job info: #{prefix_srv}#{@first_bad_commit_result_root}\n" + prefix_srv = "http://#{SRV_HTTP_DOMAIN}:#{SRV_HTTP_PORT}" + bisect_job_url = ENV['result_root'] ? "bisect job result directory:\n#{prefix_srv}#{ENV['result_root']}\n" : '' + bisect_report_doc = "bisect report:\nhttps://gitee.com/wu_fengguang/compass-ci/blob/master/doc/bisect_r…" + pkgbuild_repo_url = "PKGBUILD:\n#{prefix_srv}/#{@pkgbuild_repo}\n" + first_bad_commit_job_url = "first bad commit job result directory:\n#{prefix_srv}#{@first_bad_commit_result_root}\n" data = <<~BODY To: #{@to} @@ -38,17 +41,19 @@ class MailBisectResult Hi #{(a)git_commit.author_name}, - url: #{(a)git_commit.url} - commit: #{@commit_id} ("#{(a)git_commit.subject}") - compiler: gcc (GCC) 7.3.0 + We found some error/warning(s) and the first bad commit in the below project: + git url: #{(a)git_commit.url} + git commit: #{@commit_id} ("#{(a)git_commit.subject}") - all errors/warnings (new ones prefixed by >>): + All error/warning(s) (new ones prefixed by >>): #{@all_errors} - reference information: + Reference information: + compiler: gcc (GCC) 7.3.0 #{pkgbuild_repo_url} - #{bisect_job_url} #{first_bad_commit_job_url} + #{bisect_job_url} + #{bisect_report_doc} Regards, Compass CI team BODY @@ -67,6 +72,9 @@ class MailBisectResult 'my_commit_url' => "#{@git_commit.url}/commit/#{@commit_id}" } + account_info = ESQuery.new(index: 'accounts').query_by_id(@to) + return if account_info + apply_account = AutoAssignAccount.new(user_info) apply_account.send_account end -- 2.23.0

1 0

[PATCH lkp-tests] tests/build-pkg: add ${config} judge in common job
by Wang Yong 30 Dec '20

30 Dec '20

if not exist ${config} in general build-pkg job, ${cci_dir} should not be judeged and do not need upload file Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/build-pkg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/build-pkg b/tests/build-pkg index 801649cf4..d356b2f14 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -49,7 +49,9 @@ get_config() upload_config() { local upload_dir="${cci_dir}/${pkgname}" - [ -d "${upload_dir}" ] || return + + [ -n "${config}" ] || return + [ -d "${upload_dir}" ] || return upload_to_target_dir "${upload_dir}" } -- 2.23.0

1 0

[PATCH compass-ci] container/srv-http: import h5ai into container
by Lu Weitao 30 Dec '20

30 Dec '20

[Why] Optimize result page style to make it more modern Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/srv-http/Dockerfile | 16 +++++++++++++--- container/srv-http/build | 7 +++++++ .../srv-http/root/etc/nginx/conf.d/default.conf | 5 ++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/container/srv-http/Dockerfile b/container/srv-http/Dockerfile index 8f9a36d..987e40e 100644 --- a/container/srv-http/Dockerfile +++ b/container/srv-http/Dockerfile @@ -1,15 +1,25 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM nginx:alpine +FROM alpine:3.9 + ADD root / -RUN apk add --no-cache php7 php7-fpm php7-session php7-json php7-exif php7-imagick php7-gd php7-fileinfo +RUN apk add --no-cache nginx php7 php7-fpm php7-session php7-json php7-exif php7-imagick php7-gd php7-fileinfo nodejs npm \ + && mkdir /run/nginx/ \ + && mkdir /h5ai RUN sed -i '/\[global\]/a daemonize = no' /etc/php7/php-fpm.conf RUN sed -i "s/user = nobody/user = nginx/g" /etc/php7/php-fpm.d/www.conf RUN sed -i "s/group = nobody/group = nginx/g" /etc/php7/php-fpm.d/www.conf -RUN chown -R nginx:nginx /srv/_h5ai +WORKDIR /h5ai +COPY h5ai /h5ai + +# build h5ai package +RUN npm install \ + && npm run build \ + && cp -r ./build/_h5ai /srv \ + && chown -R nginx:nginx /srv/_h5ai ENTRYPOINT ["/sbin/entrypoint.sh"] diff --git a/container/srv-http/build b/container/srv-http/build index 3a500ea..bdda319 100755 --- a/container/srv-http/build +++ b/container/srv-http/build @@ -2,4 +2,11 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +[[ $H5AI_SRC ]] || H5AI_SRC=/c/h5ai + +cp -r $H5AI_SRC ./ + docker build -t srv-http . + +# clean h5ai foler +rm -rf ./h5ai diff --git a/container/srv-http/root/etc/nginx/conf.d/default.conf b/container/srv-http/root/etc/nginx/conf.d/default.conf index 676418a..646eee3 100644 --- a/container/srv-http/root/etc/nginx/conf.d/default.conf +++ b/container/srv-http/root/etc/nginx/conf.d/default.conf @@ -3,8 +3,8 @@ server { server_name "srv-http"; server_tokens off; - root /usr/share/nginx/html; - index index.html; + root /srv; + index index.html index.php /_h5ai/public/index.php; charset utf-8,gbk; @@ -24,7 +24,6 @@ server { autoindex on; autoindex_localtime on; allow all; - root /srv; location ~* /[-_a-z]+$ { add_header Content-Type text/plain; } -- 2.23.0

1 1

Re: [PATCH v3 compass-ci] providers/qemu: refactor kvm script
by Ren Wen 30 Dec '20

30 Dec '20

>>+ if [ ! -f "$log_file" ]; then >>>+ touch $log_file >>>+ # fluentd refresh time is 1s >>>+ # let fluentd to monitor this file first >>>+ sleep 2 >>>+ fi >> >>how about: [ -f "$log_file" ] || { ... } ? > >there are 2 line codes, if just 1 line code use like you say is good. {} is a way to group a list of commands to be executed as a unit, you can also use like this: [ ] || { expression1 expression2 } Thanks, RenWen > >Thanks, >Yinsi > > >> >>-------- >>Thanks >>Yu Chuan >> >>>+} >>> >>>-qemu=qemu-system-aarch64 >>>-command -v $qemu >/dev/null || qemu=qemu-kvm >>>+qemu_command() >>>+{ >>>+ qemu=qemu-system-aarch64 >>>+ command -v $qemu >/dev/null || qemu=qemu-kvm >>>+} >>> >>>-ipxe_script=ipxe_script >>>-curl http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/boot.ipxe/mac/${mac} > $ipxe_script >>>-cat $ipxe_script >> ${log_file} >>>+write_logfile() >>>+{ >>>+ ipxe_script=ipxe_script >>>+ curl http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/boot.ipxe/mac/${mac} > $ipxe_script >>>+ cat $ipxe_script >> ${log_file} >>>+} >>> >>>-append= >>>-initrds= >>>-while read a b c >>>-do >>>- case "$a" in >>>- '#') >>>- ;; >>>- initrd) >>>- file=$(basename "$b") >>>- wget --timestamping -a ${log_file} --progress=bar:force $b >>>- initrds+="$file " >>>- ;; >>>- kernel) >>>- kernel=$(basename "$b") >>>- wget --timestamping -a ${log_file} --progress=bar:force $b >>>- append=$(echo "$c" | sed -r "s/ initrd=[^ ]+//g") >>>- ;; >>>- *) >>>- ;; >>>- esac >>>-done < $ipxe_script >>>+option_value() >>>+{ >>>+ append= >>>+ initrds= >>>+ while read a b c >>>+ do >>>+ case "$a" in >>>+ '#') >>>+ ;; >>>+ initrd) >>>+ file=$(basename "$b") >>>+ wget --timestamping -a ${log_file} --progress=bar:force $b >>>+ initrds+="$file " >>>+ ;; >>>+ kernel) >>>+ kernel=$(basename "$b") >>>+ wget --timestamping -a ${log_file} --progress=bar:force $b >>>+ append=$(echo "$c" | sed -r "s/ initrd=[^ ]+//g") >>>+ ;; >>>+ *) >>>+ ;; >>>+ esac >>>+ done < $ipxe_script >>>+} >>> >>>-[ -s "$kernel" ] || { >>>- echo "can't get kernel or kernel size is 0" >>>- exit >>>+check_option_value() >>>+{ >>>+ [ -s "$kernel" ] || { >>>+ echo "can't get kernel or kernel size is 0" >>>+ exit >>>+ } >>>+ >>>+ [ -n "$initrds" ] || exit >>>+ >>>+ initrd=initrd >>>+ cat $initrds > $initrd >>> } >>> >>>-[ -n "$initrds" ] || { >>>- exit >>>+print_message() >>>+{ >>>+ echo $SCHED_PORT >>>+ echo kernel: $kernel >>>+ echo initrds: $initrds >>>+ echo append: $append >>>+ echo less $log_file >>>+ >>>+ sleep 5 >>> } >>> >>>-initrd=initrd >>>-cat $initrds > $initrd >>>+run_qemu() >>>+{ >>>+ kvm=( >>>+ $qemu >>>+ -machine virt-4.0,accel=kvm,gic-version=3 >>>+ -kernel $kernel >>>+ -initrd $initrd >>>+ -smp $nr_cpu >>>+ -m $memory >>>+ -cpu Kunpeng-920 >>>+ -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd >>>+ -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} >>>+ -k en-us >>>+ -no-reboot >>>+ -nographic >>>+ -serial file:${log_file} >>>+ -monitor null >>>+ ) >>>+ >>>+ "${kvm[@]}" --append "${append}" >>>+} >>> >>>-echo $SCHED_PORT >>>-echo kernel: $kernel >>>-echo initrds: $initrds >>>-echo append: $append >>>-echo less $log_file >>>+check_logfile >>>+write_logfile >>> >>>-sleep 5 >>>+qemu_command >>>+option_value >>>+check_option_value >>> >>>-kvm=( >>>- $qemu >>>- -machine virt-4.0,accel=kvm,gic-version=3 >>>- -kernel $kernel >>>- -initrd $initrd >>>- -smp $nr_cpu >>>- -m $memory >>>- -cpu Kunpeng-920 >>>- -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd >>>- -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} >>>- -k en-us >>>- -no-reboot >>>- -nographic >>>- -serial file:${log_file} >>>- -monitor null >>>-) >>>+print_message >>> >>>-"${kvm[@]}" --append "${append}" >>>+run_qemu >>>-- >>>2.23.0 >>>

2 1

[PATCH v3 compass-ci] lib/es_query.rb: get multi-field aggregation count result
by Lu Kaiyi 30 Dec '20

30 Dec '20

[why] get multi-field aggregation count for overview of report page. [input]: fields, query_items (optional for query_items, default no scope limitation) [example] input: fields = ['suite'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186214}, {"key"=>"cci-depends", "doc_count"=>1230}, {"key"=>"iperf", "doc_count"=>1103}, {"key"=>"spinlock", "doc_count"=>1000}, {"key"=>"cci-makepkg", "doc_count"=>949}, {"key"=>"iperf-walk-os-test", "doc_count"=>474}, {"key"=>"bisect", "doc_count"=>381}, ... input: fields = ['suite', 'job_state'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186227, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3830}, {"key"=>"finished", "doc_count"=>803}, {"key"=>"incomplete", "doc_count"=>196}, {"key"=>"submit", "doc_count"=>136}, ... input: fields = ['os', 'os_version', 'job_state'] query_items = {} output: [{"key"=>"openeuler", "doc_count"=>193287, "all_os_version"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"20.03", "doc_count"=>191789, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3909}, {"key"=>"finished", "doc_count"=>2401}, {"key"=>"incomplete", "doc_count"=>300}, {"key"=>"submit", "doc_count"=>136}, {"key"=>"OOM", "doc_count"=>23}, {"key"=>"post_run", "doc_count"=>1}]}}, {"key"=>"20.03-SP1", "doc_count"=>904, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>641}, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/lib/es_query.rb b/lib/es_query.rb index 88a420f..29d50e9 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -87,6 +87,50 @@ class ESQuery result end + + # input: + # fields, query_items + # + # fields => fields: Array(keyword) + # eg: + # ['suite', 'job_state'] + # + # query_items => query_items: Hash(keyword, value) + # eg: + # { + # 'os' => 'openeuler', + # ... + # } + # (optional for query_items, default no scope limitation) + # output: + # [{"key"=>"build-pkg", + # "doc_count"=>186175, + # "all_job_state"=> + # {"doc_count_error_upper_bound"=>0, + # "sum_other_doc_count"=>0, + # "buckets"=> + # [{"key"=>"failed", "doc_count"=>3830}, + # {"key"=>"finished", "doc_count"=>803}, + # {"key"=>"incomplete", "doc_count"=>196}, + # ... + def query_fields(fields, query_items = {}) + field1 = fields.first + aggs_hash = {} + build_aggs_from_fields(fields, aggs_hash) + query = { + query: { + bool: { + must: build_multi_field_subquery_body(query_items) + } + }, + aggs: aggs_hash['aggs'], + size: 0 + } + result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] + return nil if result.empty? + + result + end end # Range Query Example: @@ -146,3 +190,30 @@ def assign_desc_body(keyword) }] } end + +# input: +# fields = ['os', 'os_version', 'job_state'] +# aggs_hash = {} +# output: +# aggs_hash = { +# "aggs"=> +# {"all_os"=> +# {"terms"=>{:field=>"os", :size=>1000}, +# "aggs"=> +# {"all_os_version"=> +# {"terms"=>{:field=>"os_version", :size=>1000}, +# "aggs"=> +# {"all_job_state"=>{"terms"=>{:field=>"job_state", :size=>1000}}}}}}} +# } +def build_aggs_from_fields(fields, aggs_hash) + return if fields.empty? + + field = fields.shift + aggs_hash['aggs'] = { + "all_#{field}" => { + 'terms' => { field: field, size: 1000 } + } + } + + build_aggs_from_fields(fields, aggs_hash['aggs']["all_#{field}"]) +end -- 2.23.0

1 0

[PATCH v3 compass-ci] providers/qemu: refactor kvm script
by Liu Yinsi 30 Dec '20

30 Dec '20

encapsulating into functions. test: /srv/result/host-info/2020-12-30/vm-2p8g/openeuler-20.09-aarch64/z9.275242 Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 156 +++++++++++++++++++++++++----------------- 1 file changed, 93 insertions(+), 63 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index fd921dc..c48c0c7 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -7,78 +7,108 @@ : ${nr_cpu:=1} : ${memory:=1G} -log_file=/srv/cci/serial/logs/${hostname} -if [ ! -f "$log_file" ]; then - touch $log_file - # fluentd refresh time is 1s - # let fluentd to monitor this file first - sleep 2 -fi +check_logfile() +{ + log_file=/srv/cci/serial/logs/${hostname} + if [ ! -f "$log_file" ]; then + touch $log_file + # fluentd refresh time is 1s + # let fluentd to monitor this file first + sleep 2 + fi +} -qemu=qemu-system-aarch64 -command -v $qemu >/dev/null || qemu=qemu-kvm +qemu_command() +{ + qemu=qemu-system-aarch64 + command -v $qemu >/dev/null || qemu=qemu-kvm +} -ipxe_script=ipxe_script -curl http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/boot.ipxe/mac/${mac} > $ipxe_script -cat $ipxe_script >> ${log_file} +write_logfile() +{ + ipxe_script=ipxe_script + curl http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/boot.ipxe/mac/${mac} > $ipxe_script + cat $ipxe_script >> ${log_file} +} -append= -initrds= -while read a b c -do - case "$a" in - '#') - ;; - initrd) - file=$(basename "$b") - wget --timestamping -a ${log_file} --progress=bar:force $b - initrds+="$file " - ;; - kernel) - kernel=$(basename "$b") - wget --timestamping -a ${log_file} --progress=bar:force $b - append=$(echo "$c" | sed -r "s/ initrd=[^ ]+//g") - ;; - *) - ;; - esac -done < $ipxe_script +option_value() +{ + append= + initrds= + while read a b c + do + case "$a" in + '#') + ;; + initrd) + file=$(basename "$b") + wget --timestamping -a ${log_file} --progress=bar:force $b + initrds+="$file " + ;; + kernel) + kernel=$(basename "$b") + wget --timestamping -a ${log_file} --progress=bar:force $b + append=$(echo "$c" | sed -r "s/ initrd=[^ ]+//g") + ;; + *) + ;; + esac + done < $ipxe_script +} -[ -s "$kernel" ] || { - echo "can't get kernel or kernel size is 0" - exit +check_option_value() +{ + [ -s "$kernel" ] || { + echo "can't get kernel or kernel size is 0" + exit + } + + [ -n "$initrds" ] || exit + + initrd=initrd + cat $initrds > $initrd } -[ -n "$initrds" ] || { - exit +print_message() +{ + echo $SCHED_PORT + echo kernel: $kernel + echo initrds: $initrds + echo append: $append + echo less $log_file + + sleep 5 } -initrd=initrd -cat $initrds > $initrd +run_qemu() +{ + kvm=( + $qemu + -machine virt-4.0,accel=kvm,gic-version=3 + -kernel $kernel + -initrd $initrd + -smp $nr_cpu + -m $memory + -cpu Kunpeng-920 + -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd + -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} + -k en-us + -no-reboot + -nographic + -serial file:${log_file} + -monitor null + ) + + "${kvm[@]}" --append "${append}" +} -echo $SCHED_PORT -echo kernel: $kernel -echo initrds: $initrds -echo append: $append -echo less $log_file +check_logfile +write_logfile -sleep 5 +qemu_command +option_value +check_option_value -kvm=( - $qemu - -machine virt-4.0,accel=kvm,gic-version=3 - -kernel $kernel - -initrd $initrd - -smp $nr_cpu - -m $memory - -cpu Kunpeng-920 - -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd - -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} - -k en-us - -no-reboot - -nographic - -serial file:${log_file} - -monitor null -) +print_message -"${kvm[@]}" --append "${append}" +run_qemu -- 2.23.0

3 4

[PATCH v2 compass-ci] lib/es_query.rb: get multi-field aggregation count result
by Lu Kaiyi 30 Dec '20

30 Dec '20

[why] get multi-field aggregation count for overview of report page. [input]: fields, query_items (optional for query_items, default no scope limitation) [example1] input: fields = ['suite'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186214}, {"key"=>"cci-depends", "doc_count"=>1230}, {"key"=>"iperf", "doc_count"=>1103}, {"key"=>"spinlock", "doc_count"=>1000}, {"key"=>"cci-makepkg", "doc_count"=>949}, {"key"=>"iperf-walk-os-test", "doc_count"=>474}, {"key"=>"bisect", "doc_count"=>381}, ... [example2] input: fields = ['suite', 'job_state'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186227, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3830}, {"key"=>"finished", "doc_count"=>803}, {"key"=>"incomplete", "doc_count"=>196}, {"key"=>"submit", "doc_count"=>136}, ... [example3] input: fields = ['os', 'os_version', 'job_state'] query_items = {} output: [{"key"=>"openeuler", "doc_count"=>193287, "all_os_version"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"20.03", "doc_count"=>191789, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3909}, {"key"=>"finished", "doc_count"=>2401}, {"key"=>"incomplete", "doc_count"=>300}, {"key"=>"submit", "doc_count"=>136}, {"key"=>"OOM", "doc_count"=>23}, {"key"=>"post_run", "doc_count"=>1}]}}, {"key"=>"20.03-SP1", "doc_count"=>904, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>641}, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 52 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/lib/es_query.rb b/lib/es_query.rb index 88a420f..a9b651e 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -87,6 +87,45 @@ class ESQuery result end + + # select field1, field2, count(*) from index where query_items group by field1, field2 + # example: + # fields = ['suite', 'job_state'] + # query_items = { + # 'os' => 'openeuler' + # } + # input: + # fields, query_items + # (optional for query_items, default no scope limitation) + # output: + # [{"key"=>"build-pkg", + # "doc_count"=>186175, + # "all_job_state"=> + # {"doc_count_error_upper_bound"=>0, + # "sum_other_doc_count"=>0, + # "buckets"=> + # [{"key"=>"failed", "doc_count"=>3830}, + # {"key"=>"finished", "doc_count"=>803}, + # {"key"=>"incomplete", "doc_count"=>196}, + # ... + def query_fields(fields, query_items = {}) + field1 = fields.first + aggs_hash = {} + build_aggs_from_fields(fields, aggs_hash) + query = { + query: { + bool: { + must: build_multi_field_subquery_body(query_items) + } + }, + aggs: aggs_hash['aggs'], + size: 0 + } + result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] + return nil if result.empty? + + result + end end # Range Query Example: @@ -146,3 +185,16 @@ def assign_desc_body(keyword) }] } end + +def build_aggs_from_fields(fields, aggs_hash) + return if fields.empty? + + field = fields.shift + aggs_hash['aggs'] = { + "all_#{field}" => { + 'terms' => { field: field, size: 1000 } + } + } + + build_aggs_from_fields(fields, aggs_hash['aggs']["all_#{field}"]) +end -- 2.23.0

2 1