- Compass-ci - mailweb.openeuler.org

[PATCH v5 lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 18 Mar '21

18 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 79 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100755 tests/ansible_test diff --git a/tests/ansible_test b/tests/ansible_test new file mode 100755 index 000000000..56926bbed --- /dev/null +++ b/tests/ansible_test @@ -0,0 +1,79 @@ +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] || { + echo "Ansible failed to install." + exit 1 + } + } +} + +download_ansible_role() +{ + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git + if [ $? -eq 0 ]; then + echo "$benchmark is successfully cloned" + else + echo "Please check whether your ansible_repo is correct!" + exit 1 + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: localhost + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook is running failed!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

1 0

[PATCH v2 lkp-tests] tests/openeuler_docker: fix to build openeuler image error
by Liu Shaofei 18 Mar '21

18 Mar '21

^[[91m+ rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 ^[[0m^[[91merror: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7: import read failed(2). ^[[0m Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- tests/openeuler_docker | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/openeuler_docker b/tests/openeuler_docker index b0b5336a3..483f40135 100755 --- a/tests/openeuler_docker +++ b/tests/openeuler_docker @@ -78,6 +78,11 @@ handle_epelrpm() } } +handle_rpm_gpg() +{ + sed -i "s|RPM-GPG-KEY-CentOS-7|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" +} + add_base_commands() { # grep -qw "groupadd*" "$file" && { @@ -95,6 +100,7 @@ handle_dockerfile() is_openeuler_build_fail=1 handle_FROM handle_epel + handle_rpm_gpg add_base_commands if docker_build "-openeuler"; then docker_run "-openeuler" -- 2.23.0

1 0

[PATCH lkp-tests] tests/openeuler_docker: fix to build openeuler image error
by Liu Shaofei 18 Mar '21

18 Mar '21

^[[91m+ rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 ^[[0m^[[91merror: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7: import read failed(2). ^[[0m Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- tests/openeuler_docker | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/openeuler_docker b/tests/openeuler_docker index b0b5336a3..139b5eb56 100755 --- a/tests/openeuler_docker +++ b/tests/openeuler_docker @@ -78,6 +78,13 @@ handle_epelrpm() } } +handle_rpm_gpg() +{ + grep -qw "RPM-GPG-KEY-CentOS-7" "$dockerfile_name" && { + sed -i "s|RPM-GPG-KEY-CentOS-7|PM-GPG-KEY-openEuler|g" "$dockerfile_name" + } +} + add_base_commands() { # grep -qw "groupadd*" "$file" && { @@ -95,6 +102,7 @@ handle_dockerfile() is_openeuler_build_fail=1 handle_FROM handle_epel + handle_rpm_gpg add_base_commands if docker_build "-openeuler"; then docker_run "-openeuler" -- 2.23.0

3 4

[PATCH compass-ci] fix network problem: add domestic resource url for Debian packages
by Luan Shengde 18 Mar '21

18 Mar '21

[why] the default url for Debian package resources may lead to download timeout problems: E: Release file for http://security.debian.org/debian-security/dists/buster/updates/InRelease is expired (invalid since 9h 8min 10s). Updates for this repository will not be applied. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/Dockerfile | 2 ++ container/mail-robot/conf/sources.list | 9 +++++++++ container/send-internet-mail/Dockerfile | 2 ++ container/send-internet-mail/conf/sources.list | 9 +++++++++ container/send-mail/Dockerfile | 2 ++ container/send-mail/conf/sources.list | 9 +++++++++ 6 files changed, 33 insertions(+) create mode 100644 container/mail-robot/conf/sources.list create mode 100644 container/send-internet-mail/conf/sources.list create mode 100644 container/send-mail/conf/sources.list diff --git a/container/mail-robot/Dockerfile b/container/mail-robot/Dockerfile index 4402543..e4b3e7c 100644 --- a/container/mail-robot/Dockerfile +++ b/container/mail-robot/Dockerfile @@ -7,6 +7,8 @@ MAINTAINER Luan Shengde <shdluan(a)163.com> ENV DEBIAN_FRONTEND noninteractive +COPY conf/sources.list* /etc/apt/ + RUN apt-get update && \ apt-get install -y git uuid-runtime ruby-listen ruby-json ruby-mail ruby-rest-client diff --git a/container/mail-robot/conf/sources.list b/container/mail-robot/conf/sources.list new file mode 100644 index 0000000..8179e01 --- /dev/null +++ b/container/mail-robot/conf/sources.list @@ -0,0 +1,9 @@ +deb [trusted=yes] http://mirrors.163.com/debian/ buster main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian/ buster-updates main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian/ buster-backports main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian-security/ buster/updates main non-free contrib + +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster-updates main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster-backports main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian-security/ buster/updates main non-free contrib diff --git a/container/send-internet-mail/Dockerfile b/container/send-internet-mail/Dockerfile index 29f1bc6..fa9ec2c 100644 --- a/container/send-internet-mail/Dockerfile +++ b/container/send-internet-mail/Dockerfile @@ -5,5 +5,7 @@ FROM debian MAINTAINER Luan Shengde <shdluan(a)163.com> ENV DEBIAN_FRONTEND noninteractive +COPY conf/sources.list* /etc/apt/ + RUN apt-get update && \ apt-get install -y ruby-mail ruby-json ruby-sinatra ruby-redis ruby-rufus-scheduler diff --git a/container/send-internet-mail/conf/sources.list b/container/send-internet-mail/conf/sources.list new file mode 100644 index 0000000..8179e01 --- /dev/null +++ b/container/send-internet-mail/conf/sources.list @@ -0,0 +1,9 @@ +deb [trusted=yes] http://mirrors.163.com/debian/ buster main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian/ buster-updates main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian/ buster-backports main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian-security/ buster/updates main non-free contrib + +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster-updates main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster-backports main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian-security/ buster/updates main non-free contrib diff --git a/container/send-mail/Dockerfile b/container/send-mail/Dockerfile index 29f1bc6..fa9ec2c 100644 --- a/container/send-mail/Dockerfile +++ b/container/send-mail/Dockerfile @@ -5,5 +5,7 @@ FROM debian MAINTAINER Luan Shengde <shdluan(a)163.com> ENV DEBIAN_FRONTEND noninteractive +COPY conf/sources.list* /etc/apt/ + RUN apt-get update && \ apt-get install -y ruby-mail ruby-json ruby-sinatra ruby-redis ruby-rufus-scheduler diff --git a/container/send-mail/conf/sources.list b/container/send-mail/conf/sources.list new file mode 100644 index 0000000..8179e01 --- /dev/null +++ b/container/send-mail/conf/sources.list @@ -0,0 +1,9 @@ +deb [trusted=yes] http://mirrors.163.com/debian/ buster main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian/ buster-updates main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian/ buster-backports main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian-security/ buster/updates main non-free contrib + +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster-updates main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster-backports main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian-security/ buster/updates main non-free contrib -- 2.23.0

1 0

[PATCH compass-ci] dracut-initrd: add xz-utils
by Wang Chenglong 18 Mar '21

18 Mar '21

[Why] wcl@crystal ~/compass-ci/container/dracut-initrd% ./run /srv/os/centos/x86_64/6.10/lib/modules/3.10.0-957.1.3.el7.x86_64 dracut: Executing: /usr/bin/dracut -o iscsi --omit-drivers iscsi --force --kver 3.10.0-957.1.3.el7.x86_64 -k /lib/modules/3.10.0-957.1.3.el7.x86_64 /boot/initramfs.lkp-3.10.0-957.1.3.el7.x86_64.img dracut: Kernel modules are compressed with xz, but xz is not available. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 616183f..b8eb6f5 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -6,7 +6,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y --fix-missing --no-install-recommends -qq apt-utils \ -nfs-common netbase cifs-utils kmod rsync dracut dracut-network \ +nfs-common netbase cifs-utils kmod rsync dracut dracut-network xz-utils \ dracut-config-generic apt-get clean -- 2.23.0

2 1

[PATCH v3 lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 18 Mar '21

18 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 47 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..d6acad1ab 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,40 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + elsif error_msg.is_a? String + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/[^\w]/, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line.gsub!(/^-|-$/, '') + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

1 0

[PATCH compass-ci] container/crystal-shards: replace https:// with git:// for building image
by Cao Xueliang 17 Mar '21

17 Mar '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/crystal-shards/Dockerfile | 7 +++++-- container/crystal-shards/git | 7 +++++++ 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100755 container/crystal-shards/git diff --git a/container/crystal-shards/Dockerfile b/container/crystal-shards/Dockerfile index 2311554..15fe523 100644 --- a/container/crystal-shards/Dockerfile +++ b/container/crystal-shards/Dockerfile @@ -5,7 +5,10 @@ FROM alpine:crystal-base MAINTAINER chief <taxcom(a)tom.com> -copy shard.yml /usr/share/crystal/app/shard.yml +COPY git /usr/local/sbin/git +RUN echo -e "[url \"git://\"]\n\tinsteadOf = https://" >> /etc/gitconfig + +COPY shard.yml /usr/share/crystal/app/shard.yml WORKDIR /usr/share/crystal/app RUN shards @@ -14,7 +17,7 @@ sed -i '99s/arguments\[:id]/arguments\[:id]?/' /usr/share/crystal/app/lib/elast sed -i 's/, Utils.__listify(arguments\[:q].as(String))/ /' /usr/share/crystal/app/lib/elasticsearch-crystal/src/elasticsearch/api/actions/search.cr;\ sed -i '205a \ params.clear' /usr/share/crystal/app/lib/elasticsearch-crystal/src/elasticsearch/api/actions/search.cr; -copy shard-amqp.yml /usr/share/crystal/app/shard.yml +COPY shard-amqp.yml /usr/share/crystal/app/shard.yml RUN shards CMD ["bash"] diff --git a/container/crystal-shards/git b/container/crystal-shards/git new file mode 100755 index 0000000..7bb0b85 --- /dev/null +++ b/container/crystal-shards/git @@ -0,0 +1,7 @@ +#!/bin/bash + +if [[ "$*" = "ls-remote --get-url origin" ]]; then + /usr/bin/git "$@" | sed 's|git://|https://|g' +else + /usr/bin/git "$@" +fi -- 2.23.0

1 0

[PATCH compass-ci] container/crystal-shards: delete gitcache host and port
by Cao Xueliang 17 Mar '21

17 Mar '21

The gitcache service cannot be used to maintain branches when build crystal-shards images. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/crystal-shards/Dockerfile | 4 ---- container/crystal-shards/build | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/container/crystal-shards/Dockerfile b/container/crystal-shards/Dockerfile index bdc5b66..2311554 100644 --- a/container/crystal-shards/Dockerfile +++ b/container/crystal-shards/Dockerfile @@ -5,10 +5,6 @@ FROM alpine:crystal-base MAINTAINER chief <taxcom(a)tom.com> -ARG GITCACHE_HOST -ARG GITCACHE_PORT -RUN [ -z "$GITCACHE_HOST" ] || echo -e "[url \"http://$GITCACHE_HOST:${GITCACHE_PORT:-5000}/\"]\n\tinsteadOf = https://" >> /etc/gitconfig - copy shard.yml /usr/share/crystal/app/shard.yml WORKDIR /usr/share/crystal/app diff --git a/container/crystal-shards/build b/container/crystal-shards/build index 7c2a712..00153ab 100755 --- a/container/crystal-shards/build +++ b/container/crystal-shards/build @@ -11,4 +11,4 @@ docker_skip_rebuild "alpine:crystal-shards" load_cci_defaults -docker build --build-arg GITCACHE_HOST=$GITCACHE_HOST --build-arg GITCACHE_PORT=$GITCACHE_PORT -t alpine:crystal-shards . +docker build -t alpine:crystal-shards . -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] src/lib/web_backend.rb: implement for group jobs stats count
by Li Yuanchao 17 Mar '21

17 Mar '21

Query jobs by conditions such as group id or suite, and count the number of passed cases and failed cases. output is like: { "kezhiming" : { "all" : $all, "nr_pass" : $nr_pass, "nr_fail" : $nr_fali }, "chenqun" : { "all" : $all, "nr_pass" : $nr_pass, "nr_fail" : $nr_fali }, ... } Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 66 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 89c0f3c..4a5a51f 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -548,3 +548,69 @@ def new_refs_statistics(params) end [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] end + +def single_count(stats) + fail_count = 0 + pass_count = 0 + single_nr_fail = 0 + single_nr_pass = 0 + stats.each do |stat, value| + fail_count += 1 if stat.match(/\.fail$/i) + pass_count += 1 if stat.match(/\.pass$/i) + single_nr_fail = value if stat.match(/\.nr_fail$/i) + single_nr_pass = value if stat.match(/\.nr_pass$/i) + end + fail_count = single_nr_fail.zero? ? fail_count : single_nr_fail + pass_count = single_nr_pass.zero? ? pass_count : single_nr_pass + [fail_count, pass_count, fail_count + pass_count] +end + +def count_stats(job_list) + nr_fail = 0 + nr_pass = 0 + all = 0 + job_list.each do |job| + next unless job['_source']['stats'] + + fail_count, pass_count, all_count = single_count(job['_source']['stats']) + nr_fail += fail_count + nr_pass += pass_count + all += all_count + end + { 'nr_fail' => nr_fail, 'nr_pass' => nr_pass, 'all' => all } +end + +def get_jobs_stats_count(dimension, must, size, from) + dimension_list = get_dimension_list(dimension) + stats_count = {} + dimension_list.each do |dim| + job_list = query_dimension(dimension[0], dim, must, size, from) + stats_count[dim] = count_stats(job_list) + end + stats_count.to_json +end + +def get_stats_by_dimension(conditions, dimension, must, size, from) + must += build_multi_field_subquery_body(conditions) + count_query = { query: { bool: { must: must } } } + total = es_count(count_query) + return {} if total < 1 + + get_jobs_stats_count(dimension, must, size, from) +end + +def get_jobs_stats(params) + dimension, conditions = get_dimension_conditions(params) + must = get_es_must(params) + get_stats_by_dimension(conditions, dimension, must, 1000, 0) +end + +def group_jobs_stats(params) + begin + body = get_jobs_stats(params) + rescue StandardError => e + warn e.message + return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'group jobs table error'] + end + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] +end -- 2.23.0

1 0

[PATCH compass-ci 2/2] src/lib/web_backend.rb: implement for group jobs stats count
by Li Yuanchao 17 Mar '21

17 Mar '21

Query jobs by conditions such as group id or suite, and count the number of passed cases and failed cases. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 66 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 89c0f3c..ea45d59 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -548,3 +548,69 @@ def new_refs_statistics(params) end [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] end + +def single_count(stats) + fail_count = 0 + pass_count = 0 + single_nr_fail = 0 + single_nr_pass = 0 + stats.each do |stat, value| + fail_count += 1 if stat.match(/\.fail$/i) + pass_count += 1 if stat.match(/\.pass$/i) + single_nr_fail = value if stat.match(/\.nr_fail$/i) + single_nr_pass = value if stat.match(/\.nr_pass$/i) + end + fail_count = single_nr_fail.zero? ? fail_count : single_nr_fail + pass_count = single_nr_pass.zero? ? pass_count : single_nr_pass + [fail_count, pass_count, fail_count + pass_count] +end + +def count_stats(job_list) + nr_fail = 0 + nr_pass = 0 + all = 0 + job_list.each do |job| + next unless job['_source']['stats'] + + fail_count, pass_count, all_count = single_count(job['_source']['stats']) + nr_fail += fail_count + nr_pass += pass_count + all += all_count + end + { 'nr_fail' => nr_fail, 'nr_pass' => nr_pass, 'all' => all } +end + +def get_jobs_stats_count(dimension, must, size, from) + dimension_list = get_dimension_list(dimension) + stats_count = {} + dimension_list.each do |dim| + job_list = query_dimension(dimension[0], dim, must, size, from) + stats_count[dim] = count_stats(job_list) + end + stats_count.to_json +end + +def get_stats_by_dimension(conditions, dimension, must, size, from) + must += build_multi_field_subquery_body(conditions) + count_query = { query: { bool: { must: must } } } + total = es_count(count_query) + return {} if total < 1 + + get_jobs_stats_count(dimension, must, size, from) +end + +def get_jobs_stats(params) + dimension, conditions = get_dimension_conditions(params) + must = get_es_must(params) + get_stats_by_dimension(conditions, dimension, must, 1000, 0) +end + +def group_jobs_table(params) + begin + body = get_jobs_stats(params) + rescue StandardError => e + warn e.message + return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'group jobs table error'] + end + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] +end -- 2.23.0

2 2

[PATCH compass-ci 1/2] container/web-backend: add api for group jobs stats count
by Li Yuanchao 17 Mar '21

17 Mar '21

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/web-backend/web-backend | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 33a3fae..131cff0 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -146,3 +146,28 @@ end get '/get_repo_statistics' do new_refs_statistics(params) end + +# GET /get_group_jobs_table?suite=iperf&dimension=my_name +# must: +# - query_conditions +# - suite / group_id / ... +# - dimension +# - group_id / my_email / my_name +# +# Response like: +# - { +# "kezhiming" : { +# "all" : $all, +# "nr_pass" : $nr_pass, +# "nr_fail" : $nr_fali +# }, +# "chenqun" : { +# "all" : $all, +# "nr_pass" : $nr_pass, +# "nr_fail" : $nr_fali +# }, +# ... +# } +get '/get_group_jobs_table' do + group_jobs_table(params) +end -- 2.23.0

2 2

[PATCH compass-ci] sparrow/3-code/dev-env: set $lab default value
by Liu Yinsi 17 Mar '21

17 Mar '21

before: $lab always is nolab after: if env $lab already exists, use env value. it can make code more flexible. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 6072ce8..e182b65 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -4,6 +4,7 @@ [ -z $server_ip ] && server_ip=$(ip route get 1.2.3.4 | awk '{print $7; exit}') server_name=$(hostname | cut -f1 -d.) +: ${lab:=nolab} : ${sched_host:=$server_ip} : ${sched_port:=3000} : ${ES_HOST:=$server_ip} @@ -38,7 +39,7 @@ mkdir -p /etc/compass-ci/defaults cat > /etc/compass-ci/defaults/$server_name.yaml <<EOF SCHED_HOST: $sched_host SCHED_PORT: $sched_port -lab: nolab +lab: $lab EOF mkdir -p /etc/compass-ci/service -- 2.23.0

1 0

[PATCH lkp-tests] tests/deploy-ci: delete env GITCACHE_HOST and GITCACHE_PORT
by Liu Yinsi 17 Mar '21

17 Mar '21

[why] gitcache service cannot be used to maintain branches when build compass-ci/container/crystal-shards images, so delete. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- tests/deploy-cci | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/deploy-cci b/tests/deploy-cci index 8e6acbac4..6f97d7151 100755 --- a/tests/deploy-cci +++ b/tests/deploy-cci @@ -6,8 +6,6 @@ export_env() { export initrd_http_host=$INITRD_HTTP_HOST export initrd_http_port=$INITRD_HTTP_PORT - export gitcache_host=$GITCACHE_HOST - export gitcache_port=$GITCACHE_PORT } deploy() -- 2.23.0

1 0

[PATCH v2 compass-ci] result-webdav-webdav/nginx.conf: do not overwrite the uploaded config with the same name
by Lin Jiaxin 17 Mar '21

17 Mar '21

Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- container/result-webdav/nginx.conf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/container/result-webdav/nginx.conf b/container/result-webdav/nginx.conf index cc21926..e0e0ee2 100644 --- a/container/result-webdav/nginx.conf +++ b/container/result-webdav/nginx.conf @@ -27,6 +27,22 @@ http { dav_methods PUT MKCOL; dav_access user:rw group:rw all:rw; + set $config_upload_condition "false"; + if ( $uri ~ "^/cci/build-config.*" ) + { + set $config_upload_condition "true"; + } + + if ( -e $request_filename ) + { + set $config_upload_condition "${config_upload_condition}+true"; + } + + if ( $config_upload_condition = "true+true" ) + { + return 200; + } + access_by_lua_block { function split(str, reps) local resultstrlist = {} -- 2.23.0

1 0

[PATCH v2 lkp-tests] rpmbuild for openeuler
by Li Ping 17 Mar '21

17 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- jobs/rpmbuild-pkg.yaml | 2 +- tests/rpmbuild-pkg | 47 +++++++++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/jobs/rpmbuild-pkg.yaml b/jobs/rpmbuild-pkg.yaml index 53e145b3b..a70590afe 100644 --- a/jobs/rpmbuild-pkg.yaml +++ b/jobs/rpmbuild-pkg.yaml @@ -1,5 +1,5 @@ suite: rpmbuild-pkg category: functional -rpm_repo: +upstream_repo: rpmbuild-pkg: diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index b641e97d1..86b775334 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -1,12 +1,16 @@ #!/bin/bash -# - rpm_repo +# - upstream_repo +# - compat_os . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh -[ -n "$rpm_repo" ] || die "rpm_repo is empty" -package_name=${rpm_repo##*/} -sync_dest="/initrd/rpmbuild-pkg/${os}/${os_arch}/${os_version}/${package_name}" +: "${compat_os:=bud-openeuler}" + +[ -n "$upstream_repo" ] || die "upstream_repo is empty" +package_name=${upstream_repo##*/} +rpm_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compat_os}/${os_arch}/Packages" +src_rpm_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compat_os}/source/Packages" init_workspace() { @@ -14,21 +18,16 @@ init_workspace() rpmdev-setuptree } -get_pkgfile() +download_upstream_repo() { - curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-show", "HEAD:'$1'"]}' -o "${2}" -} - -download_rpm_repo() -{ - local filelist=$(curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-ls-files", "."]}') - for pkgfile in "${filelist[@]}" + git clone "git://$GIT_SERVER/openeuler/${upstream_repo}" || die "clone git repo ${package_name} failed: git://$GIT_SERVER/openeuler/${upstream_repo}" + cd $package_name || exit + filelist=$(git ls-files) + for pkgfile in ${filelist[@]} do local dir="SOURCES" echo $pkgfile | egrep "\.spec$" && dir="SPECS" - get_pkgfile "$pkgfile" "${HOME}/rpmbuild/${dir}/$pkgfile" + mv "$pkgfile" "${HOME}/rpmbuild/${dir}/" done } @@ -50,19 +49,29 @@ build_rpm() show_rpm_files() { find ${HOME}/rpmbuild/RPMS -type f -name "*.rpm" +} + +show_src_rpm_files() +{ find ${HOME}/rpmbuild/SRPMS -type f -name "*.rpm" } upload_rpm_pkg() { - local file - for file in $(show_rpm_files) + local rpm_file + for rpm_file in $(show_rpm_files) + do + upload_one_curl ${rpm_file} ${rpm_dest} + done + + local src_rpm_file + for src_rpm_file in $(show_src_rpm_files) do - upload_one_curl ${file} ${sync_dest} + upload_one_curl ${src_rpm_file} ${src_rpm_dest} done } init_workspace -download_rpm_repo +download_upstream_repo build_rpm upload_rpm_pkg -- 2.23.0

1 1

[PATCH v3 lkp-tests] rpmbuild for openeuler
by Li Ping 17 Mar '21

17 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- jobs/rpmbuild-pkg.yaml | 2 +- tests/rpmbuild-pkg | 47 +++++++++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/jobs/rpmbuild-pkg.yaml b/jobs/rpmbuild-pkg.yaml index 53e145b3b..a70590afe 100644 --- a/jobs/rpmbuild-pkg.yaml +++ b/jobs/rpmbuild-pkg.yaml @@ -1,5 +1,5 @@ suite: rpmbuild-pkg category: functional -rpm_repo: +upstream_repo: rpmbuild-pkg: diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index b641e97d1..256e836bf 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -1,12 +1,16 @@ #!/bin/bash -# - rpm_repo +# - upstream_repo +# - compat_os . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh -[ -n "$rpm_repo" ] || die "rpm_repo is empty" -package_name=${rpm_repo##*/} -sync_dest="/initrd/rpmbuild-pkg/${os}/${os_arch}/${os_version}/${package_name}" +: "${compat_os:=budding-openeuler}" + +[ -n "$upstream_repo" ] || die "upstream_repo is empty" +package_name=${upstream_repo##*/} +rpm_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compat_os}/${os_arch}/Packages" +src_rpm_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compat_os}/source/Packages" init_workspace() { @@ -14,21 +18,16 @@ init_workspace() rpmdev-setuptree } -get_pkgfile() +download_upstream_repo() { - curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-show", "HEAD:'$1'"]}' -o "${2}" -} - -download_rpm_repo() -{ - local filelist=$(curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-ls-files", "."]}') - for pkgfile in "${filelist[@]}" + git clone "git://$GIT_SERVER/openeuler/${upstream_repo}" || die "clone git repo ${package_name} failed: git://$GIT_SERVER/openeuler/${upstream_repo}" + cd $package_name || exit + filelist=$(git ls-files) + for pkgfile in ${filelist[@]} do local dir="SOURCES" echo $pkgfile | egrep "\.spec$" && dir="SPECS" - get_pkgfile "$pkgfile" "${HOME}/rpmbuild/${dir}/$pkgfile" + mv "$pkgfile" "${HOME}/rpmbuild/${dir}/" done } @@ -50,19 +49,29 @@ build_rpm() show_rpm_files() { find ${HOME}/rpmbuild/RPMS -type f -name "*.rpm" +} + +show_src_rpm_files() +{ find ${HOME}/rpmbuild/SRPMS -type f -name "*.rpm" } upload_rpm_pkg() { - local file - for file in $(show_rpm_files) + local rpm_file + for rpm_file in $(show_rpm_files) + do + upload_one_curl ${rpm_file} ${rpm_dest} + done + + local src_rpm_file + for src_rpm_file in $(show_src_rpm_files) do - upload_one_curl ${file} ${sync_dest} + upload_one_curl ${src_rpm_file} ${src_rpm_dest} done } init_workspace -download_rpm_repo +download_upstream_repo build_rpm upload_rpm_pkg -- 2.23.0

1 0

[PATCH lkp-tests] add new error type
by Bai Jing 17 Mar '21

17 Mar '21

[input] Error: Module or Group 'convert' does not exist [output] error.nor-exit-module-or-group.convert: 1 Signed-off-by: Bai Jing <799286817(a)qq.com> --- stats/openeuler_docker | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stats/openeuler_docker b/stats/openeuler_docker index 4c65c6bb3..97cf24d7f 100755 --- a/stats/openeuler_docker +++ b/stats/openeuler_docker @@ -18,6 +18,11 @@ while (line = STDIN.gets) puts "unknown-repo.#{repo}: 1" puts "unknown-repo.#{repo}.message: #{line}" + # Error: Module or Group 'convert' does not exist. + when /Error: Module or Group ('[^\s]+')/ + repo = $1.delete!("'") + puts "error.not-exist-module-or-group.#{repo}: 1" + puts "error.not-exist-module-or-group.#{repo}.message: #{line}" # /bin/sh: passwd: command not found when /\/bin\/sh: (.+): command not found/ puts "sh.command-not-found.#{$1}: 1" -- 2.23.0

1 0

[PATCH compass-ci] container/logging-es: remove redundant code
by Liu Yinsi 17 Mar '21

17 Mar '21

[why] these deleted code optimize the bulk write performance, but affect the real-time performance of data. At now, our fluentd architecture already been optimized, the bulk write is no longer a bottleneck, so delete codes. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/logging-es/start | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/container/logging-es/start b/container/logging-es/start index efd89de..9be7a5b 100755 --- a/container/logging-es/start +++ b/container/logging-es/start @@ -19,14 +19,3 @@ cmd=( ) "${cmd[@]}" - -# set index -check_service_ready 9202 -curl -sSH "Content-Type: application/json" -XPUT "localhost:9202/_settings" -d ' -{ - "index": { - "refresh_interval": "120s", - "translog.durability": "async", - "translog.sync_interval": "120s", - "translog.flush_threshold_size": "1024mb"} -}' -- 2.23.0

1 0

[PATCH compass-ci] dracut-initrd: add xz-utils
by Wang Chenglong 17 Mar '21

17 Mar '21

[Why] wcl@crystal ~/compass-ci/container/dracut-initrd% ./run /srv/os/centos/x86_64/6.10/lib/modules/3.10.0-957.1.3.el7.x86_64 dracut: Executing: /usr/bin/dracut -o iscsi --omit-drivers iscsi --force --kver 3.10.0-957.1.3.el7.x86_64 -k /lib/modules/3.10.0-957.1.3.el7.x86_64 /boot/initramfs.lkp-3.10.0-957.1.3.el7.x86_64.img dracut: Kernel modules are compressed with xz, but xz is not available. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 616183f..b8eb6f5 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -6,7 +6,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y --fix-missing --no-install-recommends -qq apt-utils \ -nfs-common netbase cifs-utils kmod rsync dracut dracut-network \ +nfs-common netbase cifs-utils kmod rsync dracut dracut-network xz-utils \ dracut-config-generic apt-get clean -- 2.23.0

1 0

[PATCH v2 lkp-tests] jobs: add secrets_info.yaml
by Wei Jihui 17 Mar '21

17 Mar '21

secrets_info.yaml is for user use secret field Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- jobs/secrets_info.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 jobs/secrets_info.yaml diff --git a/jobs/secrets_info.yaml b/jobs/secrets_info.yaml new file mode 100644 index 00000000..4026b3b7 --- /dev/null +++ b/jobs/secrets_info.yaml @@ -0,0 +1,6 @@ +# This file is for user add secret field, so that it's not visible from outside(job.yaml), but can be used at testbox. +# Usage: submit job.yaml -i secrets_info.yaml +# Use at testbox: testbox has env with: "secrets_" + secret field, such as $secrets_my_token + +secrets: + my_token: {{my_token}} -- 2.23.0

1 0

[PATCH lkp-tests] jobs: add secrets_info.yaml
by Wei Jihui 17 Mar '21

17 Mar '21

secrets_info.yaml is for user use secret field Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- jobs/secrets_info.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 jobs/secrets_info.yaml diff --git a/jobs/secrets_info.yaml b/jobs/secrets_info.yaml new file mode 100644 index 00000000..196cf5ed --- /dev/null +++ b/jobs/secrets_info.yaml @@ -0,0 +1,6 @@ +# This file is for user add secret field, so that it;s not visible from outside, but can be use at testbox. +# Usage: submit job.yaml -i secrets_info.yaml +# Use at testbox: testbox has env with: "secrets_" + secret field, such as $secrets_my_token + +secrets: + my_token: {{my_token}} -- 2.23.0

2 1

[PATCH compass-ci] sbin/es-jobs: add -e to get fail result
by Wu Zhende 17 Mar '21

17 Mar '21

es-jobs -e group_id=xxxx error.result.path=/srv/result/openeuler_docker/xxxx: ["Error: unable to find a match"] Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/es_jobs.rb | 24 +++++++++++++++++++++++- sbin/es-jobs | 7 ++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index b0d0fb2..e4e9530 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -12,13 +12,14 @@ require_relative './es_query' # deal jobs search from es class ESJobs - def initialize(es_query, my_refine = [], fields = [], stats_filter = []) + def initialize(es_query, my_refine = [], fields = [], stats_filter = [], error_result = false) @es_query = es_query @jobs = query_jobs_from_es @refine = my_refine @fields = fields @stats_filter = stats_filter @refine_jobs = [] + @error_result = error_result set_jobs_summary end @@ -98,6 +99,7 @@ class ESJobs 'max.stats' => {}, 'min.stats' => {} } + result['error.result'] = {} if @error_result metrics.each { |metric| result['raw.stats'][metric] = [] } result end @@ -119,7 +121,27 @@ class ESJobs result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 result['raw.id']["[#{keyword}=#{job[keyword]}]"] ||= [] result['raw.id']["[#{keyword}=#{job[keyword]}]"] << job['id'] + + next unless @error_result && job[keyword] == "failed" + + get_failed_result(result, job) + end + end + + def get_failed_result(result, job) + path = "/srv#{job['result_root']}/#{job['suite']}" + res = find_error_lines(path) + result['error.result']["path=#{path}"] = res + end + + def find_error_lines(path) + res = [] + return res unless File.exist?(path) + + File.open(path).each do |line| + res << line.to_s if line.match(/error:/i) end + res end def stats_count(result) diff --git a/sbin/es-jobs b/sbin/es-jobs index ff0d540..3804dc9 100755 --- a/sbin/es-jobs +++ b/sbin/es-jobs @@ -22,6 +22,7 @@ end opt_refine = [-1] opt_fields = [] opt_stats_filter = [] +opt_error_result = false opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: es-jobs [options] search_key1=val1[,val2..] ..' @@ -45,11 +46,15 @@ opt_parser = OptionParser.new do |opts| opts.on('-s fields', '--stats-filter fields', 'return data contains fields in stats') do |fields| opt_stats_filter = fields.split(',') end + + opts.on('-e', '--error.result', 'obtain the error information about job failure') do + opt_error_result = true + end end opt_parser.parse!(ARGV) items = parse_argv raise 'Please enter a query' if items.empty? -es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter) +es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter, opt_error_result) es_jobs.output -- 2.23.0

1 0

[PATCH compass-ci] container/git-mirror: change mount dir
by Li Yuanchao 17 Mar '21

17 Mar '21

As several repos are supported as upstream-repos, to avoid confusion, change the dir name of /c/upstream-repos to /c/git-repos. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/git-mirror/start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/git-mirror/start b/container/git-mirror/start index ec42595..f01789c 100755 --- a/container/git-mirror/start +++ b/container/git-mirror/start @@ -6,7 +6,7 @@ require_relative '../defconfig.rb' DOCKER_CCI = '/c/compass-ci' -DOCKER_REPO = '/c/upstream-repos' +DOCKER_REPO = '/c/git-repos' docker_rm "git-mirror" lkp = %x(id -u lkp) -- 2.23.0

1 0

[PATCH compass-ci] container/web-backend|-nginx: update SRV_HTTP_RESULT_PORT
by Lu Weitao 17 Mar '21

17 Mar '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend-nginx/nginx.conf | 2 +- container/web-backend/start | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/container/web-backend-nginx/nginx.conf b/container/web-backend-nginx/nginx.conf index 381888a..a3313fb 100644 --- a/container/web-backend-nginx/nginx.conf +++ b/container/web-backend-nginx/nginx.conf @@ -29,7 +29,7 @@ http { proxy_set_header X-Forward-For $proxy_add_x_forwarded_for; # srv-http server - proxy_pass http://172.17.0.1:11300; + proxy_pass http://172.17.0.1:20007; } } } diff --git a/container/web-backend/start b/container/web-backend/start index e680255..cebf949 100755 --- a/container/web-backend/start +++ b/container/web-backend/start @@ -11,7 +11,7 @@ docker_rm web-backend load_cci_defaults [[ $SRV_HTTP_RESULT_HOST ]] || SRV_HTTP_RESULT_HOST=$(curl ip.sb) -[[ $SRV_HTTP_RESULT_PORT ]] || SRV_HTTP_RESULT_PORT=11300 +[[ $SRV_HTTP_RESULT_PORT ]] || SRV_HTTP_RESULT_PORT=20007 cmd=( docker run -- 2.23.0

1 0

[PATCH v2 compass-ci] container/web-backend: add API /get_table_error
by Lu Weitao 17 Mar '21

17 Mar '21

[Example] curl -X GET 'localhost:30000/get_job_error?group_id=wcl_ansible-openeuler-03-10' return: { "filter":{"group_id":"wcl_ansible-openeuler-03-10"}, "table_fields":["job_id","error_id","error_message","link to result"], "data_table":[ [ "crystal.1354921", "ansible_test.error.Unable-to-start-service-httpd-Job-for-httpdservice-xxx.fail", "{\"changed\": false, \"msg\": \"Unable to start service httpd: Job for httpd.service failed because a timeout was exceeded.}", "http://172.17.0.1:11300/result/ansible_test/2021-03-10/vm-2p16g--wcl1/opene… ate/crystal.1354921" ], [ "crystal.1354929", ... ] ... ] } Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend/web-backend | 19 +++++++++ src/lib/web_backend.rb | 68 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 74eda85..fa2a53d 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -146,3 +146,22 @@ end get '/get_repo_statistics' do new_refs_statistics(params) end + +# GET /get_job_error?suite=virttest&tbox_group=vm-2p8g +# Response +# { +# "filter":{"group_id":"wcl_ansible-openeuler-03-10"}, +# "table_fields":["job_id","error_id","error_message","link to result"], +# "data_table":[ +# [ +# "crystal.1354921", +# "ansible_test.error.Unable-to-start-service-httpd-Job-for-httpdservice-failed.message", +# "{\"changed\": false, \"msg\": \"Unable to start service httpd: Job for httpd.service failed because a timeout was exceeded.} +# "http://172.17.0.1:11300/result/ansible_test/2021-03-10/vm-2p16g--wcl1/opene…" +# ], +# ... +# ] +# } +get '/get_job_error' do + get_job_error(params) +end diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 89c0f3c..48b7ef2 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -12,6 +12,7 @@ require "#{CCI_SRC}/lib/compare.rb" require "#{CCI_SRC}/lib/constants.rb" require "#{CCI_SRC}/lib/es_query.rb" require "#{CCI_SRC}/lib/matrix2.rb" +require "#{CCI_SRC}/lib/params_group.rb" require "#{CCI_SRC}/lib/compare_data_format.rb" UPSTREAM_REPOS_PATH = ENV['UPSTREAM_REPOS_PATH'] || '/c/upstream-repos' @@ -548,3 +549,70 @@ def new_refs_statistics(params) end [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] end + + +# ------------------------------------------------------------------------------------------- +# job error table like: +# job_id error_id error_message link to result +# ------------------------------------------------------------------------------------- +# crystal.630608 "stderr.xxx" "messag:xxxx" https://$host:$port/$result_root +# ... +# ------------------------------------------------------------------------------------------- + +def get_job_error(params) + begin + body = job_error_body(params) + rescue StandardError => e + warn e.message + return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'get error table error'] + end + + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] +end + +def job_error_body(params) + error_table = get_error_table(params) + { + filter: params, + table_fields: ["job_id", "error_id", "error_message", "link to result"], + data_table: error_table, + }.to_json +end + +def get_error_table(filter_items) + error_table = [] + + job_list = get_job_list(filter_items) + job_list.each do |job| + error_table += get_error_from_job(job) + end + + error_table +end + +def get_job_list(items) + es = ESQuery.new(ES_HOST, ES_PORT) + query_results = es.multi_field_query(items) + + extract_jobs_list(query_results['hits']['hits']) +end + +# get all error_id from one job +def get_error_from_job(job) + job_error = [] + job['stats'].each do |metric, value| + next unless metric.end_with?('.message') + + result_host = ENV['SRV_HTTP_RESULT_HOST'] || SRV_HTTP_RESULT_HOST + result_port = ENV['SRV_HTTP_RESULT_PORT'] || SRV_HTTP_RESULT_PORT + error_id = metric.sub('.message', '.fail') + job_error << [ + job['id'], + error_id, + value, + "http://#{result_host}:#{result_port}#{job['result_root']}" + ] + end + + job_error +end -- 2.23.0

2 2