April 2021 - Compass-ci - mailweb.openeuler.org

[PATCH compass-ci 2/4] container/updaterepo: add a new docker service
by Li Ping 06 Apr '21

06 Apr '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/updaterepo/Dockerfile | 21 +++++++++++++++++++++ container/updaterepo/build | 14 ++++++++++++++ container/updaterepo/start | 24 ++++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 container/updaterepo/Dockerfile create mode 100755 container/updaterepo/build create mode 100755 container/updaterepo/start diff --git a/container/updaterepo/Dockerfile b/container/updaterepo/Dockerfile new file mode 100644 index 0000000..adab626 --- /dev/null +++ b/container/updaterepo/Dockerfile @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM alpine:scheduler-dev + +MAINTAINER Li Ping <1477412247(a)qq.com> + +RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories + +RUN apk update && \ + apk add --no-cache 'ruby-dev' \ + 'g++' 'gcc' 'pcre' 'libevent' 'make' 'git' 'cpio' 'bash' + +ENV UPDATE_REPO_DIR /c/cci/update-repo + +RUN mkdir -p $UPDATE_REPO_DIR && \ + chown -R 1090:1090 /c + +WORKDIR $UPDATE_REPO_DIR + +COPY --chown=1090:1090 updaterepo . diff --git a/container/updaterepo/build b/container/updaterepo/build new file mode 100755 index 0000000..594558a --- /dev/null +++ b/container/updaterepo/build @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative '../defconfig.rb' + +docker_skip_rebuild "update-repo" + +CCI_SRC = ENV['CCI_SRC'] +system "bash #{CCI_SRC}/container/compile ." +system "docker build -t update-repo ." + +system 'rm updaterepo' diff --git a/container/updaterepo/start b/container/updaterepo/start new file mode 100755 index 0000000..4ac1925 --- /dev/null +++ b/container/updaterepo/start @@ -0,0 +1,24 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative '../defconfig.rb' + +docker_rm "update-repo" + +cmd = %W[ + docker run + --restart=always + --name update-repo + -u 1090:1090 + -p 10003:10003 + -d + -v /srv/rpm/:/srv/rpm/ + -v /etc/localtime:/etc/localtime:ro + update-repo +] + +cmd += ['sh', '-c', 'umask 002 && ./updaterepo'] + +system(*cmd) -- 2.23.0

1 0

[PATCH compass-ci 1/4] container/rpm-repo: add a new docker service
by Li Ping 06 Apr '21

06 Apr '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/rpm-repo/Dockerfile | 10 ++++++ container/rpm-repo/build | 6 ++++ container/rpm-repo/start | 41 ++++++++++++++++++++++ lib/rpm-repo.rb | 65 +++++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+) create mode 100644 container/rpm-repo/Dockerfile create mode 100755 container/rpm-repo/build create mode 100755 container/rpm-repo/start create mode 100755 lib/rpm-repo.rb diff --git a/container/rpm-repo/Dockerfile b/container/rpm-repo/Dockerfile new file mode 100644 index 0000000..588828a --- /dev/null +++ b/container/rpm-repo/Dockerfile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM fedora:latest + +RUN yum install -y createrepo ruby + +RUN umask 002 && \ + gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ + gem install bunny diff --git a/container/rpm-repo/build b/container/rpm-repo/build new file mode 100755 index 0000000..458e8f6 --- /dev/null +++ b/container/rpm-repo/build @@ -0,0 +1,6 @@ +#!/bin/sh + +. ../defconfig.sh +docker_skip_rebuild "fedora:update-repodata" + +docker build -t fedora:update-repodata . diff --git a/container/rpm-repo/start b/container/rpm-repo/start new file mode 100755 index 0000000..08fa434 --- /dev/null +++ b/container/rpm-repo/start @@ -0,0 +1,41 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig.rb' + +DOCKER_CCI = '/c/compass-ci' + +names = Set.new %w[ + MQ_HOST + MQ_PORT + LKP_SERVER +] + +defaults = relevant_defaults(names) +defaults['MQ_HOST'] ||= defaults['LKP_SERVER'] ||= '172.17.0.1' +defaults['LKP_SERVER'] ||= '172.17.0.1' +defaults['MQ_PORT'] ||= 5672 +env = docker_env(defaults) + +docker_rm "update-repodata" + +cmd = %W[ + docker run + --restart=always + --name update-repodata + -u 1090:1090 +] + env + %W[ + -d + -e CCI_SRC=#{DOCKER_CCI} + -v #{ENV['CCI_SRC']}:#{DOCKER_CCI} + -v /etc/localtime:/etc/localtime:ro + -v /srv/rpm:/srv/rpm + -w /c/compass-ci/lib + fedora:update-repodata +] + +cmd += ['sh', '-c', 'umask 002 && ./rpm-repo.rb'] +system(*cmd) diff --git a/lib/rpm-repo.rb b/lib/rpm-repo.rb new file mode 100755 index 0000000..f516d3f --- /dev/null +++ b/lib/rpm-repo.rb @@ -0,0 +1,65 @@ +#!/usr/bin/ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'fileutils' +require 'json' +require './mq_client' + +MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || '172.17.0.1' +MQ_PORT = ENV['MQ_PORT'] || 5672 + +# used to handle the mq queue "update_repo" +# +# Example items in @mq.queue "update_repo": { "upload_rpms" => ["/srv/rpm/upload/**/Packages/*.rpm", "/srv/rpm/upload/**/source/*.rpm"]} +# handle_new_rpm +# move /srv/rpm/upload/**/*.rpm to /srv/rpm/testing/**/*.rpm +# update /srv/rpm/testing/**/repodate +# update_pub_dir +# copy /srv/rpm/testing/**/*.rpm to /srv/rpm/pub/**/*.rpm +# change /srv/rpm/pub/**/repodate +# @mq.ack(info) +class HandleRepo + def initialize + @mq = MQClient.new(MQ_HOST, MQ_PORT) + @update = [] + end + + def handle_new_rpm + queue = @mq.queue("update_repo") + queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| + rpm_info = JSON.parse(msg) + rpm_info["upload_rpms"].each do |rpm| + rpm_path = File.dirname(rpm).sub("upload", "testing") + FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) + + dest = File.join(rpm_path.to_s, File.basename(rpm)) + @update << dest + FileUtils.mv(rpm, dest) + system("createrepo --update $(dirname #{rpm_path})") + end + update_pub_dir + @mq.ack(info) + end + end + + def update_pub_dir + @update.each do |rpm| + pub_path = File.dirname(rpm).sub("testing", "pub") + FileUtils.mkdir_p(pub_path) unless File.directory?(pub_path) + + dest = File.join(pub_path, File.basename(rpm)) + FileUtils.cp(rpm, dest) + + repodata_dest = File.join(File.dirname(pub_path), "repodata") + repodata_src = File.dirname(rpm).sub("Packages", "repodata") + + FileUtils.rm_r(repodata_dest) if Dir.exist?(repodata_dest) + FileUtils.cp_r(repodata_src, File.dirname(repodata_dest)) + end + end +end + +hr = HandleRepo.new +hr.handle_new_rpm -- 2.23.0

1 0

[PATCH v2 lkp-tests] stats: add build-pkg.rb to generate error_id with message
by Lin Jiaxin 06 Apr '21

06 Apr '21

output like this: "build-pkg.include/common/ist.h:warning:unused-parameter‘count’[-Wunused-parameter]": 1, "build-pkg.include/common/ist.h:warning:unused-parameter‘count’[-Wunused-parameter].message": [ "include/common/ist.h:488:74: warning: unused parameter ‘count’ [-Wunused-parameter]\n static inline struct ist ist2str(char *dst, const struct ist src, size_t count)\n ^~~~~\n", "include/common/ist.h:533:77: warning: unused parameter ‘count’ [-Wunused-parameter]\n static inline struct ist ist2str_lc(char *dst, const struct ist src, size_t count )\n ^~~~~\n", "include/common/ist.h:578:77: warning: unused parameter ‘count’ [-Wunused-parameter]\n static inline struct ist ist2str_uc(char *dst, const struct ist src, size_t count )\n ^~~~~\n" ] Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- stats/build-pkg.rb | 74 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100755 stats/build-pkg.rb diff --git a/stats/build-pkg.rb b/stats/build-pkg.rb new file mode 100755 index 00000000..4515bf23 --- /dev/null +++ b/stats/build-pkg.rb @@ -0,0 +1,74 @@ +#!/usr/bin/env ruby + +LKP_SRC ||= ENV['LKP_SRC'] || File.dirname(File.dirname(File.realpath($PROGRAM_NAME))) + +require 'set' +require "#{LKP_SRC}/lib/common" + +class ErrorMessages + def initialize + @seqno = '' + @in_stderr = false + @error_line = '' + @error_details = '' + @error_message = {} + @error_messages = Hash.new { |h, k| h[k] = Set.new } + end + + def obtain_error_messages(log_lines) + log_lines.each do |line| + next if extract_error_message(line) + next unless @in_stderr + next unless @error_message['error_line'] =~ /(error|warning):[^:]/i + + error_id = build_pkg_error_id(@error_message['error_line']) + @error_messages[error_id] << @error_message['error_line'] + @error_message['error_details'] + end + return @error_messages + end + + private + + def extract_error_message(line) + if line =~ /^ / + @error_details += line + else + update_error_message(line) + case line + when /^## ______________([0-9.]+):stderr$/ + @in_stderr = true + @seqno = $1 + return true + + when /^## ______________#{@seqno}:enderr$/ + @in_stderr = false + @seqno = '' + add_error_message + end + end + end + + def update_error_message(line) + @error_message['error_line'] = @error_line + @error_message['error_details'] = @error_details + @error_line = line + @error_details = '' + end + + def add_error_message + return unless @error_message['error_line'] =~ /(error|warning):[^:]/i + + error_id = build_pkg_error_id(@error_message['error_line']) + @error_messages[error_id] << @error_message['error_line'] + @error_message['error_details'] + end +end + +def build_pkg(log_lines) + message = {} + ErrorMessages.new.obtain_error_messages(log_lines).each do |k, v| + message[k] = '1' + message["#{k}.message"] = v.to_a + end + + return message +end -- 2.23.0

1 0

[PATCH compass-ci] sched: consume job by priority
by Cao Xueliang 06 Apr '21

06 Apr '21

before: consume job by shuffle after: our priority is reflected in email account for now, consume job by priority and the delimiter@localhost has the highest priority Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/scheduler/find_job_boot.cr | 40 ++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 01b6f1d..a0e5220 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -110,15 +110,47 @@ class Sched def consume_by_list(queues) jobs, revision = get_history_jobs(queues) - jobs.shuffle! - while true - return nil, revision if jobs.empty? + email_jobs = split_jobs_by_email(jobs) + loop do + return nil, revision if email_jobs.empty? + + job = pop_job_by_priority(email_jobs) + return nil, revision unless job - job = jobs.delete_at(0) return job, revision if ready2process(job) end end + def split_jobs_by_email(jobs) + hash = Hash(String, Array(Etcd::Model::Kv)).new + jobs.each do |job| + key = job.key.split("/")[4] + if hash.has_key?(key) + hash[key] << job + else + hash[key] = [job] + end + end + + return hash + end + + def pop_job_by_priority(email_jobs) + delimiter = "delimiter@localhost" + if email_jobs.has_key?(delimiter) + return email_jobs[delimiter].delete_at(0) unless email_jobs[delimiter].empty? + + email_jobs.delete(delimiter) + end + + keys = rand_queues(email_jobs.keys) + keys.each do |key| + return email_jobs[key].delete_at(0) unless email_jobs[key].empty? + + email_jobs.delete(key) + end + end + def get_history_jobs(queues) revisions = [] of Int64 ec = EtcdClient.new -- 2.23.0

1 0

[PATCH compass-ci] container/srv-http: fix redirect issue
by Lu Weitao 06 Apr '21

06 Apr '21

[Why] the old srv-http-result container port is: 20007:11300 when access srv-http-result like: http://$host:20007/result # (without '/' at the end) the nginx will redirect the url to: http://$host:11300/result/ # (with '/' at the end) 11300 is the nginx service listen port, so nginx redirect with this port [How] keep both ports are same by set LISTEN_PORT with start script. - set LISTEN_PORT to ENV with start script - use $LISTEN_PORT replace default port(11300) form /etc/nginx/conf.d/default.conf(in container) with container starting Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/srv-http/root/sbin/entrypoint.sh | 2 ++ container/srv-http/start-cci | 3 ++- container/srv-http/start-git | 3 ++- container/srv-http/start-initrd | 3 ++- container/srv-http/start-os | 3 ++- container/srv-http/start-pub | 3 ++- container/srv-http/start-result | 3 ++- container/srv-http/start-rpm | 3 ++- 8 files changed, 16 insertions(+), 7 deletions(-) diff --git a/container/srv-http/root/sbin/entrypoint.sh b/container/srv-http/root/sbin/entrypoint.sh index cd73c7e..ced6eda 100755 --- a/container/srv-http/root/sbin/entrypoint.sh +++ b/container/srv-http/root/sbin/entrypoint.sh @@ -1,4 +1,6 @@ #!/bin/sh +sed -i "s/listen 11300;/listen $LISTEN_PORT;/g" /etc/nginx/conf.d/default.conf + nginx /usr/sbin/php-fpm7 diff --git a/container/srv-http/start-cci b/container/srv-http/start-cci index 00d8e4c..aa539d2 100755 --- a/container/srv-http/start-cci +++ b/container/srv-http/start-cci @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-cci - -p 20011:11300 + -e LISTEN_PORT=20011 + -p 20011:20011 -v /srv/cci/libvirt-xml:/srv/cci/libvirt-xml:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-git b/container/srv-http/start-git index c1181a7..0b0ff47 100755 --- a/container/srv-http/start-git +++ b/container/srv-http/start-git @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-git - -p 20010:11300 + -e LISTEN_PORT=20010 + -p 20010:20010 -v /srv/git/archlinux:/srv/git/archlinux:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-initrd b/container/srv-http/start-initrd index 466ec1b..93be548 100755 --- a/container/srv-http/start-initrd +++ b/container/srv-http/start-initrd @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-initrd - -p 20008:11300 + -e LISTEN_PORT=20008 + -p 20008:20008 -v /srv/initrd/qemu-image:/srv/initrd/qemu-image:ro -v /srv/initrd/dockerimage:/srv/initrd/dockerimage:ro -v /srv/initrd/rpmbuild-pkg:/srv/initrd/rpmbuild-pkg:ro diff --git a/container/srv-http/start-os b/container/srv-http/start-os index d39e04e..5b63902 100755 --- a/container/srv-http/start-os +++ b/container/srv-http/start-os @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-os - -p 20009:11300 + -e LISTEN_PORT=20009 + -p 20009:20009 -v /srv/os:/srv/os:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-pub b/container/srv-http/start-pub index f42b117..6dfd5a4 100755 --- a/container/srv-http/start-pub +++ b/container/srv-http/start-pub @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-pub - -p 20006:11300 + -e LISTEN_PORT=20006 + -p 20006:20006 -v /srv/pub:/srv/pub:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-result b/container/srv-http/start-result index bf0287d..32bdf7d 100755 --- a/container/srv-http/start-result +++ b/container/srv-http/start-result @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-result - -p 20007:11300 + -e LISTEN_PORT=20007 + -p 20007:20007 -v /srv/result:/srv/result:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-rpm b/container/srv-http/start-rpm index 73e5162..2d38298 100755 --- a/container/srv-http/start-rpm +++ b/container/srv-http/start-rpm @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-rpm - -p 20012:11300 + -e LISTEN_PORT=20012 + -p 20012:20012 -v /srv/rpm/pub:/srv/rpm/pub:ro -v /etc/localtime:/etc/localtime:ro -d -- 2.23.0

1 0

[PATCH compass-ci] service/scheduler: update data before get job
by Wu Zhende 06 Apr '21

06 Apr '21

If no job, the session is suspended until the job is obtained. Therefore, testbox message and job events need to be updated before get job to ensure the timeliness. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 11 +++++++++++ src/scheduler/find_job_boot.cr | 16 ++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 2e4fd30..f62f83e 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -147,6 +147,17 @@ class Sched @log.warn(e) end + def send_mq_msg(job_state) + mq_msg = { + "job_id" => @env.get?("job_id").to_s, + "testbox" => @env.get?("testbox").to_s, + "deadline" => @env.get?("deadline").to_s, + "time" => get_time, + "job_state" => job_state + } + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) + end + def set_lifecycle(job, testbox, queues) if job deadline = job.get_deadline diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 01b6f1d..2fca531 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -25,14 +25,7 @@ class Sched rescue e @log.warn(e) ensure - mq_msg = { - "job_id" => @env.get?("job_id").to_s, - "testbox" => @env.get?("testbox").to_s, - "deadline" => @env.get?("deadline").to_s, - "time" => get_time, - "job_state" => "boot" - } - spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) + send_mq_msg("boot") end # auto submit a job to collect the host information. @@ -196,6 +189,13 @@ class Sched def get_job_boot(host, boot_type) queues = get_queues(host) + + # do before get job from etcd + # because if no job will hang + # need to update information in a timely manner + set_lifecycle(nil, host, queues) + send_mq_msg("boot") + job = get_job_from_queues(queues, host) set_lifecycle(job, host, queues) -- 2.23.0

1 0

[PATCH lkp-tests 2/2] add script for fix dockerfile file
by Bai Jing 06 Apr '21

06 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- lib/handle_dockerfile.sh | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 lib/handle_dockerfile.sh diff --git a/lib/handle_dockerfile.sh b/lib/handle_dockerfile.sh new file mode 100644 index 000000000..b71f9865d --- /dev/null +++ b/lib/handle_dockerfile.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" +} + +handle_epel() +{ + sed -i "s/epel-release/bash/g" "$dockerfile_name" +} + +handle_epelrpm() +{ + sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" +} + +handle_rpm_gpg() +{ + sed -i "s|RPM-GPG-KEY-CentOS-7|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" + sed -i "s|RPM-GPG-KEY-centosofficial|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" +} + +add_base_commands() +{ + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" +} + +handle_epel_repo() +{ + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" +} + +handle_error_exit() +{ + sed '/^RUN/RUN set -e;/g' "$dockerfile_name" +} + +handle_dockerfile() +{ + dockerfile_name=$1 + handle_FROM + handle_epel + handle_epelrpm + handle_rpm_gpg + add_base_commands + handle_epel_repo + handle_error_exit +} -- 2.23.0

1 0

[PATCH lkp-tests 1/2] add test script for dockerfile test
by Bai Jing 06 Apr '21

06 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 143 +++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 85 deletions(-) diff --git a/tests/openeuler_docker b/tests/openeuler_docker index 483f40135..3ea57e2c9 100755 --- a/tests/openeuler_docker +++ b/tests/openeuler_docker @@ -8,12 +8,12 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/log.sh +. $LKP_SRC/lib/handle_dockerfile.sh [ -n "$dockerfile_url" ] || die "dockerfile_url is empty" declare -i nr_openeuler_fail=0 declare -i nr_openeuler_success=0 -declare -i is_openeuler_build_fail=0 declare -i dockerfile_count=0 fit_repo() @@ -33,110 +33,51 @@ openeuler_ready() docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar } -centos_docker() -{ - is_openeuler_build_fail=0 - index=1 - for file in $dockerfile_path - do - [[ $file =~ ^centos ]] && continue - file=$(echo "$file" | awk -F: '{print $1}') - dockerfile_name="${file##*/}" - cd "${file%/*}" || continue - image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') - [ -n "$image" ] && image=${image}_$index - if [ -f "build.sh" ] && [ -x "build.sh" ]; then - ./build.sh > /dev/null 2> /dev/null - else - docker_build > /dev/null 2> /dev/null - fi - - [ $? == 0 ] && { - handle_dockerfile - } - - let index++ - done -} - -handle_FROM() -{ - sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" -} - -handle_epel() -{ - grep -qw "epel-release" "$dockerfile_name" && { - sed -i "s/epel-release/bash/g" "$dockerfile_name" - } -} - -handle_epelrpm() -{ - grep -qw "https://.*epel-release.*rpm" "$dockerfile_name" && { - sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" - } -} - -handle_rpm_gpg() -{ - sed -i "s|RPM-GPG-KEY-CentOS-7|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" -} - -add_base_commands() -{ - # grep -qw "groupadd*" "$file" && { - # fix missing useradd, groupadd, chpasswd, etc. commands - sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" -} - -handle_epel_repo() -{ - sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" -} - -handle_dockerfile() +build_centos() { - is_openeuler_build_fail=1 - handle_FROM - handle_epel - handle_rpm_gpg - add_base_commands - if docker_build "-openeuler"; then - docker_run "-openeuler" - let nr_openeuler_success++ + cd ${file%/*} + if [ -f "build.sh" ] && [ -x "build.sh" ]; then + ./build.sh || return 1 else - let nr_openeuler_fail++ + docker build -f "$dockerfile_name" -t "${image}_$1_$index" . || return 1 fi } -docker_build() +build_openeuler() { - docker build -f "$dockerfile_name" -t "${image}$1" . + docker build -f "$dockerfile_name" -t "${image}_$1_$index" . >> $TMP_RESULT_ROOT/${image}_$1_$index 2>&1 [ $? != 0 ] && { - [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.fail: 1" + let nr_openeuler_fail++ + echo "${image}_$index.build.fail: 1" return 1 } - [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.success: 1" - is_openeuler_build_fail=0 + echo "${image}_$index.build.success: 1" + let nr_openeuler_success++ + return 0 } docker_run() -{ - docker run -d --name "${image}" "${image}$1" - docker_stats=$(docker inspect -f '{{.State.Running}}' ${image}) - [ $docker_stats == false ] && { - echo "${image}.run.fail: 1" +{ + docker_run_cmd="docker run -d --name ${image}_$1_$index ${image}_$1_$index" + [ $1 == "openeuler" ] && { + docker_run_cmd="${docker_run_cmd} >> $TMP_RESULT_ROOT/${image}_$1_$index 2>&1" + } + + eval $docker_run_cmd + + docker_stats=$(docker inspect -f '{{.State.Running}}' $image_$1_$index) + [ $docker_stats == "false" ] && { + [ $1 == "openeuler" ] && echo "${image}_$index.run.fail: 1" return 1 } - echo "${image}.run.success: 1" + [ $1 == "openeuler"] && echo "${image}_$index.run.success: 1" } handle_build_result() { if [ "$nr_openeuler_fail" != 0 ]; then - echo "openeuler build failed times: $nr_openeuler_fail" + echo "openeuler_build.nr_failed: $nr_openeuler_fail" echo "openeuler_valid_url: $dockerfile_url" exit 1 # centos all success, openeuler all success @@ -146,6 +87,38 @@ handle_build_result() fi } +build() +{ + dockerfile_name="${file##*/}" + dockerfile_name="${dockerfile_name%:*}" + image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') + + build_centos "centos" > /dev/null 2>&1 || return 1 + handle_dockerfile "$dockerfile_name" + build_openeuler "openeuler" + [ $? != 0 ] && return 1 +} + +run() +{ + docker_run "centos" > /dev/null 2>&1 + [ $? != 0 ] && return 1 + docker_run "openeuler" +} + +centos_docker() +{ + index=1 + for file in $dockerfile_path + do + [[ $file =~ ^centos ]] && continue + + build + [ $? = 0 ] && run + let index++ + done +} + docker_test() { fit_repo -- 2.23.0

1 0

[PATCH compass-ci] sbin/es-find: support check query keyword
by Lu Weitao 06 Apr '21

06 Apr '21

[How] 1. add --keyword option 2. use like: es-find --keyword category dequeue_time end_time enqueue_time group_id hw.arch hw.memory hw.model hw.nr_cpu ... Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/es_query.rb | 25 +++++++++++++++++++++++++ sbin/es-find | 11 ++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 928bd68..22d68a9 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -130,6 +130,11 @@ class ESQuery parse_fields(result) end + + def query_mapping + mapping = @client.indices.get_mapping(index: @index) + get_mapping_key(mapping) + end end # Range Query Example: @@ -260,3 +265,23 @@ def parse_fields(es_result) result_hash end + +def get_mapping_key(mapping) + mapping_key = [] + properties = mapping.values[0]['mappings']['properties'] + properties.each do |key, value| + parse_mapping_properties(key, value, mapping_key) + end + + mapping_key +end + +def parse_mapping_properties(key, value, mapping_key) + if value['type'] + mapping_key << key + else + value['properties'].each do |k, v| + parse_mapping_properties("#{key}.#{k}", v, mapping_key) + end + end +end diff --git a/sbin/es-find b/sbin/es-find index 2c3b3dc..f548fc1 100755 --- a/sbin/es-find +++ b/sbin/es-find @@ -77,6 +77,8 @@ options = { nr_jobs: 10, desc_keyword: 'start_time' } range = nil count = nil +es = ESQuery.new(ES_HOST, ES_PORT) + opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: es-find [options] search_key1=val1[,val2..] ..' @@ -103,6 +105,14 @@ opt_parser = OptionParser.new do |opts| count = c end + opts.on_tail('--keyword', 'check all available search_keys') do + available_search_key = es.query_mapping + available_search_key.each do |key| + puts " #{key}" + end + exit + end + opts.on_tail('-h', '--help', 'show this message') do puts opts exit @@ -113,7 +123,6 @@ opt_parser.parse!(ARGV) items = parse_argv items[:range] = prase_range(range) if range -es = ESQuery.new(ES_HOST, ES_PORT) if count count_keywords = count.split(',') query_result = es.query_fields(count_keywords, items) -- 2.23.0

1 0

[PATCH v2 compass-ci] delimiter: refactor delimiter with crystal
by Cao Xueliang 04 Apr '21

04 Apr '21

use etcd list watch to submit bisect job Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter.cr | 22 ++++++++++ src/delimiter/delimiter.cr | 83 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 src/delimiter.cr create mode 100644 src/delimiter/delimiter.cr diff --git a/src/delimiter.cr b/src/delimiter.cr new file mode 100644 index 0000000..e8de306 --- /dev/null +++ b/src/delimiter.cr @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "yaml" +require "./delimiter/delimiter" + +def config_secrets_yaml + %x(#{ENV["CCI_SRC"]}/sbin/config_account_yaml.rb delimiter) + account = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/defaults/account.yaml")) + lab = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/include/lab/#{account["lab"]}.yaml")) + secrets = {"secrets" => lab} + File.open("#{ENV["HOME"]}/.config/compass-ci/defaults/secrets.yaml", "w") { |f| YAML.dump(secrets, f) } +end + +begin + config_secrets_yaml + delimiter = Delimiter.new + delimiter.consume_delimiter("delimiter") +rescue ex + puts ex +end diff --git a/src/delimiter/delimiter.cr b/src/delimiter/delimiter.cr new file mode 100644 index 0000000..c98ded9 --- /dev/null +++ b/src/delimiter/delimiter.cr @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "../lib/etcd_client" + +class Delimiter + def initialize + @ec = EtcdClient.new + end + + def consume_delimiter(queue) + channel = Channel(Etcd::Model::Kv).new + revision = consume_by_list(queue, channel) + consume_by_watch(queue, revision, channel) + end + + def consume_by_list(queue, channel) + tasks, revision = get_history_tasks(queue) + handle_history_tasks(tasks, channel) + + return revision + end + + def consume_by_watch(queue, revision, channel) + watch_queue(queue, revision, channel) + handle_events(channel) + end + + def get_history_tasks(queue) + tasks = [] of Etcd::Model::Kv + range = @ec.range_prefix(queue) + revision = range.header.not_nil!.revision + tasks += range.kvs + + return tasks, revision + end + + def handle_history_tasks(tasks, channel) + loop do + return if tasks.empty? + + task = tasks.delete_at(0) + spawn { submit_bisect_job(channel, task) } + end + end + + def watch_queue(queue, revision, channel) + watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64 + 1, filters: [Etcd::Watch::Filter::NODELETE]) do |events| + events.each do |event| + channel.send(event.kv) + end + end + + spawn { watcher.start } + Fiber.yield + end + + def handle_events(channel) + loop do + task = channel.receive + spawn { submit_bisect_job(channel, task) } + end + end + + def submit_bisect_job(channel, task) + key = task.key + value = Hash(String, String).from_json(task.value.not_nil!) + begin + response = %x(#{ENV["LKP_SRC"]}/sbin/submit bad_job_id=#{value["job_id"]} error_id=#{value["error_id"].inspect} bisect.yaml queue=dc-bisect) + puts response + if /id=0/ =~ response + channel.send(task) + return + end + + @ec.delete(key) + rescue ex + puts ex + channel.send(task) + end + end +end -- 2.23.0

1 0

[PATCH v1 compass-ci 3/3] delimiter: refactor delimiter whit crystal
by Cao Xueliang 04 Apr '21

04 Apr '21

use etcd list watch to submit bisect job Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter.cr | 22 ++++++++++ src/delimiter/delimiter.cr | 83 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 src/delimiter.cr create mode 100644 src/delimiter/delimiter.cr diff --git a/src/delimiter.cr b/src/delimiter.cr new file mode 100644 index 0000000..e8de306 --- /dev/null +++ b/src/delimiter.cr @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "yaml" +require "./delimiter/delimiter" + +def config_secrets_yaml + %x(#{ENV["CCI_SRC"]}/sbin/config_account_yaml.rb delimiter) + account = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/defaults/account.yaml")) + lab = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/include/lab/#{account["lab"]}.yaml")) + secrets = {"secrets" => lab} + File.open("#{ENV["HOME"]}/.config/compass-ci/defaults/secrets.yaml", "w") { |f| YAML.dump(secrets, f) } +end + +begin + config_secrets_yaml + delimiter = Delimiter.new + delimiter.consume_delimiter("delimiter") +rescue ex + puts ex +end diff --git a/src/delimiter/delimiter.cr b/src/delimiter/delimiter.cr new file mode 100644 index 0000000..c98ded9 --- /dev/null +++ b/src/delimiter/delimiter.cr @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "../lib/etcd_client" + +class Delimiter + def initialize + @ec = EtcdClient.new + end + + def consume_delimiter(queue) + channel = Channel(Etcd::Model::Kv).new + revision = consume_by_list(queue, channel) + consume_by_watch(queue, revision, channel) + end + + def consume_by_list(queue, channel) + tasks, revision = get_history_tasks(queue) + handle_history_tasks(tasks, channel) + + return revision + end + + def consume_by_watch(queue, revision, channel) + watch_queue(queue, revision, channel) + handle_events(channel) + end + + def get_history_tasks(queue) + tasks = [] of Etcd::Model::Kv + range = @ec.range_prefix(queue) + revision = range.header.not_nil!.revision + tasks += range.kvs + + return tasks, revision + end + + def handle_history_tasks(tasks, channel) + loop do + return if tasks.empty? + + task = tasks.delete_at(0) + spawn { submit_bisect_job(channel, task) } + end + end + + def watch_queue(queue, revision, channel) + watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64 + 1, filters: [Etcd::Watch::Filter::NODELETE]) do |events| + events.each do |event| + channel.send(event.kv) + end + end + + spawn { watcher.start } + Fiber.yield + end + + def handle_events(channel) + loop do + task = channel.receive + spawn { submit_bisect_job(channel, task) } + end + end + + def submit_bisect_job(channel, task) + key = task.key + value = Hash(String, String).from_json(task.value.not_nil!) + begin + response = %x(#{ENV["LKP_SRC"]}/sbin/submit bad_job_id=#{value["job_id"]} error_id=#{value["error_id"].inspect} bisect.yaml queue=dc-bisect) + puts response + if /id=0/ =~ response + channel.send(task) + return + end + + @ec.delete(key) + rescue ex + puts ex + channel.send(task) + end + end +end -- 2.23.0

1 0

[PATCH v1 compass-ci 2/3] container/delimiter: compile delimiter with crystal
by Cao Xueliang 04 Apr '21

04 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/delimiter/Dockerfile | 11 ++++++++++- container/delimiter/build | 4 ++++ container/delimiter/start | 3 +-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/container/delimiter/Dockerfile b/container/delimiter/Dockerfile index cc57320..a9ff279 100644 --- a/container/delimiter/Dockerfile +++ b/container/delimiter/Dockerfile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM alpine:3.13 +FROM alpine:scheduler-dev RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories && \ apk update && \ @@ -11,3 +11,12 @@ RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweiclou RUN umask 002 && \ gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ gem install rest-client activesupport git json yaml elasticsearch faye-websocket terminal-table mail io-console + +ENV RUNTIME_DIR /c/cci/delimiter + +RUN mkdir -p $RUNTIME_DIR && \ + chown -R 1090:1090 /c + +WORKDIR $RUNTIME_DIR + +COPY --chown=1090:1090 delimiter . diff --git a/container/delimiter/build b/container/delimiter/build index 606149d..7f3108b 100755 --- a/container/delimiter/build +++ b/container/delimiter/build @@ -8,6 +8,10 @@ docker_skip_rebuild "delimiter" load_cci_defaults +bash $CCI_SRC/container/compile . + docker build -t delimiter . push_image delimiter:latest + +rm delimiter diff --git a/container/delimiter/start b/container/delimiter/start index 3cff23a..9485750 100755 --- a/container/delimiter/start +++ b/container/delimiter/start @@ -32,10 +32,9 @@ cmd = %w[ -v #{DEFAULT_CONFIG_DIR}:#{DEFAULT_CONFIG_DIR}:ro -v /etc/localtime:/etc/localtime:ro -v /etc/compass-ci/register:/etc/compass-ci/register:ro - -w #{DEFAULT_CCI}/src delimiter ] -cmd += ['sh', '-c', 'umask 002 && ruby ./delimiter.rb'] +cmd += ['sh', '-c', 'umask 002 && ./delimiter'] system(*cmd) -- 2.23.0

1 0

[PATCH v1 compass-ci 1/3] extract-stats: close etcd client
by Cao Xueliang 04 Apr '21

04 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/extract_stats.cr | 11 +++++++---- src/extract-stats/stats_worker.cr | 5 +++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/extract-stats/extract_stats.cr b/src/extract-stats/extract_stats.cr index 4ae3e71..c35d3ec 100644 --- a/src/extract-stats/extract_stats.cr +++ b/src/extract-stats/extract_stats.cr @@ -6,10 +6,13 @@ require "./constants" require "./stats_worker" module ExtractStats + @@ec = EtcdClient.new + def self.consume_tasks channel = Channel(String).new - revision = self.consume_by_list(EXTRACT_STATS_QUEUE_PATH, channel) - self.consume_by_watch(EXTRACT_STATS_QUEUE_PATH, revision, channel) + queue = EXTRACT_STATS_QUEUE_PATH + revision = self.consume_by_list(queue, channel) + self.consume_by_watch(queue, revision, channel) end def self.consume_by_list(queue, channel) @@ -26,7 +29,7 @@ module ExtractStats def self.get_history_tasks(queue) tasks = [] of Etcd::Model::Kv - range = EtcdClient.new.range_prefix(queue) + range = @@ec.range_prefix(queue) revision = range.header.not_nil!.revision tasks += range.kvs @@ -45,7 +48,7 @@ module ExtractStats end def self.watch_queue(queue, revision, channel) - watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64, filters: [Etcd::Watch::Filter::NODELETE]) do |events| + watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64 + 1, filters: [Etcd::Watch::Filter::NODELETE]) do |events| events.each do |event| puts event channel.send(event.kv.key) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index 8910951..45b75a3 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -1,6 +1,5 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. - require "../lib/etcd_client" require "../scheduler/elasticsearch_client" require "../scheduler/redis_client" @@ -30,6 +29,8 @@ class StatsWorker STDERR.puts e.message # incase of many error message when task-queue, ES does not work sleep(10) + ensure + @etcd.close end end @@ -71,7 +72,7 @@ class StatsWorker sample_error_id = new_error_ids.sample STDOUT.puts "send a delimiter task: job_id is #{job_id}" queue = "#{DELIMITER_TASK_QUEUE}/#{job_id}" - value = {"error_id" => sample_error_id} + value = {"job_id" => job_id, "error_id" => sample_error_id} @etcd.put(queue, value) msg = %({"job_id": "#{job_id}", "new_error_id": "#{sample_error_id}"}) -- 2.23.0

1 0

[PATCH compass-ci] extract-stats: close etcd client
by Cao Xueliang 03 Apr '21

03 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/extract_stats.cr | 9 ++++++--- src/extract-stats/stats_worker.cr | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/extract-stats/extract_stats.cr b/src/extract-stats/extract_stats.cr index 4ae3e71..3346d12 100644 --- a/src/extract-stats/extract_stats.cr +++ b/src/extract-stats/extract_stats.cr @@ -6,10 +6,13 @@ require "./constants" require "./stats_worker" module ExtractStats + @@ec = EtcdClient.new + def self.consume_tasks channel = Channel(String).new - revision = self.consume_by_list(EXTRACT_STATS_QUEUE_PATH, channel) - self.consume_by_watch(EXTRACT_STATS_QUEUE_PATH, revision, channel) + queue = EXTRACT_STATS_QUEUE_PATH + revision = self.consume_by_list(queue, channel) + self.consume_by_watch(queue, revision, channel) end def self.consume_by_list(queue, channel) @@ -26,7 +29,7 @@ module ExtractStats def self.get_history_tasks(queue) tasks = [] of Etcd::Model::Kv - range = EtcdClient.new.range_prefix(queue) + range = @@ec.range_prefix(queue) revision = range.header.not_nil!.revision tasks += range.kvs diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index 8910951..9dea267 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -1,6 +1,5 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. - require "../lib/etcd_client" require "../scheduler/elasticsearch_client" require "../scheduler/redis_client" @@ -30,6 +29,8 @@ class StatsWorker STDERR.puts e.message # incase of many error message when task-queue, ES does not work sleep(10) + ensure + @etcd.close end end -- 2.23.0

1 0

review star
by Wu Fengguang 02 Apr '21

02 Apr '21

Cao Xueliang is our review star, thanks Cao Xueliang! baijing@crystal ~/todo/bin% ./review-star Review email records in the last 7 days total review paragraph point people 81 11 11 25 Cao Xueliang 59 12 19 12 Wei Jihui 24 13 28 5 Luan Shengde 22 7 6 3 Cui Yili 19 3 4 3 Wu Zhende 16 3 5 2 Yu Chuan 12 5 5 2 Xu Xijian 2 1 1 1 Wu Fengguang 25 0 0 1 Li Ping 17 1 1 0 Wang Chenglong 20 0 0 0 Liu Yinsi 6 0 0 0 Lu Weitao 1 0 0 0 Zhang Dewan 20 1 1 0 Wang Yong 6 0 0 0 Li Yuanchao 1 0 0 0 Bai Jing 9 4 4 0 Lin Jiaxin 18 5 4 0 Liu Shaofei

1 0

review star
by Wu Fengguang 02 Apr '21

02 Apr '21

Luan Shengde is our review star, thanks Luan Shengde! baijing@crystal ~/todo/bin% ./review-star Review email records in the last 7 days total review paragraph point people 81 11 11 25 Cao Xueliang 59 12 19 12 Wei Jihui 24 13 28 5 Luan Shengde 22 7 6 3 Cui Yili 19 3 4 3 Wu Zhende 16 3 5 2 Yu Chuan 12 5 5 2 Xu Xijian 2 1 1 1 Wu Fengguang 25 0 0 1 Li Ping 17 1 1 0 Wang Chenglong 20 0 0 0 Liu Yinsi 6 0 0 0 Lu Weitao 1 0 0 0 Zhang Dewan 20 1 1 0 Wang Yong 6 0 0 0 Li Yuanchao 1 0 0 0 Bai Jing 9 4 4 0 Lin Jiaxin 18 5 4 0 Liu Shaofei

2 1

help star
by Wu Fengguang 02 Apr '21

02 Apr '21

Wu Zhende, Yu Chuan is our help star again, thanks Wu Zhende, Yu Chuan! baijing@crystal ~/todo/bin% ./help-star ../people/*/credit/2021-04-01.md 14.3 Wu Zhende 14.2 Yu Chuan 14.0 Wei Jihui 11.3 Wu Fengguang 11.0 Liu Shaofei 10.7 Wang Yong 10.2 Zhang Dewan 9.4 Lin Jiaxin 7.5 Luan Shengde 6.8 Cao Xueliang 5.8 Liu Yinsi 5.4 Bai Jing 4.4 Lu Weitao 4.2 Du Kaitian 4.0 Li Ping 2.2 Cui Yili 1.8 Wang Chenglong 1.5 Li Yuanchao 1.4 Xu Xijian 0.0 Zhang Yale

1 0

git count
by Wu Fengguang 02 Apr '21

02 Apr '21

Cao Xueliang is our code star, thanks Cao Xueliang! baijing@crystal ~/todo/bin% ./git-count 19 Cao Xueliang 9 Liu Yinsi 8 Wu Zhende 6 Liu Shaofei 6 Lu Weitao 6 Wang Yong 4 Xu Xijian 4 Cui Yili 4 Luan Shengde 3 Wei Jihui 3 Yu Chuan 3 Li Yuanchao 3 Lin Jiaxin 3 Li Ping 2 Wang Chenglong 1 Bai Jing 1 Zhang Dewan 0 Du Kaitian 0 Wu Fengguang 0 Zhang Yale

1 0

[PATCH lkp-tests] test/ansible_test: modify ansible code to output the module name
by Wang Chenglong 02 Apr '21

02 Apr '21

[Why] modify ansible code to output the name of the module for tasks execution. [How] print("{'action':'%s'}" % task_result.__dict__['_task_fields']['action']) {'action':'package'} Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/ansible_test b/tests/ansible_test index 56926bbed..1c19c2b0f 100755 --- a/tests/ansible_test +++ b/tests/ansible_test @@ -58,7 +58,18 @@ ansible_playbook_run() EOF [ $? -eq 0 ] && { echo "Playbook has been generated!" - } + } + [ $(command -v ansible-playbook) ] || { + echo "ansible-playbook: command not found" + exit 1 + } + + sed -i "536a\ print$\"{\'action\':\'%s\'}\" \% task_result.__dict__[\'_task_fields\'][\'action\']$" \ + /usr/local/lib/python3.7/site-packages/ansible/plugins/strategy/__init__.py + + sed -i "550a\ print$\"{\'action\':\'%s\'}\" \% task_result.__dict__[\'_task_fields\'][\'action\']$" \ + /usr/local/lib/python3.7/site-packages/ansible/plugins/strategy/__init__.py + ansible-playbook "$ansible_path"/site.yml if [ $? -eq 0 ]; then -- 2.23.0

2 1

[PATCH v2 lkp-tests] stats/ansible_test: optimize the output of error_id
by Wang Chenglong 02 Apr '21

02 Apr '21

[Why] optimize the output of error_id standard format: @module_name:error_id before: error.[message].Could-not-find-or-access-RedHat-20-yml-Searched-in-root-ansible-roles-ansible-role-postgresql-vars-RedHat-20-yml-root-ansible- roles-ansible-role-postgresql-RedHat-20-yml-root-ansible-roles-ansible-role-postgresql-tasks-vars-RedHat-20-yml-root-ansible-roles-ansible-role- postgresql-tasks-RedHat-20-yml-root-ansible-vars-RedHat-20-yml-root-ansible-RedHat-20-yml-on-the-Ansible-Controller-If-you-are-using-a-module- and-expect-the-file-to-exist-on-the-remote-see-the-remote_src-option: 1 after: @inculde_vars:Could-not-find-or-access-RedHat-20-yml: 1 Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 101 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 85 insertions(+), 16 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 3d3b0b0cb..d7f084027 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,40 +1,109 @@ #!/usr/bin/env ruby require 'json' +require 'yaml' -def output_error(error_msg, error_key) - return if error_msg.nil? - +def output_error(error_msg) if error_msg.is_a? Array error_msg.each do |i| error_id = common_error_id i - puts "error.[#{error_key}].#{error_id}: 1" - puts "error.#{error_id}.message: #{@ansible_failed_info}" + puts "@#{@module_name['action']}:#{error_id}: 1" + puts "@#{@module_name['action']}:#{error_id}.message: #{@ansible_failed_info}" end elsif error_msg.is_a? String error_id = common_error_id error_msg - puts "error.[#{error_key}].#{error_id}: 1" - puts "error.#{error_id}.message: #{@ansible_failed_info}" + puts "@#{@module_name['action']}:#{error_id}: 1" + puts "@#{@module_name['action']}:#{error_id}.message: #{@ansible_failed_info}" end end def common_error_id(line) - line.gsub!(/[^\w]/, '-') + line.gsub!(/[^\w]/, '--') + line.gsub!(/[0-9]{3,}/, '#') line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one line.gsub!(/^-|-$/, '') line end +def parse_msg(ansible_failed_json) + case ansible_failed_json['msg'] + when /package (.*64)/ + output_error "cannot install the #{$1} for the job" + when /(The error appears to be in .*')/ + output_error $1 + when /(Failed to import the required Python library .*python3\.)/ + output_error $1 + # when /(An unhandled exception occurred while running the lookup plugin .*'\.)/ + when /original message: (.*)/ + output_error $1 + when /(Unable to start service (\w{1,}):)/ + output_error $1 + when /(Failed to download metadata for repo .*'):/ + output_error $1 + when /(Failed to download packages: (.*):)/ + output_error $2 + when /(Failed to find required .* in paths):/ + output_error $1 + when /(Failure downloading .*),/ + output_error $1 + when /(Could not find or access.*)'/ + output_error $1 + when /(Could not find .*):/ + output_error $1 + when /(Unsupported parameters) for/ + output_error $1 + when /aajdhch/ + output_error $1 + when /non-zero return code|Failed to install some of the specified packages/ + return + else + output_error ansible_failed_json['msg'] + end +end + +def parse_stderr(ansible_failed_json) + case ansible_failed_json['stderr'] + when /^$/ + return + else + ansible_failed_json['stderr_lines'].each do |i| + output_error i + end + end +end + +def parse_message(ansible_failed_json) + case ansible_failed_json['message'] + when /(Could not find or access.*)'/ + output_error $1 + else + output_error ansible_failed_json['message'] + end +end + +def parse_failures(ansible_failed_json) + ansible_failed_json['failures'].each do |i| + case i + when /(.*)/ + output_error $1 + end + end +end + while (line = STDIN.gets) - next unless line =~ /(FAILED!|failed:).*=>(.*)/ + case line + when /({'action'.*})/ + @module_name = YAML.load($1) + when /(FAILED!|failed:).*=>(.*)/ - @ansible_failed_info = $2 - next if @ansible_failed_info.empty? + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? - ansible_failed_json = JSON.parse @ansible_failed_info + ansible_failed_json = JSON.parse @ansible_failed_info - output_error ansible_failed_json['msg'],'msg' - output_error ansible_failed_json['message'],'message' - output_error ansible_failed_json['cmd'],'cmd' - output_error ansible_failed_json['failures'],'failures' + parse_msg ansible_failed_json unless ansible_failed_json['msg'].nil? + parse_stderr ansible_failed_json unless ansible_failed_json['stderr'].nil? + parse_message ansible_failed_json unless ansible_failed_json['message'].nil? + parse_failures ansible_failed_json unless ansible_failed_json['failures'].nil? + end end -- 2.23.0

2 1

[PATCH compass-ci] service/scheduler: modify the data saved in the testbox
by Wu Zhende 02 Apr '21

02 Apr '21

1. add name: for search in es 2. add type: vm/dc/physical 3. delete history content: no need this info testbox's history info can find in logging Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 19 ++++++++++++++++++- src/scheduler/elasticsearch_client.cr | 7 ++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 305d070..2e4fd30 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -160,18 +160,35 @@ class Sched state = "requesting" end + type = get_type(testbox) queues = JSON.parse(queues.to_json) hash = { "job_id" => job_id, "state" => state, "time" => get_time, "deadline" => deadline, - "queues" => queues + "queues" => queues, + "type" => type, + "name" => testbox } + @redis.update_wtmp(testbox.to_s, hash) @es.update_tbox(testbox.to_s, hash) end + def get_type(testbox) + return unless testbox + + if testbox.includes?("vm") + type = "vm" + elsif testbox.includes?("dc") + type = "dc" + else + type = "physical" + end + type + end + def report_ssh_port testbox = @env.params.query["tbox_name"] ssh_port = @env.params.query["ssh_port"].to_s diff --git a/src/scheduler/elasticsearch_client.cr b/src/scheduler/elasticsearch_client.cr index 219cf3d..f32256b 100644 --- a/src/scheduler/elasticsearch_client.cr +++ b/src/scheduler/elasticsearch_client.cr @@ -117,17 +117,14 @@ class Elasticsearch::Client result = @client.get_source(query) raise result unless result.is_a?(JSON::Any) - history = result["history"].as_a? result = result.as_h else result = wtmp_hash end - history ||= [] of JSON::Any - history << JSON.parse(wtmp_hash.to_json) unless wtmp_hash["state"]?.to_s == "requesting" - history = JSON.parse(history.to_json) - + history = JSON::Any.new([] of JSON::Any) body = { "history" => history} + body.any_merge!(result) body.any_merge!(wtmp_hash) -- 2.23.0

1 0

[PATCH v2 lkp-tests] tests/rpmbuild-pkg: add parameters annotation
by Wang Yong 02 Apr '21

02 Apr '21

it's for choosing parameters clearly Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/rpmbuild-pkg | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index 7a101b6a0..a91ae4a08 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -3,6 +3,7 @@ # - compat_os # - repo_name # - repo_addr +# Git repo jobs only use the first two parameters, SRPMs jobs use all . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh -- 2.23.0

1 0

[PATCH v2 lkp-tests] stat: optimize to obtain error for build DockerFile
by Liu Shaofei 02 Apr '21

02 Apr '21

Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- stats/openeuler_docker.rb | 101 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 stats/openeuler_docker.rb diff --git a/stats/openeuler_docker.rb b/stats/openeuler_docker.rb new file mode 100755 index 000000000..706b5b249 --- /dev/null +++ b/stats/openeuler_docker.rb @@ -0,0 +1,101 @@ +#!/usr/bin/env ruby + +def pre_handel(result, file_path) + status = false + repo_set = Set[] + sys_set = Set[] + + File.readlines(file_path).each do |line| + case line.chomp! + # Error: Unable to find a match: docker-registry mock xx + when /Error: Unable to find a match: (.+)/ + $1.split.each do |repo| + repo_set << repo + end + + # RUN yum -y swap -- remove fakesystemd -- install systemd systemd-libs + # yum swap: error: unrecognized arguments: install systemd systemd-libs + when /yum swap: error: .*: install (.+)/ + $1.split.each do |sys| + sys_set << sys + end + + # curl: (22) The requested URL returned error: 404 Not Found + # error: skipping https://dl.fedoraproject.org/pub/epel/bash-latest-7.noarch.rpm - transfer failed + when /.*error: .* (https.*)/ + result['requested-URL-returned.error'] = [1] + result['requested-URL-returned.error.message'] = [line.to_s] + status = true + + # Error: Unknown repo: 'powertools' + when /Error: Unknown repo: (.+)/ + repo = $1.delete!("'") + result["unknown-repo.#{repo}"] = [1] + result["unknown-repo.#{repo}.message"] = [line.to_s] + status = true + + # Error: Module or Group 'convert' does not exist. + when /Error: Module or Group ('[^\s]+')/ + repo = $1.delete!("'") + result["error.not-exist-module-or-group.#{repo}"] = [1] + result["error.not-exist-module-or-group.#{repo}.message"] = [line.to_s] + status = true + # /bin/sh: passwd: command not found + when /\/bin\/sh: (.+): command not found/ + result["sh.command-not-found.#{$1}"] = [1] + result["sh.command-not-found.#{$1}.message"] = [line.to_s] + status = true + end + + repo_set.each do |repo| + result["yum.error.Unable-to-find-a-match.#{repo}"] = [1] + result["yum.error.Unable-to-find-a-match.#{repo}.message"] = ["Error: Unable to find a match #{repo}"] + status = true + end + + sys_set.each do |sys| + result["yum.swap.error.unrecognized-arguments-install.#{sys}"] = [1] + result["yum.swap.error.unrecognized-arguments.#{sys}.message"] = + ["yum swap: error: unrecognized arguments install #{sys}"] + status = true + end + end + status +end + +def handle_unknown_error(_result, file_path) + line_num = %x(cat #{file_path} | grep -n 'Step ' | tail -1 | awk -F: '{print $1}') + + index = 1 + message = '' + File.readlines(file_path).each do |line| + if index == Integer(line_num) + message += line + else + index += 1 + end + end + + message = $1 if message =~ %r(\u001b\[91m(.+)) + message +end + +def openeuler_docker(log_lines) + result = Hash.new { |hash, key| hash[key] = [] } + + log_lines.each do |line| + next unless line =~ %r(([^\s]+).(build|run)\.fail) + + key, value = line.split(':') + key.chomp! + result[key] << value.to_i + + file_path = "#{RESULT_ROOT}/#{$1}" # $1 named by docker-image name + next unless File.exist?(file_path) + next if pre_handel(result, file_path) + + result["#{key}.message"] << handle_unknown_error(result, file_path) + end + + result +end -- 2.23.0

3 3

[PATCH compass-ci] sched: stop ectd watcher before close etcd client
by Cao Xueliang 01 Apr '21

01 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/scheduler/find_job_boot.cr | 43 +++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 6e8d5d1..74a27af 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -129,13 +129,26 @@ class Sched jobs += job.kvs end + ec.close + return jobs, revisions.min end def consume_by_watch(queues, revision) ready_queues = split_ready_queues(queues) - channel = watch_queues(ready_queues, revision) - loop_handle_event(channel) + + channel = Channel(Array(Etcd::Model::WatchEvent)).new + ech = Hash(EtcdClient, Etcd::Watch::Watcher).new + ready_queues.each do |queue| + ec = EtcdClient.new + watcher = ec.watch_prefix(queue, start_revision: revision.to_i64, progress_notify: false, filters: [Etcd::Watch::Filter::NODELETE]) do |events| + channel.send(events) + end + ech[ec] = watcher + end + + watchers = start_watcher(ech) + loop_handle_event(channel, ech) end def split_ready_queues(queues) @@ -148,25 +161,25 @@ class Sched ready_queues.uniq end - def watch_queues(queues, revision) - puts "watch #{queues}, revision #{revision}" - channel = Channel(Array(Etcd::Model::WatchEvent)).new - queues.each do |queue| - watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64, progress_notify: false ,filters: [Etcd::Watch::Filter::NODELETE]) do |events| - channel.send(events) - end + def start_watcher(ech) + ech.each do |ec, watcher| spawn { watcher.start } Fiber.yield end - - return channel end - def loop_handle_event(channel) + def loop_handle_event(channel, ech) while true events = channel.receive events.each do |event| - return event.kv if ready2process(event.kv) + if ready2process(event.kv) + ech.each do |ec, watcher| + watcher.stop + ec.close + end + + return event.kv + end end end end @@ -176,7 +189,9 @@ class Sched f_queue = job.key t_queue = f_queue.gsub("/ready/", "/in_process/") value = job.value - ec.move(f_queue, t_queue, value) + res = ec.move(f_queue, t_queue, value) + ec.close + return res end def get_job_boot(host, boot_type) -- 2.23.0

1 0

[PATCH compass-ci] sched: close etcd client in every http request
by Cao Xueliang 01 Apr '21

01 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/lib/etcd_client.cr | 4 ++++ src/lib/sched.cr | 4 ++++ src/scheduler/scheduler.cr | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/src/lib/etcd_client.cr b/src/lib/etcd_client.cr index 7891eb3..e0d4239 100644 --- a/src/lib/etcd_client.cr +++ b/src/lib/etcd_client.cr @@ -13,6 +13,10 @@ class EtcdClient @etcd = Etcd.client(host, port, version) end + def close + @etcd.close + end + def put(queue, content) queue = "#{BASE}/#{queue}" unless queue.starts_with?(BASE) @etcd.kv.put_not_exists(queue, content) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 6fd6657..7e01221 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -49,6 +49,10 @@ class Sched @log.info(%({"from": "#{(a)env.request.remote_address}", "response": #{response.to_json}})) end + def etcd_close + @etcd.close + end + def alive(version) debug_message("Env= {\n#{`export`}}") "LKP Alive! The time is #{Time.local}, version = #{version}" diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr index 6b032ee..ee13fd3 100644 --- a/src/scheduler/scheduler.cr +++ b/src/scheduler/scheduler.cr @@ -41,6 +41,10 @@ module Scheduler env.create_sched end + after_all do |env| + env.sched.etcd_close + end + # echo alive get "/" do |env| env.sched.alive(VERSION) -- 2.23.0

1 0