April 2021 - Compass-ci - mailweb.openeuler.org

[PATCH compass-ci 2/4] container/updaterepo: add a new docker service
by Li Ping 06 Apr '21

06 Apr '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/updaterepo/Dockerfile | 21 +++++++++++++++++++++ container/updaterepo/build | 14 ++++++++++++++ container/updaterepo/start | 24 ++++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 container/updaterepo/Dockerfile create mode 100755 container/updaterepo/build create mode 100755 container/updaterepo/start diff --git a/container/updaterepo/Dockerfile b/container/updaterepo/Dockerfile new file mode 100644 index 0000000..adab626 --- /dev/null +++ b/container/updaterepo/Dockerfile @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM alpine:scheduler-dev + +MAINTAINER Li Ping <1477412247(a)qq.com> + +RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories + +RUN apk update && \ + apk add --no-cache 'ruby-dev' \ + 'g++' 'gcc' 'pcre' 'libevent' 'make' 'git' 'cpio' 'bash' + +ENV UPDATE_REPO_DIR /c/cci/update-repo + +RUN mkdir -p $UPDATE_REPO_DIR && \ + chown -R 1090:1090 /c + +WORKDIR $UPDATE_REPO_DIR + +COPY --chown=1090:1090 updaterepo . diff --git a/container/updaterepo/build b/container/updaterepo/build new file mode 100755 index 0000000..594558a --- /dev/null +++ b/container/updaterepo/build @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative '../defconfig.rb' + +docker_skip_rebuild "update-repo" + +CCI_SRC = ENV['CCI_SRC'] +system "bash #{CCI_SRC}/container/compile ." +system "docker build -t update-repo ." + +system 'rm updaterepo' diff --git a/container/updaterepo/start b/container/updaterepo/start new file mode 100755 index 0000000..4ac1925 --- /dev/null +++ b/container/updaterepo/start @@ -0,0 +1,24 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative '../defconfig.rb' + +docker_rm "update-repo" + +cmd = %W[ + docker run + --restart=always + --name update-repo + -u 1090:1090 + -p 10003:10003 + -d + -v /srv/rpm/:/srv/rpm/ + -v /etc/localtime:/etc/localtime:ro + update-repo +] + +cmd += ['sh', '-c', 'umask 002 && ./updaterepo'] + +system(*cmd) -- 2.23.0

1 0

[PATCH compass-ci 1/4] container/rpm-repo: add a new docker service
by Li Ping 06 Apr '21

06 Apr '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/rpm-repo/Dockerfile | 10 ++++++ container/rpm-repo/build | 6 ++++ container/rpm-repo/start | 41 ++++++++++++++++++++++ lib/rpm-repo.rb | 65 +++++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+) create mode 100644 container/rpm-repo/Dockerfile create mode 100755 container/rpm-repo/build create mode 100755 container/rpm-repo/start create mode 100755 lib/rpm-repo.rb diff --git a/container/rpm-repo/Dockerfile b/container/rpm-repo/Dockerfile new file mode 100644 index 0000000..588828a --- /dev/null +++ b/container/rpm-repo/Dockerfile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM fedora:latest + +RUN yum install -y createrepo ruby + +RUN umask 002 && \ + gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ + gem install bunny diff --git a/container/rpm-repo/build b/container/rpm-repo/build new file mode 100755 index 0000000..458e8f6 --- /dev/null +++ b/container/rpm-repo/build @@ -0,0 +1,6 @@ +#!/bin/sh + +. ../defconfig.sh +docker_skip_rebuild "fedora:update-repodata" + +docker build -t fedora:update-repodata . diff --git a/container/rpm-repo/start b/container/rpm-repo/start new file mode 100755 index 0000000..08fa434 --- /dev/null +++ b/container/rpm-repo/start @@ -0,0 +1,41 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig.rb' + +DOCKER_CCI = '/c/compass-ci' + +names = Set.new %w[ + MQ_HOST + MQ_PORT + LKP_SERVER +] + +defaults = relevant_defaults(names) +defaults['MQ_HOST'] ||= defaults['LKP_SERVER'] ||= '172.17.0.1' +defaults['LKP_SERVER'] ||= '172.17.0.1' +defaults['MQ_PORT'] ||= 5672 +env = docker_env(defaults) + +docker_rm "update-repodata" + +cmd = %W[ + docker run + --restart=always + --name update-repodata + -u 1090:1090 +] + env + %W[ + -d + -e CCI_SRC=#{DOCKER_CCI} + -v #{ENV['CCI_SRC']}:#{DOCKER_CCI} + -v /etc/localtime:/etc/localtime:ro + -v /srv/rpm:/srv/rpm + -w /c/compass-ci/lib + fedora:update-repodata +] + +cmd += ['sh', '-c', 'umask 002 && ./rpm-repo.rb'] +system(*cmd) diff --git a/lib/rpm-repo.rb b/lib/rpm-repo.rb new file mode 100755 index 0000000..f516d3f --- /dev/null +++ b/lib/rpm-repo.rb @@ -0,0 +1,65 @@ +#!/usr/bin/ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'fileutils' +require 'json' +require './mq_client' + +MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || '172.17.0.1' +MQ_PORT = ENV['MQ_PORT'] || 5672 + +# used to handle the mq queue "update_repo" +# +# Example items in @mq.queue "update_repo": { "upload_rpms" => ["/srv/rpm/upload/**/Packages/*.rpm", "/srv/rpm/upload/**/source/*.rpm"]} +# handle_new_rpm +# move /srv/rpm/upload/**/*.rpm to /srv/rpm/testing/**/*.rpm +# update /srv/rpm/testing/**/repodate +# update_pub_dir +# copy /srv/rpm/testing/**/*.rpm to /srv/rpm/pub/**/*.rpm +# change /srv/rpm/pub/**/repodate +# @mq.ack(info) +class HandleRepo + def initialize + @mq = MQClient.new(MQ_HOST, MQ_PORT) + @update = [] + end + + def handle_new_rpm + queue = @mq.queue("update_repo") + queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| + rpm_info = JSON.parse(msg) + rpm_info["upload_rpms"].each do |rpm| + rpm_path = File.dirname(rpm).sub("upload", "testing") + FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) + + dest = File.join(rpm_path.to_s, File.basename(rpm)) + @update << dest + FileUtils.mv(rpm, dest) + system("createrepo --update $(dirname #{rpm_path})") + end + update_pub_dir + @mq.ack(info) + end + end + + def update_pub_dir + @update.each do |rpm| + pub_path = File.dirname(rpm).sub("testing", "pub") + FileUtils.mkdir_p(pub_path) unless File.directory?(pub_path) + + dest = File.join(pub_path, File.basename(rpm)) + FileUtils.cp(rpm, dest) + + repodata_dest = File.join(File.dirname(pub_path), "repodata") + repodata_src = File.dirname(rpm).sub("Packages", "repodata") + + FileUtils.rm_r(repodata_dest) if Dir.exist?(repodata_dest) + FileUtils.cp_r(repodata_src, File.dirname(repodata_dest)) + end + end +end + +hr = HandleRepo.new +hr.handle_new_rpm -- 2.23.0

1 0

[PATCH v2 lkp-tests] stats: add build-pkg.rb to generate error_id with message
by Lin Jiaxin 06 Apr '21

06 Apr '21

output like this: "build-pkg.include/common/ist.h:warning:unused-parameter‘count’[-Wunused-parameter]": 1, "build-pkg.include/common/ist.h:warning:unused-parameter‘count’[-Wunused-parameter].message": [ "include/common/ist.h:488:74: warning: unused parameter ‘count’ [-Wunused-parameter]\n static inline struct ist ist2str(char *dst, const struct ist src, size_t count)\n ^~~~~\n", "include/common/ist.h:533:77: warning: unused parameter ‘count’ [-Wunused-parameter]\n static inline struct ist ist2str_lc(char *dst, const struct ist src, size_t count )\n ^~~~~\n", "include/common/ist.h:578:77: warning: unused parameter ‘count’ [-Wunused-parameter]\n static inline struct ist ist2str_uc(char *dst, const struct ist src, size_t count )\n ^~~~~\n" ] Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- stats/build-pkg.rb | 74 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100755 stats/build-pkg.rb diff --git a/stats/build-pkg.rb b/stats/build-pkg.rb new file mode 100755 index 00000000..4515bf23 --- /dev/null +++ b/stats/build-pkg.rb @@ -0,0 +1,74 @@ +#!/usr/bin/env ruby + +LKP_SRC ||= ENV['LKP_SRC'] || File.dirname(File.dirname(File.realpath($PROGRAM_NAME))) + +require 'set' +require "#{LKP_SRC}/lib/common" + +class ErrorMessages + def initialize + @seqno = '' + @in_stderr = false + @error_line = '' + @error_details = '' + @error_message = {} + @error_messages = Hash.new { |h, k| h[k] = Set.new } + end + + def obtain_error_messages(log_lines) + log_lines.each do |line| + next if extract_error_message(line) + next unless @in_stderr + next unless @error_message['error_line'] =~ /(error|warning):[^:]/i + + error_id = build_pkg_error_id(@error_message['error_line']) + @error_messages[error_id] << @error_message['error_line'] + @error_message['error_details'] + end + return @error_messages + end + + private + + def extract_error_message(line) + if line =~ /^ / + @error_details += line + else + update_error_message(line) + case line + when /^## ______________([0-9.]+):stderr$/ + @in_stderr = true + @seqno = $1 + return true + + when /^## ______________#{@seqno}:enderr$/ + @in_stderr = false + @seqno = '' + add_error_message + end + end + end + + def update_error_message(line) + @error_message['error_line'] = @error_line + @error_message['error_details'] = @error_details + @error_line = line + @error_details = '' + end + + def add_error_message + return unless @error_message['error_line'] =~ /(error|warning):[^:]/i + + error_id = build_pkg_error_id(@error_message['error_line']) + @error_messages[error_id] << @error_message['error_line'] + @error_message['error_details'] + end +end + +def build_pkg(log_lines) + message = {} + ErrorMessages.new.obtain_error_messages(log_lines).each do |k, v| + message[k] = '1' + message["#{k}.message"] = v.to_a + end + + return message +end -- 2.23.0

1 0

[PATCH compass-ci] sched: consume job by priority
by Cao Xueliang 06 Apr '21

06 Apr '21

before: consume job by shuffle after: our priority is reflected in email account for now, consume job by priority and the delimiter@localhost has the highest priority Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/scheduler/find_job_boot.cr | 40 ++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 01b6f1d..a0e5220 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -110,15 +110,47 @@ class Sched def consume_by_list(queues) jobs, revision = get_history_jobs(queues) - jobs.shuffle! - while true - return nil, revision if jobs.empty? + email_jobs = split_jobs_by_email(jobs) + loop do + return nil, revision if email_jobs.empty? + + job = pop_job_by_priority(email_jobs) + return nil, revision unless job - job = jobs.delete_at(0) return job, revision if ready2process(job) end end + def split_jobs_by_email(jobs) + hash = Hash(String, Array(Etcd::Model::Kv)).new + jobs.each do |job| + key = job.key.split("/")[4] + if hash.has_key?(key) + hash[key] << job + else + hash[key] = [job] + end + end + + return hash + end + + def pop_job_by_priority(email_jobs) + delimiter = "delimiter@localhost" + if email_jobs.has_key?(delimiter) + return email_jobs[delimiter].delete_at(0) unless email_jobs[delimiter].empty? + + email_jobs.delete(delimiter) + end + + keys = rand_queues(email_jobs.keys) + keys.each do |key| + return email_jobs[key].delete_at(0) unless email_jobs[key].empty? + + email_jobs.delete(key) + end + end + def get_history_jobs(queues) revisions = [] of Int64 ec = EtcdClient.new -- 2.23.0

1 0

[PATCH compass-ci] container/srv-http: fix redirect issue
by Lu Weitao 06 Apr '21

06 Apr '21

[Why] the old srv-http-result container port is: 20007:11300 when access srv-http-result like: http://$host:20007/result # (without '/' at the end) the nginx will redirect the url to: http://$host:11300/result/ # (with '/' at the end) 11300 is the nginx service listen port, so nginx redirect with this port [How] keep both ports are same by set LISTEN_PORT with start script. - set LISTEN_PORT to ENV with start script - use $LISTEN_PORT replace default port(11300) form /etc/nginx/conf.d/default.conf(in container) with container starting Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/srv-http/root/sbin/entrypoint.sh | 2 ++ container/srv-http/start-cci | 3 ++- container/srv-http/start-git | 3 ++- container/srv-http/start-initrd | 3 ++- container/srv-http/start-os | 3 ++- container/srv-http/start-pub | 3 ++- container/srv-http/start-result | 3 ++- container/srv-http/start-rpm | 3 ++- 8 files changed, 16 insertions(+), 7 deletions(-) diff --git a/container/srv-http/root/sbin/entrypoint.sh b/container/srv-http/root/sbin/entrypoint.sh index cd73c7e..ced6eda 100755 --- a/container/srv-http/root/sbin/entrypoint.sh +++ b/container/srv-http/root/sbin/entrypoint.sh @@ -1,4 +1,6 @@ #!/bin/sh +sed -i "s/listen 11300;/listen $LISTEN_PORT;/g" /etc/nginx/conf.d/default.conf + nginx /usr/sbin/php-fpm7 diff --git a/container/srv-http/start-cci b/container/srv-http/start-cci index 00d8e4c..aa539d2 100755 --- a/container/srv-http/start-cci +++ b/container/srv-http/start-cci @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-cci - -p 20011:11300 + -e LISTEN_PORT=20011 + -p 20011:20011 -v /srv/cci/libvirt-xml:/srv/cci/libvirt-xml:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-git b/container/srv-http/start-git index c1181a7..0b0ff47 100755 --- a/container/srv-http/start-git +++ b/container/srv-http/start-git @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-git - -p 20010:11300 + -e LISTEN_PORT=20010 + -p 20010:20010 -v /srv/git/archlinux:/srv/git/archlinux:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-initrd b/container/srv-http/start-initrd index 466ec1b..93be548 100755 --- a/container/srv-http/start-initrd +++ b/container/srv-http/start-initrd @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-initrd - -p 20008:11300 + -e LISTEN_PORT=20008 + -p 20008:20008 -v /srv/initrd/qemu-image:/srv/initrd/qemu-image:ro -v /srv/initrd/dockerimage:/srv/initrd/dockerimage:ro -v /srv/initrd/rpmbuild-pkg:/srv/initrd/rpmbuild-pkg:ro diff --git a/container/srv-http/start-os b/container/srv-http/start-os index d39e04e..5b63902 100755 --- a/container/srv-http/start-os +++ b/container/srv-http/start-os @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-os - -p 20009:11300 + -e LISTEN_PORT=20009 + -p 20009:20009 -v /srv/os:/srv/os:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-pub b/container/srv-http/start-pub index f42b117..6dfd5a4 100755 --- a/container/srv-http/start-pub +++ b/container/srv-http/start-pub @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-pub - -p 20006:11300 + -e LISTEN_PORT=20006 + -p 20006:20006 -v /srv/pub:/srv/pub:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-result b/container/srv-http/start-result index bf0287d..32bdf7d 100755 --- a/container/srv-http/start-result +++ b/container/srv-http/start-result @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-result - -p 20007:11300 + -e LISTEN_PORT=20007 + -p 20007:20007 -v /srv/result:/srv/result:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-rpm b/container/srv-http/start-rpm index 73e5162..2d38298 100755 --- a/container/srv-http/start-rpm +++ b/container/srv-http/start-rpm @@ -9,7 +9,8 @@ cmd=( docker run --restart=always --name srv-http-rpm - -p 20012:11300 + -e LISTEN_PORT=20012 + -p 20012:20012 -v /srv/rpm/pub:/srv/rpm/pub:ro -v /etc/localtime:/etc/localtime:ro -d -- 2.23.0

1 0

[PATCH compass-ci] service/scheduler: update data before get job
by Wu Zhende 06 Apr '21

06 Apr '21

If no job, the session is suspended until the job is obtained. Therefore, testbox message and job events need to be updated before get job to ensure the timeliness. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 11 +++++++++++ src/scheduler/find_job_boot.cr | 16 ++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 2e4fd30..f62f83e 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -147,6 +147,17 @@ class Sched @log.warn(e) end + def send_mq_msg(job_state) + mq_msg = { + "job_id" => @env.get?("job_id").to_s, + "testbox" => @env.get?("testbox").to_s, + "deadline" => @env.get?("deadline").to_s, + "time" => get_time, + "job_state" => job_state + } + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) + end + def set_lifecycle(job, testbox, queues) if job deadline = job.get_deadline diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 01b6f1d..2fca531 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -25,14 +25,7 @@ class Sched rescue e @log.warn(e) ensure - mq_msg = { - "job_id" => @env.get?("job_id").to_s, - "testbox" => @env.get?("testbox").to_s, - "deadline" => @env.get?("deadline").to_s, - "time" => get_time, - "job_state" => "boot" - } - spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) + send_mq_msg("boot") end # auto submit a job to collect the host information. @@ -196,6 +189,13 @@ class Sched def get_job_boot(host, boot_type) queues = get_queues(host) + + # do before get job from etcd + # because if no job will hang + # need to update information in a timely manner + set_lifecycle(nil, host, queues) + send_mq_msg("boot") + job = get_job_from_queues(queues, host) set_lifecycle(job, host, queues) -- 2.23.0

1 0

[PATCH lkp-tests 2/2] add script for fix dockerfile file
by Bai Jing 06 Apr '21

06 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- lib/handle_dockerfile.sh | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 lib/handle_dockerfile.sh diff --git a/lib/handle_dockerfile.sh b/lib/handle_dockerfile.sh new file mode 100644 index 000000000..b71f9865d --- /dev/null +++ b/lib/handle_dockerfile.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" +} + +handle_epel() +{ + sed -i "s/epel-release/bash/g" "$dockerfile_name" +} + +handle_epelrpm() +{ + sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" +} + +handle_rpm_gpg() +{ + sed -i "s|RPM-GPG-KEY-CentOS-7|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" + sed -i "s|RPM-GPG-KEY-centosofficial|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" +} + +add_base_commands() +{ + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" +} + +handle_epel_repo() +{ + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" +} + +handle_error_exit() +{ + sed '/^RUN/RUN set -e;/g' "$dockerfile_name" +} + +handle_dockerfile() +{ + dockerfile_name=$1 + handle_FROM + handle_epel + handle_epelrpm + handle_rpm_gpg + add_base_commands + handle_epel_repo + handle_error_exit +} -- 2.23.0

1 0

[PATCH lkp-tests 1/2] add test script for dockerfile test
by Bai Jing 06 Apr '21

06 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 143 +++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 85 deletions(-) diff --git a/tests/openeuler_docker b/tests/openeuler_docker index 483f40135..3ea57e2c9 100755 --- a/tests/openeuler_docker +++ b/tests/openeuler_docker @@ -8,12 +8,12 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/log.sh +. $LKP_SRC/lib/handle_dockerfile.sh [ -n "$dockerfile_url" ] || die "dockerfile_url is empty" declare -i nr_openeuler_fail=0 declare -i nr_openeuler_success=0 -declare -i is_openeuler_build_fail=0 declare -i dockerfile_count=0 fit_repo() @@ -33,110 +33,51 @@ openeuler_ready() docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar } -centos_docker() -{ - is_openeuler_build_fail=0 - index=1 - for file in $dockerfile_path - do - [[ $file =~ ^centos ]] && continue - file=$(echo "$file" | awk -F: '{print $1}') - dockerfile_name="${file##*/}" - cd "${file%/*}" || continue - image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') - [ -n "$image" ] && image=${image}_$index - if [ -f "build.sh" ] && [ -x "build.sh" ]; then - ./build.sh > /dev/null 2> /dev/null - else - docker_build > /dev/null 2> /dev/null - fi - - [ $? == 0 ] && { - handle_dockerfile - } - - let index++ - done -} - -handle_FROM() -{ - sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" -} - -handle_epel() -{ - grep -qw "epel-release" "$dockerfile_name" && { - sed -i "s/epel-release/bash/g" "$dockerfile_name" - } -} - -handle_epelrpm() -{ - grep -qw "https://.*epel-release.*rpm" "$dockerfile_name" && { - sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" - } -} - -handle_rpm_gpg() -{ - sed -i "s|RPM-GPG-KEY-CentOS-7|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" -} - -add_base_commands() -{ - # grep -qw "groupadd*" "$file" && { - # fix missing useradd, groupadd, chpasswd, etc. commands - sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" -} - -handle_epel_repo() -{ - sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" -} - -handle_dockerfile() +build_centos() { - is_openeuler_build_fail=1 - handle_FROM - handle_epel - handle_rpm_gpg - add_base_commands - if docker_build "-openeuler"; then - docker_run "-openeuler" - let nr_openeuler_success++ + cd ${file%/*} + if [ -f "build.sh" ] && [ -x "build.sh" ]; then + ./build.sh || return 1 else - let nr_openeuler_fail++ + docker build -f "$dockerfile_name" -t "${image}_$1_$index" . || return 1 fi } -docker_build() +build_openeuler() { - docker build -f "$dockerfile_name" -t "${image}$1" . + docker build -f "$dockerfile_name" -t "${image}_$1_$index" . >> $TMP_RESULT_ROOT/${image}_$1_$index 2>&1 [ $? != 0 ] && { - [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.fail: 1" + let nr_openeuler_fail++ + echo "${image}_$index.build.fail: 1" return 1 } - [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.success: 1" - is_openeuler_build_fail=0 + echo "${image}_$index.build.success: 1" + let nr_openeuler_success++ + return 0 } docker_run() -{ - docker run -d --name "${image}" "${image}$1" - docker_stats=$(docker inspect -f '{{.State.Running}}' ${image}) - [ $docker_stats == false ] && { - echo "${image}.run.fail: 1" +{ + docker_run_cmd="docker run -d --name ${image}_$1_$index ${image}_$1_$index" + [ $1 == "openeuler" ] && { + docker_run_cmd="${docker_run_cmd} >> $TMP_RESULT_ROOT/${image}_$1_$index 2>&1" + } + + eval $docker_run_cmd + + docker_stats=$(docker inspect -f '{{.State.Running}}' $image_$1_$index) + [ $docker_stats == "false" ] && { + [ $1 == "openeuler" ] && echo "${image}_$index.run.fail: 1" return 1 } - echo "${image}.run.success: 1" + [ $1 == "openeuler"] && echo "${image}_$index.run.success: 1" } handle_build_result() { if [ "$nr_openeuler_fail" != 0 ]; then - echo "openeuler build failed times: $nr_openeuler_fail" + echo "openeuler_build.nr_failed: $nr_openeuler_fail" echo "openeuler_valid_url: $dockerfile_url" exit 1 # centos all success, openeuler all success @@ -146,6 +87,38 @@ handle_build_result() fi } +build() +{ + dockerfile_name="${file##*/}" + dockerfile_name="${dockerfile_name%:*}" + image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') + + build_centos "centos" > /dev/null 2>&1 || return 1 + handle_dockerfile "$dockerfile_name" + build_openeuler "openeuler" + [ $? != 0 ] && return 1 +} + +run() +{ + docker_run "centos" > /dev/null 2>&1 + [ $? != 0 ] && return 1 + docker_run "openeuler" +} + +centos_docker() +{ + index=1 + for file in $dockerfile_path + do + [[ $file =~ ^centos ]] && continue + + build + [ $? = 0 ] && run + let index++ + done +} + docker_test() { fit_repo -- 2.23.0

1 0

[PATCH compass-ci] sbin/es-find: support check query keyword
by Lu Weitao 06 Apr '21

06 Apr '21

[How] 1. add --keyword option 2. use like: es-find --keyword category dequeue_time end_time enqueue_time group_id hw.arch hw.memory hw.model hw.nr_cpu ... Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/es_query.rb | 25 +++++++++++++++++++++++++ sbin/es-find | 11 ++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 928bd68..22d68a9 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -130,6 +130,11 @@ class ESQuery parse_fields(result) end + + def query_mapping + mapping = @client.indices.get_mapping(index: @index) + get_mapping_key(mapping) + end end # Range Query Example: @@ -260,3 +265,23 @@ def parse_fields(es_result) result_hash end + +def get_mapping_key(mapping) + mapping_key = [] + properties = mapping.values[0]['mappings']['properties'] + properties.each do |key, value| + parse_mapping_properties(key, value, mapping_key) + end + + mapping_key +end + +def parse_mapping_properties(key, value, mapping_key) + if value['type'] + mapping_key << key + else + value['properties'].each do |k, v| + parse_mapping_properties("#{key}.#{k}", v, mapping_key) + end + end +end diff --git a/sbin/es-find b/sbin/es-find index 2c3b3dc..f548fc1 100755 --- a/sbin/es-find +++ b/sbin/es-find @@ -77,6 +77,8 @@ options = { nr_jobs: 10, desc_keyword: 'start_time' } range = nil count = nil +es = ESQuery.new(ES_HOST, ES_PORT) + opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: es-find [options] search_key1=val1[,val2..] ..' @@ -103,6 +105,14 @@ opt_parser = OptionParser.new do |opts| count = c end + opts.on_tail('--keyword', 'check all available search_keys') do + available_search_key = es.query_mapping + available_search_key.each do |key| + puts " #{key}" + end + exit + end + opts.on_tail('-h', '--help', 'show this message') do puts opts exit @@ -113,7 +123,6 @@ opt_parser.parse!(ARGV) items = parse_argv items[:range] = prase_range(range) if range -es = ESQuery.new(ES_HOST, ES_PORT) if count count_keywords = count.split(',') query_result = es.query_fields(count_keywords, items) -- 2.23.0

1 0

[PATCH v2 compass-ci] delimiter: refactor delimiter with crystal
by Cao Xueliang 04 Apr '21

04 Apr '21

use etcd list watch to submit bisect job Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter.cr | 22 ++++++++++ src/delimiter/delimiter.cr | 83 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 src/delimiter.cr create mode 100644 src/delimiter/delimiter.cr diff --git a/src/delimiter.cr b/src/delimiter.cr new file mode 100644 index 0000000..e8de306 --- /dev/null +++ b/src/delimiter.cr @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "yaml" +require "./delimiter/delimiter" + +def config_secrets_yaml + %x(#{ENV["CCI_SRC"]}/sbin/config_account_yaml.rb delimiter) + account = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/defaults/account.yaml")) + lab = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/include/lab/#{account["lab"]}.yaml")) + secrets = {"secrets" => lab} + File.open("#{ENV["HOME"]}/.config/compass-ci/defaults/secrets.yaml", "w") { |f| YAML.dump(secrets, f) } +end + +begin + config_secrets_yaml + delimiter = Delimiter.new + delimiter.consume_delimiter("delimiter") +rescue ex + puts ex +end diff --git a/src/delimiter/delimiter.cr b/src/delimiter/delimiter.cr new file mode 100644 index 0000000..c98ded9 --- /dev/null +++ b/src/delimiter/delimiter.cr @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "../lib/etcd_client" + +class Delimiter + def initialize + @ec = EtcdClient.new + end + + def consume_delimiter(queue) + channel = Channel(Etcd::Model::Kv).new + revision = consume_by_list(queue, channel) + consume_by_watch(queue, revision, channel) + end + + def consume_by_list(queue, channel) + tasks, revision = get_history_tasks(queue) + handle_history_tasks(tasks, channel) + + return revision + end + + def consume_by_watch(queue, revision, channel) + watch_queue(queue, revision, channel) + handle_events(channel) + end + + def get_history_tasks(queue) + tasks = [] of Etcd::Model::Kv + range = @ec.range_prefix(queue) + revision = range.header.not_nil!.revision + tasks += range.kvs + + return tasks, revision + end + + def handle_history_tasks(tasks, channel) + loop do + return if tasks.empty? + + task = tasks.delete_at(0) + spawn { submit_bisect_job(channel, task) } + end + end + + def watch_queue(queue, revision, channel) + watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64 + 1, filters: [Etcd::Watch::Filter::NODELETE]) do |events| + events.each do |event| + channel.send(event.kv) + end + end + + spawn { watcher.start } + Fiber.yield + end + + def handle_events(channel) + loop do + task = channel.receive + spawn { submit_bisect_job(channel, task) } + end + end + + def submit_bisect_job(channel, task) + key = task.key + value = Hash(String, String).from_json(task.value.not_nil!) + begin + response = %x(#{ENV["LKP_SRC"]}/sbin/submit bad_job_id=#{value["job_id"]} error_id=#{value["error_id"].inspect} bisect.yaml queue=dc-bisect) + puts response + if /id=0/ =~ response + channel.send(task) + return + end + + @ec.delete(key) + rescue ex + puts ex + channel.send(task) + end + end +end -- 2.23.0

1 0