- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci] src/lib/lifecycle.cr: use close_job close job
by Wei Jihui 30 Mar '21

30 Mar '21

unified use close_job to close job, reduce code Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- src/lib/lifecycle.cr | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 4bd2ac0..adcb965 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -177,8 +177,7 @@ class Lifecycle event_job_id = event["job_id"].to_s return unless @jobs.has_key?(event_job_id) - @jobs.delete(event_job_id) - spawn @scheduler_api.close_job(event_job_id, "abnormal", "lifecycle") + close_job(event_job_id, "abnormal") end def on_job_close(event) @@ -222,8 +221,7 @@ class Lifecycle return if machine_job_id.empty? return unless @jobs.has_key?(machine_job_id) - @jobs.delete(machine_job_id) - spawn @scheduler_api.close_job(machine_job_id, "abnormal", "lifecycle") + close_job(machine_job_id, "abnormal") end def max_time(times) -- 2.23.0

1 0

[PATCH compass-ci 3/4] container/rpm-repo: add rpm-repo.rb for update repodata
by Li Ping 30 Mar '21

30 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/rpm-repo/rpm-repo.rb | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 container/rpm-repo/rpm-repo.rb diff --git a/container/rpm-repo/rpm-repo.rb b/container/rpm-repo/rpm-repo.rb new file mode 100755 index 0000000..18f0507 --- /dev/null +++ b/container/rpm-repo/rpm-repo.rb @@ -0,0 +1,73 @@ +#!/usr/bin/ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "bunny" +require "fileutils" +require "json" + + +MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || '172.17.0.1' +MQ_PORT = ENV['MQ_PORT'] || 5672 + +class MQClient + def initialize(hostname = "172.17.0.1", port = "5672") + @conn = Bunny.new(hostname: hostname, port: port) + @conn.start + @channel = @conn.create_channel + end + + def queue(queue_name, opts = {}) + @channel.queue(queue_name, opts) + end + + def ack(delivery_info) + @channel.ack(delivery_info.delivery_tag) + end +end + +class HandleRepo + def initialize + @mq = MQClient.new(MQ_HOST, MQ_PORT) + @update = [] + end + + def handle_new_rpm + queue = @mq.queue("update_repo") + queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| + rpm_info = JSON.parse(msg) + puts rpm_info + rpm_info["upload_rpms"].each do |rpm| + rpm_path = File.dirname(rpm).sub("upload", "testing") + FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) + + dest = File.join(rpm_path.to_s, File.basename(rpm)) + @update << dest + FileUtils.mv(rpm, dest) + system("createrepo --update $(dirname #{rpm_path})") + end + update_pub_dir + @mq.ack(info) + end + end + + def update_pub_dir + @update.each do |rpm| + pub_path = File.dirname(rpm).sub("testing", "pub") + FileUtils.mkdir_p(pub_path) unless File.directory?(pub_path) + + dest = File.join(pub_path, File.basename(rpm)) + FileUtils.cp(rpm, dest) + + repodata_dest = File.join(File.dirname(pub_path), "repodata") + repodata_src = File.dirname(rpm).sub("Packages", "repodata") + + FileUtils.rm_r(repodata_dest) if Dir.exist?(repodata_dest) + FileUtils.cp_r(repodata_src, File.dirname(repodata_dest)) + end + end +end + +hr = HandleRepo.new +hr.handle_new_rpm -- 2.23.0

4 5

[PATCH compass-ci 5/5] container/updaterepo: add update-repo start script
by Li Ping 30 Mar '21

30 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/updaterepo/start | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 container/updaterepo/start diff --git a/container/updaterepo/start b/container/updaterepo/start new file mode 100755 index 0000000..46e3ad6 --- /dev/null +++ b/container/updaterepo/start @@ -0,0 +1,24 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative '../defconfig.rb' + +docker_rm "update-repo" + +cmd = %W[ + docker run + --restart=always + --name update-repo + -u 1090:1090 + -p 8132:8132 + -d + -v /srv/rpm/:/srv/rpm/ + -v /etc/localtime:/etc/localtime:ro + update-repo +] + +cmd += ['sh', '-c', 'umask 002 && ./updaterepo'] + +system(*cmd) -- 2.23.0

2 2

[PATCH compass-ci 2/5] src/lib/updaterepo.cr: send messages to update-repo queue
by Li Ping 30 Mar '21

30 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/updaterepo.cr | 60 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/lib/updaterepo.cr diff --git a/src/lib/updaterepo.cr b/src/lib/updaterepo.cr new file mode 100644 index 0000000..4826fac --- /dev/null +++ b/src/lib/updaterepo.cr @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +require "json" +require "kemal" +require "./json_logger" +require "./mq" + +class Repo + def initialize(env : HTTP::Server::Context) + @mq = MQClient.instance + @env = env + @log = JSONLogger.new + end + + def upload_repo + begin + body = @env.request.body.not_nil!.gets_to_end + data = JSON.parse(body.to_s).as_h? + rescue e + @log.error(e) + end + + puts "-" * 50 + puts "post body:", data + + begin + # check if the parameters are complete + check_params_complete(data) + mq_publish(data) + rescue e + response = { "errcode" => "101", "errmsg" => "upload rpm failed" } + @log.info(response.to_json) + puts "error message: #{e.message}" + end + end + + def mq_publish(data) + mq_msg = data + puts mq_msg.to_json + + spawn mq_publish_check("update_repo", mq_msg.to_json) + end + + def mq_publish_check(queue, msg) + 3.times do + @mq.publish_confirm(queue, msg) + break + rescue e + res = @mq.reconnect + sleep 5 + end + end + + def check_params_complete(params) + params = params.not_nil! + tmp_hash = {"errcode" => "101", "errmsg" => "no upload_rpms params"}.to_json + raise tmp_hash unless params["upload_rpms"]? + end +end -- 2.23.0

2 2

[PATCH v3 compass-ci 3/3] sparrow/3-code/dev-env: add key EMAIL_LIMIT_COUNT
by Luan Shengde 30 Mar '21

30 Mar '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- sparrow/3-code/dev-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 27258fc..7327083 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -12,6 +12,7 @@ server_name=$(hostname | cut -f1 -d.) : ${OS_HTTP_HOST:=$server_ip} : ${send_mail_host:=$server_ip} : ${SEND_MAIL_PORT:=10001} +: ${EMAIL_LIMIT_COUNT:=10} : ${srv_http_result_host:=$server_ip} : ${srv_http_os_host:=$server_ip} : ${srv_http_git_host:=$server_ip} @@ -90,6 +91,7 @@ SRV_HTTP_CCI_PORT: $SRV_HTTP_CCI_PORT TASKQUEUE_HOST: $sched_host SEND_MAIL_HOST: $send_mail_host SEND_MAIL_PORT: $SEND_MAIL_PORT +EMAIL_LIMIT_COUNT: $EMAIL_LIMIT_COUNT LOGGING_ES_HOST: $LOGGING_ES_HOST LOGGING_ES_PORT: $LOGGING_ES_PORT INITRD_HTTP_HOST: $initrd_http_host -- 2.23.0

1 0

[PATCH v3 compass-ci 2/3] lib/mail-post/email_limit_queue: use ENV instead of static value
by Luan Shengde 30 Mar '21

30 Mar '21

use ENV variable instead of static value. enable user-defined new value without edit the code. just change the key and restart the service. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- lib/mail-post/email_limit_queue.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mail-post/email_limit_queue.rb b/lib/mail-post/email_limit_queue.rb index c7e3954..62e9634 100755 --- a/lib/mail-post/email_limit_queue.rb +++ b/lib/mail-post/email_limit_queue.rb @@ -54,9 +54,9 @@ class EmailRateLimit end def change_queue(email) - return unless (@redis.hget 'email_in_limit', email).to_i >= 10 + return unless (@redis.hget 'email_in_limit', email).to_i >= ENV['EMAIL_LIMIT_COUNT'].to_i @redis.hdel 'email_in_limit', email - @redis.hset 'email_out_limit', email, 10 + @redis.hset 'email_out_limit', email, ENV['EMAIL_LIMIT_COUNT'] end end -- 2.23.0

1 0

[PATCH v3 compass-ci 1/3] container/send-internet-mail: add key EMAIL_LIMIT_COUNT
by Luan Shengde 30 Mar '21

30 Mar '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/send-internet-mail/start b/container/send-internet-mail/start index 136435d..57bf1f6 100755 --- a/container/send-internet-mail/start +++ b/container/send-internet-mail/start @@ -14,6 +14,7 @@ names = Set.new %w[ SEND_MAIL_PORT SENT_MAILDIR REDIS_PORT + EMAIL_LIMIT_COUNT ] defaults = relevant_defaults(names) -- 2.23.0

1 0

[PATCH v2 compass-ci] sparrow/3-code/dev-env: rename defaults vars
by Liu Yinsi 30 Mar '21

30 Mar '21

before: : {$A:=$server_ip}; $A=$A after: : {$a:=$server_ip}; $A=$a [why] when submit a job to run a testbox, sched will get variables(from /etc/compass-ci/scheduler), and export them in a running testbox env, but for selftest, it will deploy a new Compass CI, no need to use variables from the host(crystal or z9), it should be the testbox own server ip, so change defaults variables name. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 8cb9aa5..27258fc 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -10,23 +10,23 @@ server_name=$(hostname | cut -f1 -d.) : ${ES_HOST:=$server_ip} : ${ES_PORT:=9200} : ${OS_HTTP_HOST:=$server_ip} -: ${SEND_MAIL_HOST:=$server_ip} +: ${send_mail_host:=$server_ip} : ${SEND_MAIL_PORT:=10001} -: ${SRV_HTTP_RESULT_HOST:=$server_ip} -: ${SRV_HTTP_OS_HOST:=$server_ip} -: ${SRV_HTTP_GIT_HOST:=$server_ip} -: ${SRV_HTTP_CCI_HOST:=$server_ip} +: ${srv_http_result_host:=$server_ip} +: ${srv_http_os_host:=$server_ip} +: ${srv_http_git_host:=$server_ip} +: ${srv_http_cci_host:=$server_ip} : ${SRV_HTTP_RESULT_PORT:=20007} : ${SRV_HTTP_OS_PORT:=20009} : ${SRV_HTTP_GIT_PORT:=20010} : ${SRV_HTTP_CCI_PORT:=20011} : ${LOGGING_ES_HOST:=$server_ip} : ${LOGGING_ES_PORT:=9202} -: ${INITRD_HTTP_HOST:=$server_ip} +: ${initrd_http_host:=$server_ip} : ${INITRD_HTTP_PORT:=8800} -: ${ASSIST_RESULT_HOST:=$server_ip} +: ${assist_result_host:=$server_ip} : ${ASSIST_RESULT_PORT:=8102} -: ${RESULT_WEBDAV_HOST:=$server_ip} +: ${result_webdav_host:=$server_ip} : ${RESULT_WEBDAV_PORT:=3080} : ${MASTER_FLUENTD_HOST:=$server_ip} : ${MASTER_FLUENTD_PORT:=24224} @@ -34,7 +34,7 @@ server_name=$(hostname | cut -f1 -d.) : ${DOCKER_REGISTRY_PORT:=5001} : ${LOCAL_SEND_MAIL_PORT:=11311} : ${LOCAL_ROBOT_EMAIL_ADDRESS:=Crystal TEAM} -: ${GIT_SERVER:=$server_ip} +: ${git_server:=$server_ip} : ${SCHED_HTTPS_HOST:=$server_ip} : ${SCHED_HTTPS_PORT:=20002} : ${DOCKER_PROXY_HOST:=$server_ip} @@ -77,26 +77,26 @@ ES_PORT: $ES_PORT SCHED_HOST: $sched_host SCHED_PORT: $sched_port LKP_SERVER: $server_ip -GIT_SERVER: $GIT_SERVER +GIT_SERVER: $git_server OS_HTTP_HOST: $OS_HTTP_HOST -SRV_HTTP_RESULT_HOST: $SRV_HTTP_RESULT_HOST -SRV_HTTP_OS_HOST: $SRV_HTTP_OS_HOST -SRV_HTTP_GIT_HOST: $SRV_HTTP_GIT_HOST -SRV_HTTP_CCI_HOST: $SRV_HTTP_CCI_HOST +SRV_HTTP_RESULT_HOST: $srv_http_result_host +SRV_HTTP_OS_HOST: $srv_http_os_host +SRV_HTTP_GIT_HOST: $srv_http_git_host +SRV_HTTP_CCI_HOST: $srv_http_cci_host SRV_HTTP_RESULT_PORT: $SRV_HTTP_RESULT_PORT SRV_HTTP_OS_PORT: $SRV_HTTP_OS_PORT SRV_HTTP_GIT_PORT: $SRV_HTTP_GIT_PORT SRV_HTTP_CCI_PORT: $SRV_HTTP_CCI_PORT TASKQUEUE_HOST: $sched_host -SEND_MAIL_HOST: $SEND_MAIL_HOST +SEND_MAIL_HOST: $send_mail_host SEND_MAIL_PORT: $SEND_MAIL_PORT LOGGING_ES_HOST: $LOGGING_ES_HOST LOGGING_ES_PORT: $LOGGING_ES_PORT -INITRD_HTTP_HOST: $INITRD_HTTP_HOST +INITRD_HTTP_HOST: $initrd_http_host INITRD_HTTP_PORT: $INITRD_HTTP_PORT -ASSIST_RESULT_HOST: $ASSIST_RESULT_HOST +ASSIST_RESULT_HOST: $assist_result_host ASSIST_RESULT_PORT: $ASSIST_RESULT_PORT -RESULT_WEBDAV_HOST: $RESULT_WEBDAV_HOST +RESULT_WEBDAV_HOST: $result_webdav_host RESULT_WEBDAV_PORT: $RESULT_WEBDAV_PORT MASTER_FLUENTD_HOST: $MASTER_FLUENTD_HOST MASTER_FLUENTD_PORT: $MASTER_FLUENTD_PORT -- 2.23.0

1 0

[PATCH compass-ci 1/4] container/rpm-repo: add Dockerfile
by Li Ping 30 Mar '21

30 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/rpm-repo/Dockerfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 container/rpm-repo/Dockerfile diff --git a/container/rpm-repo/Dockerfile b/container/rpm-repo/Dockerfile new file mode 100644 index 0000000..ae340e8 --- /dev/null +++ b/container/rpm-repo/Dockerfile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM fedora:latest + +RUN yum repolist +RUN yum install -y createrepo ruby && \ + yum clean all + +RUN umask 002 && \ + gem install bunny + +COPY --chown=1090:1090 rpm-repo.rb /usr/local/bin/ +CMD ["/usr/local/bin/rpm-repo.rb"] -- 2.23.0

2 2

[PATCH compass-ci 4/4] container/rpm-repo: add start script
by Li Ping 30 Mar '21

30 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/rpm-repo/start | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 container/rpm-repo/start diff --git a/container/rpm-repo/start b/container/rpm-repo/start new file mode 100755 index 0000000..3027855 --- /dev/null +++ b/container/rpm-repo/start @@ -0,0 +1,33 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig.rb' + +names = Set.new %w[ + MQ_HOST + LKP_SERVER +] + +defaults = relevant_defaults(names) +defaults['MQ_HOST'] ||= defaults['LKP_SERVER'] ||= '172.17.0.1' +defaults['LKP_SERVER'] ||= '172.17.0.1' +env = docker_env(defaults) + +docker_rm "update-repodata" + +cmd = %W[ + docker run + --restart=always + --name update-repodata + -u 1090:1090 +] + env + %W[ + -d + -v /etc/localtime:/etc/localtime:ro + -v /srv/rpm/:/srv/rpm/ + fedora:update-repodata +] + +system(*cmd) -- 2.23.0

2 2

[PATCH compass-ci] container/ssh-r/start: rename docker name
by Wei Jihui 30 Mar '21

30 Mar '21

vm_name --> docker_name Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- container/ssh-r/start | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/container/ssh-r/start b/container/ssh-r/start index 68a63f2..17d4493 100755 --- a/container/ssh-r/start +++ b/container/ssh-r/start @@ -4,14 +4,14 @@ . $CCI_SRC/container/defconfig.sh -vm_name=ssh_r +docker_name=ssh_r -docker_rm ${vm_name} +#docker_rm ${docker_name} cmd=( docker run --restart=always - --name ${vm_name} + --name ${docker_name} -e GATEWAY_PORTS=true -e TCP_FORWARDING=true -d -- 2.23.0

2 1

[PATCH compass-ci] update container expose port
by Cui Yili 30 Mar '21

30 Mar '21

Signed-off-by: Cui Yili <2268260388(a)qq.com> --- container/minio/start | 2 +- container/openresty-proxy-cache/start | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/container/minio/start b/container/minio/start index 1798cfa..2cbc9c8 100755 --- a/container/minio/start +++ b/container/minio/start @@ -7,7 +7,7 @@ docker_rm minio cmd=( docker run -d - -p 11321:9000 + -p 20014:9000 --name minio -v /etc/localtime:/etc/localtime:ro -v /srv/minio/:/srv/minio/ diff --git a/container/openresty-proxy-cache/start b/container/openresty-proxy-cache/start index a1f095d..59e2a28 100755 --- a/container/openresty-proxy-cache/start +++ b/container/openresty-proxy-cache/start @@ -11,8 +11,8 @@ cmd=( docker run -d --name openresty-proxy-cache --restart=always - -p 11333:80 - -p 11334:443 + -p 20015:80 + -p 20016:443 -v $PWD/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf -v /srv/initrd/tar:/srv/initrd/tar:ro -v /etc/localtime:/etc/localtime:ro -- 2.23.0

1 0

[PATCH compass-ci] crystal-shards/build: delete same annotation about gitcache
by Cui Yili 30 Mar '21

30 Mar '21

Signed-off-by: Cui Yili <2268260388(a)qq.com> --- container/crystal-shards/build | 3 --- 1 file changed, 3 deletions(-) diff --git a/container/crystal-shards/build b/container/crystal-shards/build index 00153ab..872282b 100755 --- a/container/crystal-shards/build +++ b/container/crystal-shards/build @@ -2,9 +2,6 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -# GITCACHE_HOST and GITCACHE_PORT parameters come from gitcache_host and gitcache_port in the job.yaml file you submitted. -# If job.yaml does not have these two parameters, git will use the local proxy cache, -# GITCACHE_HOST defaults to the local ip, GITCACHE_PORT default port is 5000 . ../defconfig.sh docker_skip_rebuild "alpine:crystal-shards" -- 2.23.0

1 0

[PATCH compass-ci] container: docker run param -it => -d
by Cao Xueliang 30 Mar '21

30 Mar '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/etcd/start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/etcd/start b/container/etcd/start index 7657b86..db94833 100755 --- a/container/etcd/start +++ b/container/etcd/start @@ -8,7 +8,7 @@ docker_rm etcd-server cmd=( docker run - -it + -d -p 2379:2379 -p 2380:2380 -p 4001:4001 -- 2.23.0

1 0

[PATCH compass-ci 2/4] container/rpm-repo: add build script
by Li Ping 29 Mar '21

29 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/rpm-repo/build | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 container/rpm-repo/build diff --git a/container/rpm-repo/build b/container/rpm-repo/build new file mode 100755 index 0000000..458e8f6 --- /dev/null +++ b/container/rpm-repo/build @@ -0,0 +1,6 @@ +#!/bin/sh + +. ../defconfig.sh +docker_skip_rebuild "fedora:update-repodata" + +docker build -t fedora:update-repodata . -- 2.23.0

1 0

[PATCH compass-ci] lib/git_mirror: fix rubocop errors
by Li Yuanchao 29 Mar '21

29 Mar '21

[error msg] git_mirror.rb:112:1: C: Metrics/ClassLength: Class has too many lines. [101/100] class MirrorMain ... ^^^^^^^^^^^^^^^^ git_mirror.rb:192:3: C: Metrics/AbcSize: Assignment Branch Condition size for handle_feedback is too high. [<4, 17, 6> 18.47/15] def handle_feedback ... ^^^^^^^^^^^^^^^^^^^ git_mirror.rb:286:3: C: Metrics/AbcSize: Assignment Branch Condition size for reload_fork_info is too high. [<5, 17, 2> 17.83/15] def reload_fork_info(upstream_repos) ... ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ git_mirror.rb:293:121: C: Layout/LineLength: Line is too long. [149/120] changed_files = %x(git -C /srv/git/#{@git_info[upstream_repos]['belong']}/#{upstream_repos}.git diff --name-only #{old_commit}...#{new_commit}) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ git_mirror.rb:328:3: C: Metrics/AbcSize: Assignment Branch Condition size for update_stat_new_refs is too high. [<6, 14, 2> 15.36/15] def update_stat_new_refs(git_repo) ... ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ git_mirror.rb:508:7: C: Naming/PredicateName: Rename is_upstream_repo? to upstream_repo?. def is_upstream_repo?(git_repo) ^^^^^^^^^^^^^^^^^ Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 45 +++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 5ba9154..c967c8b 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -199,14 +199,7 @@ class MirrorMain update_fork_stat(git_repo, feedback_info[:possible_new_refs]) return unless feedback_info[:possible_new_refs] - return reload_fork_info(git_repo) if is_upstream_repo?(git_repo) - - new_refs = check_new_refs(git_repo) - return if new_refs[:heads].empty? - - feedback_info[:new_refs] = new_refs - send_message(feedback_info) - new_refs_log(git_repo, new_refs[:heads].length) if last_commit_new?(git_repo) + handle_feedback_new_refs(git_repo, feedback_info) end def do_push(fork_key) @@ -283,14 +276,19 @@ class MirrorMain return new_refs end + def get_change_files(upstream_repos) + old_commit = @git_info[upstream_repos][:cur_refs][:heads]['refs/heads/master'] + new_refs = check_new_refs(upstream_repos) + new_commit = new_refs[:heads]['refs/heads/master'] + mirror_dir = "/srv/git/#{@git_info[upstream_repos]['belong']}/#{upstream_repos}.git" + %x(git -C #{mirror_dir} diff --name-only #{old_commit}...#{new_commit}) + end + def reload_fork_info(upstream_repos) if @git_info[upstream_repos][:cur_refs].empty? @git_info[upstream_repos][:cur_refs] = get_cur_refs(upstream_repos) else - old_commit = @git_info[upstream_repos][:cur_refs][:heads]['refs/heads/master'] - new_refs = check_new_refs(upstream_repos) - new_commit = new_refs[:heads]['refs/heads/master'] - changed_files = %x(git -C /srv/git/#{@git_info[upstream_repos]['belong']}/#{upstream_repos}.git diff --name-only #{old_commit}...#{new_commit}) + changed_files = get_change_files(upstream_repos) reload(changed_files, @git_info[upstream_repos]['belong']) end end @@ -325,13 +323,17 @@ class MirrorMain @fork_stat[git_repo][:offset_fetch] = offset_fetch + 1 end + def update_new_refs_info(git_repo, offset_new_refs) + @fork_stat[git_repo][:new_refs_time][offset_new_refs] = Time.now.to_s + @fork_stat[git_repo][:offset_new_refs] = offset_new_refs + 1 + @fork_stat[git_repo][:new_refs_count] = update_new_refs_count(@fork_stat[git_repo][:new_refs_count]) + end + def update_stat_new_refs(git_repo) @fork_stat[git_repo][:priority] += 1 offset_new_refs = @fork_stat[git_repo][:offset_new_refs] offset_new_refs = 0 if offset_new_refs >= 10 - @fork_stat[git_repo][:new_refs_time][offset_new_refs] = Time.now.to_s - @fork_stat[git_repo][:offset_new_refs] = offset_new_refs + 1 - @fork_stat[git_repo][:new_refs_count] = update_new_refs_count(@fork_stat[git_repo][:new_refs_count]) + update_new_refs_info(git_repo, offset_new_refs) end def update_fork_stat(git_repo, possible_new_refs) @@ -471,6 +473,17 @@ end class MirrorMain WEEK_SECONDS = 604800 + def handle_feedback_new_refs(git_repo, feedback_info) + return reload_fork_info(git_repo) if upstream_repo?(git_repo) + + new_refs = check_new_refs(git_repo) + return if new_refs[:heads].empty? + + feedback_info[:new_refs] = new_refs + send_message(feedback_info) + new_refs_log(git_repo, new_refs[:heads].length) if last_commit_new?(git_repo) + end + def merge_defaults(object_key, object, belong) return object if object_key == belong @@ -505,7 +518,7 @@ class MirrorMain return url end - def is_upstream_repo?(git_repo) + def upstream_repo?(git_repo) @upstreams['upstreams'].each do |repo| return true if git_repo == repo['git_repo'] end -- 2.23.0

2 2

[PATCH compass-ci] container: add etcd to start-depends
by Cao Xueliang 29 Mar '21

29 Mar '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/extract-stats/start-depends | 1 + container/scheduler/start-depends | 1 + 2 files changed, 2 insertions(+) diff --git a/container/extract-stats/start-depends b/container/extract-stats/start-depends index efbe5a2..abd487d 100755 --- a/container/extract-stats/start-depends +++ b/container/extract-stats/start-depends @@ -1 +1,2 @@ taskqueue +etcd diff --git a/container/scheduler/start-depends b/container/scheduler/start-depends index 1aced31..09a0b1d 100755 --- a/container/scheduler/start-depends +++ b/container/scheduler/start-depends @@ -2,3 +2,4 @@ redis es taskqueue rabbitmq +etcd -- 2.23.0

1 0

[PATCH compass-ci] src/monitor: detect crash and send msg to mq queue.
by Li Ping 29 Mar '21

29 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/monitoring/filter.cr | 2 +- src/monitoring/parse_serial_logs.cr | 50 ++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/monitoring/filter.cr b/src/monitoring/filter.cr index 39cfbb6..b663f11 100644 --- a/src/monitoring/filter.cr +++ b/src/monitoring/filter.cr @@ -56,7 +56,7 @@ class Filter msg = JSON.parse(msg.to_s).as_h? return unless msg - @sp.save_dmesg_to_result_root(msg) + @sp.deal_serial_log(msg) @hash.keys.each do |query| if match_query(query.as_h, msg) send_msg(query, msg) diff --git a/src/monitoring/parse_serial_logs.cr b/src/monitoring/parse_serial_logs.cr index 0589523..9430d8c 100644 --- a/src/monitoring/parse_serial_logs.cr +++ b/src/monitoring/parse_serial_logs.cr @@ -4,6 +4,7 @@ require "../scheduler/elasticsearch_client" require "set" require "json" +require "../lib/mq" # This parses dmesg in a stream of serial log, finding a number of patterns # in various places of the dmesg and take actions accordingly. @@ -40,9 +41,15 @@ class SerialParser "Restarting system", ] + CRASH_PATTERNS = [ + "mount.nfs: Connection timed out", + "No space left on device", + ] + def initialize @host2head = Hash(String, Array(String)).new @host2rt = Hash(String, String).new + @mq = MQClient.instance end def host_in_msg(msg) @@ -59,6 +66,14 @@ class SerialParser end end + def detect_crash(msg, host) + message = msg["message"].to_s + CRASH_PATTERNS.each do |pattern| + matched = message.match(/.*(?<crash>#{pattern})/) + return matched.named_captures["crash"] unless matched.nil? + end + end + def delete_host(msg, host, signal) boundary_signal = detect_start_or_end(msg, host, signal) return unless boundary_signal @@ -67,10 +82,43 @@ class SerialParser @host2rt.delete(host) end - def save_dmesg_to_result_root(msg) + def mq_publish(msg, host) + crash_signal = detect_crash(msg, host) + return unless crash_signal + + job_id = "" + if @host2rt.has_key?(host) + job_id = File.basename(@host2rt[host]) + end + + mq_msg = { + "job_id" => job_id, + "testbox" => host, + "time" => msg["time"]? || Time.local.to_s("%Y-%m-%dT%H:%M:%S+0800"), + "job_state" => "crash" + } + spawn mq_publish_check("job_mq", mq_msg.to_json) + end + + def mq_publish_check(queue, msg) + 3.times do + @mq.publish_confirm(queue, msg) + break + rescue e + res = @mq.reconnect + sleep 5 + end + end + + def deal_serial_log(msg) host = host_in_msg(msg) return unless host + mq_publish(msg, host) + save_dmesg_to_result_root(msg, host) + end + + def save_dmesg_to_result_root(msg, host) delete_host(msg, host, START_PATTERNS) check_save = check_save_dmesg(msg, host) -- 2.23.0

1 0

[PATCH compass-ci] src/lib: add on_job_crash
by Li Ping 29 Mar '21

29 Mar '21

[how] mq publish the crash event, lifecycle will deal the job event. when job_state is crash and the job is consumed, then close the job and reboot crash machine, if no job, just reboot crash machine. Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/lifecycle.cr | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 4bd2ac0..8f34903 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -122,6 +122,8 @@ class Lifecycle on_job_close(event) when "abnormal" on_abnormal_job(event) + when "crash" + on_job_crash(event) else on_other_job(event) end @@ -190,6 +192,17 @@ class Lifecycle update_cached_machine(job["testbox"].to_s, event) end + def on_job_crash(event) + event_job_id = event["job_id"].to_s + if @jobs[event_job_id]? + @jobs.delete(event_job_id) + spawn @scheduler_api.close_job(event_job_id, "crash", "lifecycle") + end + + testbox = event["testbox"].to_s + reboot_crash_machine(testbox, event) + end + def on_job_boot(event) event_job_id = event["job_id"]?.to_s @jobs[event_job_id] = event unless event_job_id.empty? @@ -298,6 +311,22 @@ class Lifecycle spawn @scheduler_api.close_job(job_id, reason, "lifecycle") end + def reboot_crash_machine(testbox, event) + @machines.delete(testbox) + machine = @es.get_tbox(testbox) + return unless machine + return unless event["time"].to_s.bigger_than?(machine["time"]?) + + mq_queue = get_machine_reboot_queue(testbox) + machine.as_h.delete("history") + machine.as_h["testbox"] = JSON::Any.new(testbox) + @mq.publish_confirm(mq_queue, machine.to_json, durable: true) + + machine["state"] = "rebooting_queue" + machine["time"] = Time.local.to_s("%Y-%m-%dT%H:%M:%S+0800") + @es.update_tbox(testbox, machine.as_h) + end + def reboot_timeout_machine(testbox) @machines.delete(testbox) machine = @es.get_tbox(testbox) -- 2.23.0

1 0

[PATCH compass-ci 4/5] container/updaterepo: add build script
by Li Ping 29 Mar '21

29 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/updaterepo/build | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 container/updaterepo/build diff --git a/container/updaterepo/build b/container/updaterepo/build new file mode 100755 index 0000000..594558a --- /dev/null +++ b/container/updaterepo/build @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative '../defconfig.rb' + +docker_skip_rebuild "update-repo" + +CCI_SRC = ENV['CCI_SRC'] +system "bash #{CCI_SRC}/container/compile ." +system "docker build -t update-repo ." + +system 'rm updaterepo' -- 2.23.0

1 0

[PATCH compass-ci 3/5] container/updaterepo: add update-repo Dockerfile
by Li Ping 29 Mar '21

29 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/updaterepo/Dockerfile | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 container/updaterepo/Dockerfile diff --git a/container/updaterepo/Dockerfile b/container/updaterepo/Dockerfile new file mode 100644 index 0000000..adab626 --- /dev/null +++ b/container/updaterepo/Dockerfile @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM alpine:scheduler-dev + +MAINTAINER Li Ping <1477412247(a)qq.com> + +RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories + +RUN apk update && \ + apk add --no-cache 'ruby-dev' \ + 'g++' 'gcc' 'pcre' 'libevent' 'make' 'git' 'cpio' 'bash' + +ENV UPDATE_REPO_DIR /c/cci/update-repo + +RUN mkdir -p $UPDATE_REPO_DIR && \ + chown -R 1090:1090 /c + +WORKDIR $UPDATE_REPO_DIR + +COPY --chown=1090:1090 updaterepo . -- 2.23.0

1 0

[PATCH compass-ci 1/5] add update-repo service
by Li Ping 29 Mar '21

29 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/web_env.cr | 9 +++++++++ src/updaterepo.cr | 17 +++++++++++++++++ src/updaterepo/constants.cr | 4 ++++ src/updaterepo/updaterepo.cr | 16 ++++++++++++++++ 4 files changed, 46 insertions(+) create mode 100644 src/updaterepo.cr create mode 100644 src/updaterepo/constants.cr create mode 100644 src/updaterepo/updaterepo.cr diff --git a/src/lib/web_env.cr b/src/lib/web_env.cr index b3a90a8..d47e7e5 100644 --- a/src/lib/web_env.cr +++ b/src/lib/web_env.cr @@ -4,6 +4,7 @@ require "./sched" require "./lifecycle" require "./json_logger" +require "./updaterepo" class HTTP::Server # Instances of this class are passed to an `HTTP::Server` handler. @@ -31,5 +32,13 @@ class HTTP::Server def create_lifecycle @lifecycle = Lifecycle.new(self) end + + def repo + @repo ||= create_repo + end + + def create_repo + @repo = Repo.new(self) + end end end diff --git a/src/updaterepo.cr b/src/updaterepo.cr new file mode 100644 index 0000000..05fa4fe --- /dev/null +++ b/src/updaterepo.cr @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +require "./updaterepo/updaterepo" +require "./updaterepo/constants" +require "./lib/json_logger" + + +module Updaterepo + log = JSONLogger.new + + begin + Kemal.run(REPO_PORT) + rescue e + log.error(e) + end +end diff --git a/src/updaterepo/constants.cr b/src/updaterepo/constants.cr new file mode 100644 index 0000000..c9daaf2 --- /dev/null +++ b/src/updaterepo/constants.cr @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +REPO_PORT = (ENV.has_key?("REPO_PORT") ? ENV["REPO_PORT"] : 8132).to_i32 diff --git a/src/updaterepo/updaterepo.cr b/src/updaterepo/updaterepo.cr new file mode 100644 index 0000000..7ad9d20 --- /dev/null +++ b/src/updaterepo/updaterepo.cr @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +require "kemal" + +require "../lib/web_env" +require "../lib/updaterepo" +require "../lib/json_logger" + +module Updaterepo + VERSION = "0.1.0" + post "/upload" do |env| + env.repo.upload_repo + "done" + end +end -- 2.23.0

1 0

[PATCH v2 compass-ci 3/3] sparrow/3-code/dev-env: add key EMAIL_LIMIT_COUNT
by Luan Shengde 29 Mar '21

29 Mar '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- sparrow/3-code/dev-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index f96972e..3123c2a 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -12,6 +12,7 @@ server_name=$(hostname | cut -f1 -d.) : ${OS_HTTP_HOST:=$server_ip} : ${SEND_MAIL_HOST:=$server_ip} : ${SEND_MAIL_PORT:=10001} +: ${EMAIL_LIMIT_COUNT:=10} : ${SRV_HTTP_RESULT_HOST:=$server_ip} : ${SRV_HTTP_OS_HOST:=$server_ip} : ${SRV_HTTP_GIT_HOST:=$server_ip} @@ -91,6 +92,7 @@ SRV_HTTP_CCI_PORT: $SRV_HTTP_CCI_PORT TASKQUEUE_HOST: $sched_host SEND_MAIL_HOST: $SEND_MAIL_HOST SEND_MAIL_PORT: $SEND_MAIL_PORT +EMAIL_LIMIT_COUNT: ${EMAIL_LIMIT_COUNT} LOGGING_ES_HOST: $LOGGING_ES_HOST LOGGING_ES_PORT: $LOGGING_ES_PORT INITRD_HTTP_HOST: $INITRD_HTTP_HOST -- 2.23.0

1 0

[PATCH v2 compass-ci 2/3] lib/mail-post/email_limit_queue: use ENV instead of static value
by Luan Shengde 29 Mar '21

29 Mar '21

use ENV variable instead of static value. enable user-defined new value without edit the code. just change the key and restart the service. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- lib/mail-post/email_limit_queue.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mail-post/email_limit_queue.rb b/lib/mail-post/email_limit_queue.rb index c7e3954..62e9634 100755 --- a/lib/mail-post/email_limit_queue.rb +++ b/lib/mail-post/email_limit_queue.rb @@ -54,9 +54,9 @@ class EmailRateLimit end def change_queue(email) - return unless (@redis.hget 'email_in_limit', email).to_i >= 10 + return unless (@redis.hget 'email_in_limit', email).to_i >= ENV['EMAIL_LIMIT_COUNT'].to_i @redis.hdel 'email_in_limit', email - @redis.hset 'email_out_limit', email, 10 + @redis.hset 'email_out_limit', email, ENV['EMAIL_LIMIT_COUNT'] end end -- 2.23.0

1 0

[PATCH v2 compass-ci 1/3] container/send-internet-mail: add key EMAIL_LIMIT_COUNT
by Luan Shengde 29 Mar '21

29 Mar '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/send-internet-mail/start b/container/send-internet-mail/start index 136435d..57bf1f6 100755 --- a/container/send-internet-mail/start +++ b/container/send-internet-mail/start @@ -14,6 +14,7 @@ names = Set.new %w[ SEND_MAIL_PORT SENT_MAILDIR REDIS_PORT + EMAIL_LIMIT_COUNT ] defaults = relevant_defaults(names) -- 2.23.0

1 0