- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci] src/lib/lifecycle.cr: use close_job close job
by Wei Jihui 30 Mar '21

30 Mar '21

unified use close_job to close job, reduce code Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- src/lib/lifecycle.cr | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 4bd2ac0..adcb965 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -177,8 +177,7 @@ class Lifecycle event_job_id = event["job_id"].to_s return unless @jobs.has_key?(event_job_id) - @jobs.delete(event_job_id) - spawn @scheduler_api.close_job(event_job_id, "abnormal", "lifecycle") + close_job(event_job_id, "abnormal") end def on_job_close(event) @@ -222,8 +221,7 @@ class Lifecycle return if machine_job_id.empty? return unless @jobs.has_key?(machine_job_id) - @jobs.delete(machine_job_id) - spawn @scheduler_api.close_job(machine_job_id, "abnormal", "lifecycle") + close_job(machine_job_id, "abnormal") end def max_time(times) -- 2.23.0

1 0

[PATCH compass-ci 3/4] container/rpm-repo: add rpm-repo.rb for update repodata
by Li Ping 30 Mar '21

30 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/rpm-repo/rpm-repo.rb | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 container/rpm-repo/rpm-repo.rb diff --git a/container/rpm-repo/rpm-repo.rb b/container/rpm-repo/rpm-repo.rb new file mode 100755 index 0000000..18f0507 --- /dev/null +++ b/container/rpm-repo/rpm-repo.rb @@ -0,0 +1,73 @@ +#!/usr/bin/ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "bunny" +require "fileutils" +require "json" + + +MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || '172.17.0.1' +MQ_PORT = ENV['MQ_PORT'] || 5672 + +class MQClient + def initialize(hostname = "172.17.0.1", port = "5672") + @conn = Bunny.new(hostname: hostname, port: port) + @conn.start + @channel = @conn.create_channel + end + + def queue(queue_name, opts = {}) + @channel.queue(queue_name, opts) + end + + def ack(delivery_info) + @channel.ack(delivery_info.delivery_tag) + end +end + +class HandleRepo + def initialize + @mq = MQClient.new(MQ_HOST, MQ_PORT) + @update = [] + end + + def handle_new_rpm + queue = @mq.queue("update_repo") + queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| + rpm_info = JSON.parse(msg) + puts rpm_info + rpm_info["upload_rpms"].each do |rpm| + rpm_path = File.dirname(rpm).sub("upload", "testing") + FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) + + dest = File.join(rpm_path.to_s, File.basename(rpm)) + @update << dest + FileUtils.mv(rpm, dest) + system("createrepo --update $(dirname #{rpm_path})") + end + update_pub_dir + @mq.ack(info) + end + end + + def update_pub_dir + @update.each do |rpm| + pub_path = File.dirname(rpm).sub("testing", "pub") + FileUtils.mkdir_p(pub_path) unless File.directory?(pub_path) + + dest = File.join(pub_path, File.basename(rpm)) + FileUtils.cp(rpm, dest) + + repodata_dest = File.join(File.dirname(pub_path), "repodata") + repodata_src = File.dirname(rpm).sub("Packages", "repodata") + + FileUtils.rm_r(repodata_dest) if Dir.exist?(repodata_dest) + FileUtils.cp_r(repodata_src, File.dirname(repodata_dest)) + end + end +end + +hr = HandleRepo.new +hr.handle_new_rpm -- 2.23.0

4 5

[PATCH compass-ci 5/5] container/updaterepo: add update-repo start script
by Li Ping 30 Mar '21

30 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/updaterepo/start | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 container/updaterepo/start diff --git a/container/updaterepo/start b/container/updaterepo/start new file mode 100755 index 0000000..46e3ad6 --- /dev/null +++ b/container/updaterepo/start @@ -0,0 +1,24 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative '../defconfig.rb' + +docker_rm "update-repo" + +cmd = %W[ + docker run + --restart=always + --name update-repo + -u 1090:1090 + -p 8132:8132 + -d + -v /srv/rpm/:/srv/rpm/ + -v /etc/localtime:/etc/localtime:ro + update-repo +] + +cmd += ['sh', '-c', 'umask 002 && ./updaterepo'] + +system(*cmd) -- 2.23.0

2 2

[PATCH compass-ci 2/5] src/lib/updaterepo.cr: send messages to update-repo queue
by Li Ping 30 Mar '21

30 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/updaterepo.cr | 60 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/lib/updaterepo.cr diff --git a/src/lib/updaterepo.cr b/src/lib/updaterepo.cr new file mode 100644 index 0000000..4826fac --- /dev/null +++ b/src/lib/updaterepo.cr @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +require "json" +require "kemal" +require "./json_logger" +require "./mq" + +class Repo + def initialize(env : HTTP::Server::Context) + @mq = MQClient.instance + @env = env + @log = JSONLogger.new + end + + def upload_repo + begin + body = @env.request.body.not_nil!.gets_to_end + data = JSON.parse(body.to_s).as_h? + rescue e + @log.error(e) + end + + puts "-" * 50 + puts "post body:", data + + begin + # check if the parameters are complete + check_params_complete(data) + mq_publish(data) + rescue e + response = { "errcode" => "101", "errmsg" => "upload rpm failed" } + @log.info(response.to_json) + puts "error message: #{e.message}" + end + end + + def mq_publish(data) + mq_msg = data + puts mq_msg.to_json + + spawn mq_publish_check("update_repo", mq_msg.to_json) + end + + def mq_publish_check(queue, msg) + 3.times do + @mq.publish_confirm(queue, msg) + break + rescue e + res = @mq.reconnect + sleep 5 + end + end + + def check_params_complete(params) + params = params.not_nil! + tmp_hash = {"errcode" => "101", "errmsg" => "no upload_rpms params"}.to_json + raise tmp_hash unless params["upload_rpms"]? + end +end -- 2.23.0

2 2

[PATCH v3 compass-ci 3/3] sparrow/3-code/dev-env: add key EMAIL_LIMIT_COUNT
by Luan Shengde 30 Mar '21

30 Mar '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- sparrow/3-code/dev-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 27258fc..7327083 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -12,6 +12,7 @@ server_name=$(hostname | cut -f1 -d.) : ${OS_HTTP_HOST:=$server_ip} : ${send_mail_host:=$server_ip} : ${SEND_MAIL_PORT:=10001} +: ${EMAIL_LIMIT_COUNT:=10} : ${srv_http_result_host:=$server_ip} : ${srv_http_os_host:=$server_ip} : ${srv_http_git_host:=$server_ip} @@ -90,6 +91,7 @@ SRV_HTTP_CCI_PORT: $SRV_HTTP_CCI_PORT TASKQUEUE_HOST: $sched_host SEND_MAIL_HOST: $send_mail_host SEND_MAIL_PORT: $SEND_MAIL_PORT +EMAIL_LIMIT_COUNT: $EMAIL_LIMIT_COUNT LOGGING_ES_HOST: $LOGGING_ES_HOST LOGGING_ES_PORT: $LOGGING_ES_PORT INITRD_HTTP_HOST: $initrd_http_host -- 2.23.0

1 0

[PATCH v3 compass-ci 2/3] lib/mail-post/email_limit_queue: use ENV instead of static value
by Luan Shengde 30 Mar '21

30 Mar '21

use ENV variable instead of static value. enable user-defined new value without edit the code. just change the key and restart the service. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- lib/mail-post/email_limit_queue.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/mail-post/email_limit_queue.rb b/lib/mail-post/email_limit_queue.rb index c7e3954..62e9634 100755 --- a/lib/mail-post/email_limit_queue.rb +++ b/lib/mail-post/email_limit_queue.rb @@ -54,9 +54,9 @@ class EmailRateLimit end def change_queue(email) - return unless (@redis.hget 'email_in_limit', email).to_i >= 10 + return unless (@redis.hget 'email_in_limit', email).to_i >= ENV['EMAIL_LIMIT_COUNT'].to_i @redis.hdel 'email_in_limit', email - @redis.hset 'email_out_limit', email, 10 + @redis.hset 'email_out_limit', email, ENV['EMAIL_LIMIT_COUNT'] end end -- 2.23.0

1 0

[PATCH v3 compass-ci 1/3] container/send-internet-mail: add key EMAIL_LIMIT_COUNT
by Luan Shengde 30 Mar '21

30 Mar '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/send-internet-mail/start b/container/send-internet-mail/start index 136435d..57bf1f6 100755 --- a/container/send-internet-mail/start +++ b/container/send-internet-mail/start @@ -14,6 +14,7 @@ names = Set.new %w[ SEND_MAIL_PORT SENT_MAILDIR REDIS_PORT + EMAIL_LIMIT_COUNT ] defaults = relevant_defaults(names) -- 2.23.0

1 0

[PATCH v2 compass-ci] sparrow/3-code/dev-env: rename defaults vars
by Liu Yinsi 30 Mar '21

30 Mar '21

before: : {$A:=$server_ip}; $A=$A after: : {$a:=$server_ip}; $A=$a [why] when submit a job to run a testbox, sched will get variables(from /etc/compass-ci/scheduler), and export them in a running testbox env, but for selftest, it will deploy a new Compass CI, no need to use variables from the host(crystal or z9), it should be the testbox own server ip, so change defaults variables name. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 8cb9aa5..27258fc 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -10,23 +10,23 @@ server_name=$(hostname | cut -f1 -d.) : ${ES_HOST:=$server_ip} : ${ES_PORT:=9200} : ${OS_HTTP_HOST:=$server_ip} -: ${SEND_MAIL_HOST:=$server_ip} +: ${send_mail_host:=$server_ip} : ${SEND_MAIL_PORT:=10001} -: ${SRV_HTTP_RESULT_HOST:=$server_ip} -: ${SRV_HTTP_OS_HOST:=$server_ip} -: ${SRV_HTTP_GIT_HOST:=$server_ip} -: ${SRV_HTTP_CCI_HOST:=$server_ip} +: ${srv_http_result_host:=$server_ip} +: ${srv_http_os_host:=$server_ip} +: ${srv_http_git_host:=$server_ip} +: ${srv_http_cci_host:=$server_ip} : ${SRV_HTTP_RESULT_PORT:=20007} : ${SRV_HTTP_OS_PORT:=20009} : ${SRV_HTTP_GIT_PORT:=20010} : ${SRV_HTTP_CCI_PORT:=20011} : ${LOGGING_ES_HOST:=$server_ip} : ${LOGGING_ES_PORT:=9202} -: ${INITRD_HTTP_HOST:=$server_ip} +: ${initrd_http_host:=$server_ip} : ${INITRD_HTTP_PORT:=8800} -: ${ASSIST_RESULT_HOST:=$server_ip} +: ${assist_result_host:=$server_ip} : ${ASSIST_RESULT_PORT:=8102} -: ${RESULT_WEBDAV_HOST:=$server_ip} +: ${result_webdav_host:=$server_ip} : ${RESULT_WEBDAV_PORT:=3080} : ${MASTER_FLUENTD_HOST:=$server_ip} : ${MASTER_FLUENTD_PORT:=24224} @@ -34,7 +34,7 @@ server_name=$(hostname | cut -f1 -d.) : ${DOCKER_REGISTRY_PORT:=5001} : ${LOCAL_SEND_MAIL_PORT:=11311} : ${LOCAL_ROBOT_EMAIL_ADDRESS:=Crystal TEAM} -: ${GIT_SERVER:=$server_ip} +: ${git_server:=$server_ip} : ${SCHED_HTTPS_HOST:=$server_ip} : ${SCHED_HTTPS_PORT:=20002} : ${DOCKER_PROXY_HOST:=$server_ip} @@ -77,26 +77,26 @@ ES_PORT: $ES_PORT SCHED_HOST: $sched_host SCHED_PORT: $sched_port LKP_SERVER: $server_ip -GIT_SERVER: $GIT_SERVER +GIT_SERVER: $git_server OS_HTTP_HOST: $OS_HTTP_HOST -SRV_HTTP_RESULT_HOST: $SRV_HTTP_RESULT_HOST -SRV_HTTP_OS_HOST: $SRV_HTTP_OS_HOST -SRV_HTTP_GIT_HOST: $SRV_HTTP_GIT_HOST -SRV_HTTP_CCI_HOST: $SRV_HTTP_CCI_HOST +SRV_HTTP_RESULT_HOST: $srv_http_result_host +SRV_HTTP_OS_HOST: $srv_http_os_host +SRV_HTTP_GIT_HOST: $srv_http_git_host +SRV_HTTP_CCI_HOST: $srv_http_cci_host SRV_HTTP_RESULT_PORT: $SRV_HTTP_RESULT_PORT SRV_HTTP_OS_PORT: $SRV_HTTP_OS_PORT SRV_HTTP_GIT_PORT: $SRV_HTTP_GIT_PORT SRV_HTTP_CCI_PORT: $SRV_HTTP_CCI_PORT TASKQUEUE_HOST: $sched_host -SEND_MAIL_HOST: $SEND_MAIL_HOST +SEND_MAIL_HOST: $send_mail_host SEND_MAIL_PORT: $SEND_MAIL_PORT LOGGING_ES_HOST: $LOGGING_ES_HOST LOGGING_ES_PORT: $LOGGING_ES_PORT -INITRD_HTTP_HOST: $INITRD_HTTP_HOST +INITRD_HTTP_HOST: $initrd_http_host INITRD_HTTP_PORT: $INITRD_HTTP_PORT -ASSIST_RESULT_HOST: $ASSIST_RESULT_HOST +ASSIST_RESULT_HOST: $assist_result_host ASSIST_RESULT_PORT: $ASSIST_RESULT_PORT -RESULT_WEBDAV_HOST: $RESULT_WEBDAV_HOST +RESULT_WEBDAV_HOST: $result_webdav_host RESULT_WEBDAV_PORT: $RESULT_WEBDAV_PORT MASTER_FLUENTD_HOST: $MASTER_FLUENTD_HOST MASTER_FLUENTD_PORT: $MASTER_FLUENTD_PORT -- 2.23.0

1 0

[PATCH compass-ci 1/4] container/rpm-repo: add Dockerfile
by Li Ping 30 Mar '21

30 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/rpm-repo/Dockerfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 container/rpm-repo/Dockerfile diff --git a/container/rpm-repo/Dockerfile b/container/rpm-repo/Dockerfile new file mode 100644 index 0000000..ae340e8 --- /dev/null +++ b/container/rpm-repo/Dockerfile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM fedora:latest + +RUN yum repolist +RUN yum install -y createrepo ruby && \ + yum clean all + +RUN umask 002 && \ + gem install bunny + +COPY --chown=1090:1090 rpm-repo.rb /usr/local/bin/ +CMD ["/usr/local/bin/rpm-repo.rb"] -- 2.23.0

2 2

[PATCH compass-ci 4/4] container/rpm-repo: add start script
by Li Ping 30 Mar '21

30 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/rpm-repo/start | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 container/rpm-repo/start diff --git a/container/rpm-repo/start b/container/rpm-repo/start new file mode 100755 index 0000000..3027855 --- /dev/null +++ b/container/rpm-repo/start @@ -0,0 +1,33 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig.rb' + +names = Set.new %w[ + MQ_HOST + LKP_SERVER +] + +defaults = relevant_defaults(names) +defaults['MQ_HOST'] ||= defaults['LKP_SERVER'] ||= '172.17.0.1' +defaults['LKP_SERVER'] ||= '172.17.0.1' +env = docker_env(defaults) + +docker_rm "update-repodata" + +cmd = %W[ + docker run + --restart=always + --name update-repodata + -u 1090:1090 +] + env + %W[ + -d + -v /etc/localtime:/etc/localtime:ro + -v /srv/rpm/:/srv/rpm/ + fedora:update-repodata +] + +system(*cmd) -- 2.23.0

2 2