- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci] container/mail-robot: fix bugs
by Luan Shengde 07 Apr '21

07 Apr '21

fix no file errors: it may throw error if the specified file not exists: Errno::ENOENT: No such file or directory @ rb_sysopen Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/apply-account.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/container/mail-robot/lib/apply-account.rb b/container/mail-robot/lib/apply-account.rb index 798546a..6b9bdfe 100755 --- a/container/mail-robot/lib/apply-account.rb +++ b/container/mail-robot/lib/apply-account.rb @@ -56,7 +56,11 @@ class ApplyAccount @es_port = ES_PORT # email info file for account issuers. - @account_issuer = YAML.load_file ENV['ACCOUNT_ISSUER'] + @account_issuer = if File.exists? ENV['ACCOUNT_ISSUER'] + YAML.load_file ENV['ACCOUNT_ISSUER'] + else + {} + end @my_info = {} end -- 2.23.0

2 4

[PATCH compass-ci] lib/git_mirror.rb: bigger step to change priority
by Li Yuanchao 07 Apr '21

07 Apr '21

Make the step to change priority of repo bigger, so bad repos would be put behind faster. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index b543f02..60a4b76 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -481,7 +481,7 @@ end # main thread class MirrorMain - WEEK_SECONDS = 604800 + MONTH_SECONDS = 2592000 def handle_feedback_new_refs(git_repo, feedback_info) return reload_fork_info(git_repo) if upstream_repo?(git_repo) @@ -584,18 +584,18 @@ class MirrorMain mirror_dir = "/srv/git/#{@git_info[git_repo]['belong']}/#{git_repo}" mirror_dir = "#{mirror_dir}.git" unless @git_info[git_repo]['is_submodule'] - return old_pri + Math.cbrt(WEEK_SECONDS) unless File.directory?(mirror_dir) + return old_pri + Math.cbrt(MONTH_SECONDS) unless File.directory?(mirror_dir) return cal_priority(mirror_dir, old_pri) end def cal_priority(mirror_dir, old_pri) last_commit_time = %x(git -C #{mirror_dir} log --pretty=format:"%ct" -1 2>/dev/null).to_i - return old_pri + Math.cbrt(WEEK_SECONDS) if last_commit_time.zero? + return old_pri + Math.cbrt(MONTH_SECONDS) if last_commit_time.zero? t = Time.now.to_i interval = t - last_commit_time - return old_pri + Math.cbrt(WEEK_SECONDS) if interval <= 0 + return old_pri + Math.cbrt(MONTH_SECONDS) if interval <= 0 return old_pri + Math.cbrt(interval) end -- 2.23.0

2 2

[PATCH v2 compass-ci 2/2] lib/git_mirror.rb: add logs for kibana
by Li Yuanchao 07 Apr '21

07 Apr '21

Add logs when git clone fail, git fetch fail and clone a new repo. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 6340400..ad8847a 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -49,11 +49,18 @@ class GitMirror url = get_url(Array(url)[0]) 10.times do stderr = %x(git clone --mirror --depth 1 #{url} #{mirror_dir} 2>&1) - return true if File.directory?(mirror_dir) && File.exist?("#{mirror_dir}/config") + return 2 if File.directory?(mirror_dir) && File.exist?("#{mirror_dir}/config") url = "git://#{url.split('://')[1]}" if stderr_443?(stderr) end - return false + return -2 + end + + def fetch_443(mirror_dir, fetch_info) + return unless stderr_443?(fetch_info) + + url = %x(git -C #{mirror_dir} ls-remote --get-url origin).chomp + %x(git -C #{mirror_dir} remote set-url origin git://#{url.split('://')[1]}) end def git_fetch(mirror_dir) @@ -67,12 +74,10 @@ class GitMirror if fetch_info.include?(ERR_MESSAGE) && Dir.empty?(mirror_dir) FileUtils.rmdir(mirror_dir) end + fetch_443(mirror_dir, fetch_info) + return -1 if fetch_info.include?('fatal') - if stderr_443?(fetch_info) - url = %x(git -C #{mirror_dir} ls-remote --get-url origin).chomp - %x(git -C #{mirror_dir} remote set-url origin git://#{url.split('://')[1]}) - end - return fetch_info.include? '->' + return fetch_info.include?('->') ? 1 : 0 end def url_changed?(url, mirror_dir) @@ -196,8 +201,14 @@ class MirrorMain git_repo = feedback_info[:git_repo] return if check_submodule(git_repo) + # values of possible_new_refs: + # 2: git clone a new repo + # 1: git fetch and get new refs + # 0: git fetch and no new refs + # -1: git fetch fail + # -2: git clone fail update_fork_stat(git_repo, feedback_info[:possible_new_refs]) - return unless feedback_info[:possible_new_refs] + return if feedback_info[:possible_new_refs] < 1 handle_feedback_new_refs(git_repo, feedback_info) end @@ -335,7 +346,9 @@ class MirrorMain def update_fork_stat(git_repo, possible_new_refs) update_stat_fetch(git_repo) - update_stat_new_refs(git_repo) if possible_new_refs && last_commit_new?(git_repo) + git_fail_log(git_repo, possible_new_refs) if possible_new_refs.negative? + update_stat_new_refs(git_repo) if possible_new_refs.positive? && last_commit_new?(git_repo) + new_repo_log(git_repo) if possible_new_refs == 2 es_repo_update(git_repo) end end @@ -533,10 +546,24 @@ class MirrorMain end end + def new_repo_log(git_repo) + @log.info({ + msg: 'new repo', + repo: git_repo + }.to_json) + end + + def git_fail_log(git_repo, possible_new_refs) + msg = possible_new_refs == -1 ? 'git fetch fail' : 'git clone fail' + @log.info({ + msg: msg, + repo: git_repo + }.to_json) + end + def new_refs_log(git_repo, nr_new_branch) @log.info({ msg: 'new refs', - time: Time.now, repo: git_repo, nr_new_branch: nr_new_branch }.to_json) -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] lib/git_mirror.rb: bigger step to change priority
by Li Yuanchao 07 Apr '21

07 Apr '21

Make the step to change priority of repo bigger, so bad repos would be put behind faster. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 1f4e5b5..6340400 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -468,7 +468,7 @@ end # main thread class MirrorMain - WEEK_SECONDS = 604800 + STEP_SECONDS = 2592000 def handle_feedback_new_refs(git_repo, feedback_info) return reload_fork_info(git_repo) if upstream_repo?(git_repo) @@ -554,18 +554,18 @@ class MirrorMain mirror_dir = "/srv/git/#{@git_info[git_repo]['belong']}/#{git_repo}" mirror_dir = "#{mirror_dir}.git" unless @git_info[git_repo]['is_submodule'] - return old_pri + Math.cbrt(WEEK_SECONDS) unless File.directory?(mirror_dir) + return old_pri + Math.cbrt(STEP_SECONDS) unless File.directory?(mirror_dir) return cal_priority(mirror_dir, old_pri) end def cal_priority(mirror_dir, old_pri) last_commit_time = %x(git -C #{mirror_dir} log --pretty=format:"%ct" -1 2>/dev/null).to_i - return old_pri + Math.cbrt(WEEK_SECONDS) if last_commit_time.zero? + return old_pri + Math.cbrt(STEP_SECONDS) if last_commit_time.zero? t = Time.now.to_i interval = t - last_commit_time - return old_pri + Math.cbrt(WEEK_SECONDS) if interval <= 0 + return old_pri + Math.cbrt(STEP_SECONDS) if interval <= 0 return old_pri + Math.cbrt(interval) end -- 2.23.0

1 0

[PATCH compass-ci] sbin/es-*-mapping.sh: fix illegal_argument_exception
by Lu Weitao 07 Apr '21

07 Apr '21

[Why] { "error":{ "root_cause":[{ "type":"illegal_argument_exception", "reason":"The mapping definition cannot be nested under a type [_doc] unless include_type_name is set to true." }], "type":"illegal_argument_exception", "reason":"The mapping definition cannot be nested under a type [_doc] unless include_type_name is set to true." }, "status":400 } Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- sbin/es-accounts-mapping.sh | 34 ++++++++++----------- sbin/es-jobs-mapping.sh | 2 -- sbin/es-regression-mapping.sh | 16 +++++----- sbin/es-repo-mapping.sh | 56 +++++++++++++++++------------------ 4 files changed, 50 insertions(+), 58 deletions(-) diff --git a/sbin/es-accounts-mapping.sh b/sbin/es-accounts-mapping.sh index d04b0f3..c4ac2fc 100755 --- a/sbin/es-accounts-mapping.sh +++ b/sbin/es-accounts-mapping.sh @@ -12,24 +12,22 @@ else echo "begin create index." curl -sSH 'Content-Type: Application/json' -XPUT 'http://localhost:9200/accounts' -d '{ "mappings": { - "_doc": { - "dynamic": false, - "properties": { - "my_commit_url": { - "type": "keyword" - }, - "my_token": { - "type": "keyword" - }, - "my_email": { - "type": "keyword" - }, - "my_name": { - "type": "keyword" - }, - "my_login_name": { - "type": "keyword" - } + "dynamic": false, + "properties": { + "my_commit_url": { + "type": "keyword" + }, + "my_token": { + "type": "keyword" + }, + "my_email": { + "type": "keyword" + }, + "my_name": { + "type": "keyword" + }, + "my_login_name": { + "type": "keyword" } } } diff --git a/sbin/es-jobs-mapping.sh b/sbin/es-jobs-mapping.sh index edc6159..1c4aa43 100755 --- a/sbin/es-jobs-mapping.sh +++ b/sbin/es-jobs-mapping.sh @@ -37,7 +37,6 @@ else echo "jobs index not exists, begin create index." curl -sSH 'Content-Type: Application/json' -XPUT 'http://localhost:9200/jobs' -d '{ "mappings": { - "_doc": { "dynamic": false, "dynamic_templates": [ { @@ -166,7 +165,6 @@ else "format": "yyyy-MM-dd HH:mm:ss" } } - } } }' if [ $? -ne 0 ] diff --git a/sbin/es-regression-mapping.sh b/sbin/es-regression-mapping.sh index 8e04bdb..5ae0f71 100755 --- a/sbin/es-regression-mapping.sh +++ b/sbin/es-regression-mapping.sh @@ -10,15 +10,13 @@ else echo "begin create index." curl -sSH 'Content-Type: Application/json' -XPUT 'http://localhost:9200/regression' -d '{ "mappings": { - "_doc": { - "dynamic": false, - "properties": { - "error_id": { - "type": "keyword" - }, - "job_id": { - "type": "keyword" - } + "dynamic": false, + "properties": { + "error_id": { + "type": "keyword" + }, + "job_id": { + "type": "keyword" } } } diff --git a/sbin/es-repo-mapping.sh b/sbin/es-repo-mapping.sh index 17f0986..290efb3 100755 --- a/sbin/es-repo-mapping.sh +++ b/sbin/es-repo-mapping.sh @@ -11,35 +11,33 @@ echo 'Start to create "repo" index.' curl -sSH 'Content-Type: Application/json' -XPUT 'http://localhost:9200/repo' -d ' { "mappings": { - "_doc": { - "properties": { - "git_repo": { - "type": "keyword" - }, - "pkgbuild_repo": { - "type": "keyword" - }, - "url": { - "type": "keyword" - }, - "fetch_time": { - "type": "keyword" - }, - "new_refs_time": { - "type": "keyword" - }, - "offset_fetch": { - "type": "long" - }, - "offset_new_refs": { - "type": "long" - }, - "priority": { - "type": "long" - }, - "queued": { - "type": "boolean" - } + "properties": { + "git_repo": { + "type": "keyword" + }, + "pkgbuild_repo": { + "type": "keyword" + }, + "url": { + "type": "keyword" + }, + "fetch_time": { + "type": "keyword" + }, + "new_refs_time": { + "type": "keyword" + }, + "offset_fetch": { + "type": "long" + }, + "offset_new_refs": { + "type": "long" + }, + "priority": { + "type": "long" + }, + "queued": { + "type": "boolean" } } } -- 2.23.0

1 0

[PATCH compass-ci] lib/json_logger: unified time format
by Wu Zhende 07 Apr '21

07 Apr '21

The ES parses the time in this format to the date type. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/json_logger.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/json_logger.rb b/lib/json_logger.rb index 2c33c2f..7a05f79 100644 --- a/lib/json_logger.rb +++ b/lib/json_logger.rb @@ -21,7 +21,7 @@ class JSONLogger < Logger logger_hash = { 'level' => severity.to_s, 'level_num' => level_num, - 'time' => datetime + 'time' => Time.now.strftime("%Y-%m-%dT%H:%M:%S.%3N+0800") } logger_hash['progname'] = progname if progname -- 2.23.0

1 0

[PATCH compass-ci] lib/git_mirror.rb: add logs for kibana
by Li Yuanchao 07 Apr '21

07 Apr '21

Add logs when git clone fail, git fetch fail and clone a new repo. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 48 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 932a10e..60a4b76 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -49,11 +49,18 @@ class GitMirror url = get_url(Array(url)[0]) 10.times do stderr = %x(git clone --mirror --depth 1 #{url} #{mirror_dir} 2>&1) - return true if File.directory?(mirror_dir) && File.exist?("#{mirror_dir}/config") + return 2 if File.directory?(mirror_dir) && File.exist?("#{mirror_dir}/config") url = "git://#{url.split('://')[1]}" if stderr_443?(stderr) end - return false + return -2 + end + + def fetch_443(mirror_dir, fetch_info) + return unless stderr_443?(fetch_info) + + url = %x(git -C #{mirror_dir} ls-remote --get-url origin).chomp + %x(git -C #{mirror_dir} remote set-url origin git://#{url.split('://')[1]}) end def git_fetch(mirror_dir) @@ -67,12 +74,10 @@ class GitMirror if fetch_info.include?(ERR_MESSAGE) && Dir.empty?(mirror_dir) FileUtils.rmdir(mirror_dir) end + fetch_443(mirror_dir, fetch_info) + return -1 if fetch_info.include?('fatal') - if stderr_443?(fetch_info) - url = %x(git -C #{mirror_dir} ls-remote --get-url origin).chomp - %x(git -C #{mirror_dir} remote set-url origin git://#{url.split('://')[1]}) - end - return fetch_info.include? '->' + return fetch_info.include?('->') ? 1 : 0 end def url_changed?(url, mirror_dir) @@ -196,8 +201,14 @@ class MirrorMain git_repo = feedback_info[:git_repo] return if check_submodule(git_repo) + # values of possible_new_refs: + # 2: git clone a new repo + # 1: git fetch and get new refs + # 0: git fetch and no new refs + # -1: git fetch fail + # -2: git clone fail update_fork_stat(git_repo, feedback_info[:possible_new_refs]) - return unless feedback_info[:possible_new_refs] + return if feedback_info[:possible_new_refs] < 1 handle_feedback_new_refs(git_repo, feedback_info) end @@ -335,7 +346,9 @@ class MirrorMain def update_fork_stat(git_repo, possible_new_refs) update_stat_fetch(git_repo) - update_stat_new_refs(git_repo) if possible_new_refs && last_commit_new?(git_repo) + git_fail_log(git_repo, possible_new_refs) if possible_new_refs.negative? + update_stat_new_refs(git_repo) if possible_new_refs.positive? && last_commit_new?(git_repo) + new_repo_log(git_repo) if possible_new_refs == 2 es_repo_update(git_repo) end end @@ -533,6 +546,23 @@ class MirrorMain end end + def new_repo_log(git_repo) + @log.info({ + msg: 'new repo', + time: Time.now, + repo: git_repo + }.to_json) + end + + def git_fail_log(git_repo, possible_new_refs) + msg = possible_new_refs == -1 ? 'git fetch fail' : 'git clone fail' + @log.info({ + msg: msg, + time: Time.now, + repo: git_repo + }.to_json) + end + def new_refs_log(git_repo, nr_new_branch) @log.info({ msg: 'new refs', -- 2.23.0

1 0

[PATCH compass-ci] lib/lifecycle: add some logs
by Wu Zhende 07 Apr '21

07 Apr '21

Help locate problems. Collects statistics on the number and information of close jobs. Collects statistics on the number and information of reboot machines. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/lifecycle.cr | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 0e6cb82..e56a681 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -43,7 +43,10 @@ class Lifecycle def alive(version) "Lifecycle Alive! The time is #{Time.local}, version = #{version}" rescue e - @log.warn(e) + @log.warn({ + "resource" => "/alive", + "message" => e.to_s + }.to_json) end def init_from_es @@ -68,6 +71,11 @@ class Lifecycle deal_match_job(testbox, machine["job_id"].to_s) end + rescue e + @log.warn({ + "resource" => "init_from_es", + "message" => e.to_s + }.to_json) end def deal_match_job(testbox, job_id) @@ -129,6 +137,11 @@ class Lifecycle end @mq.ch.basic_ack(msg.delivery_tag) end + rescue e + @log.warn({ + "resource" => "mq_event_loop", + "message" => e.to_s + }.to_json) end def on_other_job(event) @@ -253,6 +266,11 @@ class Lifecycle next if dead_job_id && deadline <= Time.local sleep_until(deadline) + rescue e + @log.warn({ + "resource" => "timeout_job_loop", + "message" => e.to_s + }.to_json) end end @@ -264,6 +282,11 @@ class Lifecycle next if dead_machine_name && deadline <= Time.local sleep_until(deadline) + rescue e + @log.warn({ + "resource" => "timeout_machine_loop", + "message" => e.to_s + }.to_json) end end @@ -306,6 +329,12 @@ class Lifecycle def close_job(job_id, reason) @jobs.delete(job_id) spawn @scheduler_api.close_job(job_id, reason, "lifecycle") + @log.info({ + "job_id" => job_id, + "state" => "close", + "reason" => reason, + "type" => "job" + }.to_json) end def reboot_crash_machine(testbox, event) @@ -314,7 +343,7 @@ class Lifecycle return unless machine return unless event["time"].to_s.bigger_than?(machine["time"]?) - reboot_machine(testbox, machine) + reboot_machine(testbox, machine, "crash") end def reboot_timeout_machine(testbox) @@ -329,10 +358,10 @@ class Lifecycle deadline = Time.parse(deadline.to_s, "%Y-%m-%dT%H:%M:%S", Time.local.location) return if Time.local < deadline - reboot_machine(testbox, machine) + reboot_machine(testbox, machine, "timeout") end - def reboot_machine(testbox, machine) + def reboot_machine(testbox, machine, reason) mq_queue = get_machine_reboot_queue(testbox) machine.as_h.delete("history") machine.as_h["testbox"] = JSON::Any.new(testbox) @@ -341,6 +370,12 @@ class Lifecycle machine["state"] = "rebooting_queue" machine["time"] = Time.local.to_s("%Y-%m-%dT%H:%M:%S+0800") @es.update_tbox(testbox, machine.as_h) + @log.info({ + "type" => "testbox", + "reason" => reason, + "testbox" => testbox, + "state" => "reboot" + }.to_json) end def get_machine_reboot_queue(testbox) -- 2.23.0

1 0

[PATCH v2 compass-ci 4/4] container/rpm-repo: add a new docker service
by Li Ping 07 Apr '21

07 Apr '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/rpm-repo/Dockerfile | 10 +++++ container/rpm-repo/build | 6 +++ container/rpm-repo/start | 41 ++++++++++++++++++++ lib/rpm-repo.rb | 73 +++++++++++++++++++++++++++++++++++ 4 files changed, 130 insertions(+) create mode 100644 container/rpm-repo/Dockerfile create mode 100755 container/rpm-repo/build create mode 100755 container/rpm-repo/start create mode 100755 lib/rpm-repo.rb diff --git a/container/rpm-repo/Dockerfile b/container/rpm-repo/Dockerfile new file mode 100644 index 0000000..588828a --- /dev/null +++ b/container/rpm-repo/Dockerfile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM fedora:latest + +RUN yum install -y createrepo ruby + +RUN umask 002 && \ + gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ + gem install bunny diff --git a/container/rpm-repo/build b/container/rpm-repo/build new file mode 100755 index 0000000..458e8f6 --- /dev/null +++ b/container/rpm-repo/build @@ -0,0 +1,6 @@ +#!/bin/sh + +. ../defconfig.sh +docker_skip_rebuild "fedora:update-repodata" + +docker build -t fedora:update-repodata . diff --git a/container/rpm-repo/start b/container/rpm-repo/start new file mode 100755 index 0000000..08fa434 --- /dev/null +++ b/container/rpm-repo/start @@ -0,0 +1,41 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig.rb' + +DOCKER_CCI = '/c/compass-ci' + +names = Set.new %w[ + MQ_HOST + MQ_PORT + LKP_SERVER +] + +defaults = relevant_defaults(names) +defaults['MQ_HOST'] ||= defaults['LKP_SERVER'] ||= '172.17.0.1' +defaults['LKP_SERVER'] ||= '172.17.0.1' +defaults['MQ_PORT'] ||= 5672 +env = docker_env(defaults) + +docker_rm "update-repodata" + +cmd = %W[ + docker run + --restart=always + --name update-repodata + -u 1090:1090 +] + env + %W[ + -d + -e CCI_SRC=#{DOCKER_CCI} + -v #{ENV['CCI_SRC']}:#{DOCKER_CCI} + -v /etc/localtime:/etc/localtime:ro + -v /srv/rpm:/srv/rpm + -w /c/compass-ci/lib + fedora:update-repodata +] + +cmd += ['sh', '-c', 'umask 002 && ./rpm-repo.rb'] +system(*cmd) diff --git a/lib/rpm-repo.rb b/lib/rpm-repo.rb new file mode 100755 index 0000000..658a527 --- /dev/null +++ b/lib/rpm-repo.rb @@ -0,0 +1,73 @@ +#!/usr/bin/ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'fileutils' +require 'json' +require './mq_client' + +MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || '172.17.0.1' +MQ_PORT = ENV['MQ_PORT'] || 5672 + +# --------------------------------------------------------------------------------------------------------------------------------------- +# end_user can use cmd: +# curl -H 'Content-Type: Application/json' -XPOST 'localhost:10003/upload' \ +# -d '{"upload_rpms": ["/srv/rpm/upload/openeuler-20.03-LTS/compat-centos8/centos-appstream/aarch64/Packages/*.rpm"]}' +# +# the post data would be stored in mq queue "update_repo" through update-repo service +# --------------------------------------------------------------------------------------------------------------------------------------- +# +# HandleRepo used to handle the mq queue "update_repo" +# +# Example items in @mq.queue "update_repo": { "upload_rpms" => ["/srv/rpm/upload/**/Packages/*.rpm", "/srv/rpm/upload/**/source/*.rpm"]} +# handle_new_rpm +# move /srv/rpm/upload/**/*.rpm to /srv/rpm/testing/**/*.rpm +# update /srv/rpm/testing/**/repodate +# update_pub_dir +# copy /srv/rpm/testing/**/*.rpm to /srv/rpm/pub/**/*.rpm +# change /srv/rpm/pub/**/repodate +# @mq.ack(info) +class HandleRepo + def initialize + @mq = MQClient.new(MQ_HOST, MQ_PORT) + @update = [] + end + + def handle_new_rpm + queue = @mq.queue("update_repo") + queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| + rpm_info = JSON.parse(msg) + rpm_info["upload_rpms"].each do |rpm| + rpm_path = File.dirname(rpm).sub("upload", "testing") + FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) + + dest = File.join(rpm_path.to_s, File.basename(rpm)) + @update << dest + FileUtils.mv(rpm, dest) + system("createrepo --update $(dirname #{rpm_path})") + end + update_pub_dir + @mq.ack(info) + end + end + + def update_pub_dir + @update.each do |rpm| + pub_path = File.dirname(rpm).sub("testing", "pub") + FileUtils.mkdir_p(pub_path) unless File.directory?(pub_path) + + dest = File.join(pub_path, File.basename(rpm)) + FileUtils.cp(rpm, dest) + + repodata_dest = File.join(File.dirname(pub_path), "repodata") + repodata_src = File.dirname(rpm).sub("Packages", "repodata") + + FileUtils.rm_r(repodata_dest) if Dir.exist?(repodata_dest) + FileUtils.cp_r(repodata_src, File.dirname(repodata_dest)) + end + end +end + +hr = HandleRepo.new +hr.handle_new_rpm -- 2.23.0

1 0

[PATCH v2 compass-ci 3/4] src/lib/updaterepo.cr: send messages to update-repo queue
by Li Ping 07 Apr '21

07 Apr '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/updaterepo.cr | 65 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 src/lib/updaterepo.cr diff --git a/src/lib/updaterepo.cr b/src/lib/updaterepo.cr new file mode 100644 index 0000000..78d6a52 --- /dev/null +++ b/src/lib/updaterepo.cr @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +require "json" +require "kemal" +require "./json_logger" +require "./mq" + +# ------------------------------------------------------------------------------------------------------------------------------------- +# end_user: +# - restful API [post "/upload"] to post data {"upload_rpms": ["/srv/rpm/upload/**/Packages/*.rpm", "/srv/rpm/upload/**/source/*.rpm"]} +# +# ------------------------------------------------------------------------------------------------------------------------------------- +# Repo: +# - get json formated request data +# - use @mq.queue("update_repo") publish data +# - json formated data stored in the @mq.queue("update_repo") +# +class Repo + def initialize(env : HTTP::Server::Context) + @mq = MQClient.instance + @env = env + @log = JSONLogger.new + end + + def upload_repo + begin + body = @env.request.body.not_nil!.gets_to_end + data = JSON.parse(body.to_s).as_h? + rescue e + @log.warn(e) + end + @log.info(data.to_json) + + begin + check_params_complete(data) + mq_publish(data) + rescue e + @log.warn(e) + response = { "errcode" => "101", "errmsg" => "upload rpm failed" } + @log.warn(response.to_json) + end + end + + def mq_publish(data) + mq_msg = data + spawn mq_publish_check("update_repo", mq_msg.to_json) + end + + def mq_publish_check(queue, msg) + 3.times do + @mq.publish_confirm(queue, msg) + break + rescue e + res = @mq.reconnect + sleep 5 + end + end + + def check_params_complete(params) + params = params.not_nil! + tmp_hash = {"errcode" => "101", "errmsg" => "no upload_rpms params"}.to_json + raise tmp_hash unless params["upload_rpms"]? + end +end -- 2.23.0

1 0