- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci 1/2] extract-stats/stats_worker.cr: add errid to the job
by Wu Zhende 08 Apr '21

08 Apr '21

Save errid in es as type "text". In this way, we can search for errid by keyword. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/extract-stats/stats_worker.cr | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index 56dad9e..e371bf3 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. require "../lib/etcd_client" @@ -8,6 +9,9 @@ require "./regression_client" require "./constants.cr" class StatsWorker + @@metric_failure = File.read("#{ENV["LKP_SRC"]}/etc/failure").strip.split("\n").as(Array(String)) + @@__is_failure_cache = Hash(String, Bool).new + def initialize @es = Elasticsearch::Client.new @etcd = EtcdClient.new @@ -45,6 +49,21 @@ class StatsWorker system "#{ENV["CCI_SRC"]}/sbin/mail-job #{job_id}" end + def is_failure(stats_field) + if @@__is_failure_cache.has_key?(stats_field) + @@__is_failure_cache[stats_field] + else + @@__is_failure_cache[stats_field] = __is_failure(stats_field) + end + end + + def __is_failure(stats_field) + return false if stats_field.index(".time.") + return false if stats_field.index(".timestamp.") + return true if @@metric_failure.any? { |pattern| stats_field =~ %r{^#{pattern}} } + false + end + def store_stats_es(result_root : String, job_id : String, queue_path : String) stats_path = "#{result_root}/stats.json" return unless File.exists?(stats_path) @@ -53,8 +72,13 @@ class StatsWorker JSON.parse(file) end + errid = Array(String).new + stats.as_h.keys.each do |k| + errid << k.to_s if is_failure(k.to_s) + end + update_content = Hash(String, Array(String) | Hash(String, JSON::Any)).new - update_content.merge!({"stats" => stats.as_h}) + update_content.merge!({"stats" => stats.as_h, "errid" => errid}) error_ids = get_error_ids_by_json(result_root) update_content.merge!({"error_ids" => error_ids}) unless error_ids.empty? -- 2.23.0

2 1

[PATCH compass-ci 2/2] sbin/es-jobs-mapping.sh: add errid
by Wu Zhende 08 Apr '21

08 Apr '21

The errid of the text type is added. Can use errid's keyword as a query. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- sbin/es-jobs-mapping.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbin/es-jobs-mapping.sh b/sbin/es-jobs-mapping.sh index 1c4aa43..28067e2 100755 --- a/sbin/es-jobs-mapping.sh +++ b/sbin/es-jobs-mapping.sh @@ -85,6 +85,9 @@ else "suite": { "type": "keyword" }, + "errid": { + "type": "text" + }, "category": { "type": "keyword" }, -- 2.23.0

1 0

[PATCH compass-ci] sbin/es-jobs-mapping.sh: add submit_date field
by Lu Weitao 08 Apr '21

08 Apr '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- sbin/es-jobs-mapping.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sbin/es-jobs-mapping.sh b/sbin/es-jobs-mapping.sh index 1c4aa43..fbfd4be 100755 --- a/sbin/es-jobs-mapping.sh +++ b/sbin/es-jobs-mapping.sh @@ -150,6 +150,10 @@ else "type": "date", "format": "yyyy-MM-dd HH:mm:ss" }, + "submit_date": { + "type": "date", + "format": "yyyy-MM-dd" + }, "user": { "type": "keyword" }, -- 2.23.0

1 0

[PATCH compass-ci] src/lib/job.cr: when os_mount=local, add suffix '-iso' to os_version automically
by Xu Xijian 08 Apr '21

08 Apr '21

The version change example like follows: 1. 20.03 => 20.03-iso 2. 20.03-iso => 20.03-iso, no need to add iso suffix, just keep Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/job.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/job.cr b/src/lib/job.cr index f62073d..d83bbe5 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -240,6 +240,7 @@ class Job end private def set_os_dir + self["os_version"] = "#{os_version}".chmop("-iso") + "-iso" if "#{self.os_mount}" == "local" self["os_dir"] = "#{os}/#{os_arch}/#{os_version}" end -- 2.23.0

1 0

[PATCH v2 compass-ci] update container expose port
by Cui Yili 08 Apr '21

08 Apr '21

Signed-off-by: Cui Yili <2268260388(a)qq.com> --- container/minio/start | 2 +- container/openresty-proxy-cache/start | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/container/minio/start b/container/minio/start index 1798cfa..2cbc9c8 100755 --- a/container/minio/start +++ b/container/minio/start @@ -7,7 +7,7 @@ docker_rm minio cmd=( docker run -d - -p 11321:9000 + -p 20014:9000 --name minio -v /etc/localtime:/etc/localtime:ro -v /srv/minio/:/srv/minio/ diff --git a/container/openresty-proxy-cache/start b/container/openresty-proxy-cache/start index a1f095d..59e2a28 100755 --- a/container/openresty-proxy-cache/start +++ b/container/openresty-proxy-cache/start @@ -11,8 +11,8 @@ cmd=( docker run -d --name openresty-proxy-cache --restart=always - -p 11333:80 - -p 11334:443 + -p 20015:80 + -p 20016:443 -v $PWD/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf -v /srv/initrd/tar:/srv/initrd/tar:ro -v /etc/localtime:/etc/localtime:ro -- 2.23.0

1 0

[PATCH compass-ci] lib/git_mirror.rb: take git fail count into calculation of priority
by Li Yuanchao 07 Apr '21

07 Apr '21

If a repo git clone or git fetch failed many times, it would be put to the end of the queue. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index ad8847a..bcdb077 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -323,6 +323,11 @@ class MirrorMain @es_client.index(index: 'repo', type: '_doc', id: git_repo, body: body) end + def update_fail_count(git_repo, possible_new_refs) + @fork_stat[git_repo][:clone_fail_cnt] += 1 if possible_new_refs == -2 + @fork_stat[git_repo][:fetch_fail_cnt] += 1 if possible_new_refs == -1 + end + def update_stat_fetch(git_repo) @fork_stat[git_repo][:queued] = false offset_fetch = @fork_stat[git_repo][:offset_fetch] @@ -338,7 +343,6 @@ class MirrorMain end def update_stat_new_refs(git_repo) - @fork_stat[git_repo][:priority] += 1 offset_new_refs = @fork_stat[git_repo][:offset_new_refs] offset_new_refs = 0 if offset_new_refs >= 10 update_new_refs_info(git_repo, offset_new_refs) @@ -346,6 +350,7 @@ class MirrorMain def update_fork_stat(git_repo, possible_new_refs) update_stat_fetch(git_repo) + update_fail_count(git_repo, possible_new_refs) git_fail_log(git_repo, possible_new_refs) if possible_new_refs.negative? update_stat_new_refs(git_repo) if possible_new_refs.positive? && last_commit_new?(git_repo) new_repo_log(git_repo) if possible_new_refs == 2 @@ -416,7 +421,8 @@ class MirrorMain def get_fork_stat(git_repo) fork_stat = { queued: false, - priority: 0, + fetch_fail_cnt: 0, + clone_fail_cnt: 0, fetch_time: [], offset_fetch: 0, new_refs_time: [], @@ -581,18 +587,20 @@ class MirrorMain mirror_dir = "/srv/git/#{@git_info[git_repo]['belong']}/#{git_repo}" mirror_dir = "#{mirror_dir}.git" unless @git_info[git_repo]['is_submodule'] - return old_pri + Math.cbrt(STEP_SECONDS) unless File.directory?(mirror_dir) + step = (@fork_stat[git_repo][:clone_fail_cnt] + 1) * Math.cbrt(STEP_SECONDS) + return old_pri + step unless File.directory?(mirror_dir) - return cal_priority(mirror_dir, old_pri) + return cal_priority(mirror_dir, old_pri, git_repo) end - def cal_priority(mirror_dir, old_pri) + def cal_priority(mirror_dir, old_pri, git_repo) last_commit_time = %x(git -C #{mirror_dir} log --pretty=format:"%ct" -1 2>/dev/null).to_i - return old_pri + Math.cbrt(STEP_SECONDS) if last_commit_time.zero? + step = (@fork_stat[git_repo][:fetch_fail_cnt] + 1) * Math.cbrt(STEP_SECONDS) + return old_pri + step if last_commit_time.zero? t = Time.now.to_i interval = t - last_commit_time - return old_pri + Math.cbrt(STEP_SECONDS) if interval <= 0 + return old_pri + step if interval <= 0 return old_pri + Math.cbrt(interval) end -- 2.23.0

1 0

[PATCH v2 compass-ci] container/mail-robot: fix bugs
by Luan Shengde 07 Apr '21

07 Apr '21

fix no file errors: it may throw error if the specified file not exists: Errno::ENOENT: No such file or directory @ rb_sysopen Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/apply-account.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/mail-robot/lib/apply-account.rb b/container/mail-robot/lib/apply-account.rb index 798546a..d8cb4da 100755 --- a/container/mail-robot/lib/apply-account.rb +++ b/container/mail-robot/lib/apply-account.rb @@ -56,7 +56,7 @@ class ApplyAccount @es_port = ES_PORT # email info file for account issuers. - @account_issuer = YAML.load_file ENV['ACCOUNT_ISSUER'] + @account_issuer = File.exists?(ENV['ACCOUNT_ISSUER']) ? YAML.load_file(ENV['ACCOUNT_ISSUER']) : {} @my_info = {} end -- 2.23.0

1 0

[PATCH compass-ci] container/mail-robot: fix bugs
by Luan Shengde 07 Apr '21

07 Apr '21

fix no file errors: it may throw error if the specified file not exists: Errno::ENOENT: No such file or directory @ rb_sysopen Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/apply-account.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/container/mail-robot/lib/apply-account.rb b/container/mail-robot/lib/apply-account.rb index 798546a..6b9bdfe 100755 --- a/container/mail-robot/lib/apply-account.rb +++ b/container/mail-robot/lib/apply-account.rb @@ -56,7 +56,11 @@ class ApplyAccount @es_port = ES_PORT # email info file for account issuers. - @account_issuer = YAML.load_file ENV['ACCOUNT_ISSUER'] + @account_issuer = if File.exists? ENV['ACCOUNT_ISSUER'] + YAML.load_file ENV['ACCOUNT_ISSUER'] + else + {} + end @my_info = {} end -- 2.23.0

2 4

[PATCH compass-ci] lib/git_mirror.rb: bigger step to change priority
by Li Yuanchao 07 Apr '21

07 Apr '21

Make the step to change priority of repo bigger, so bad repos would be put behind faster. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index b543f02..60a4b76 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -481,7 +481,7 @@ end # main thread class MirrorMain - WEEK_SECONDS = 604800 + MONTH_SECONDS = 2592000 def handle_feedback_new_refs(git_repo, feedback_info) return reload_fork_info(git_repo) if upstream_repo?(git_repo) @@ -584,18 +584,18 @@ class MirrorMain mirror_dir = "/srv/git/#{@git_info[git_repo]['belong']}/#{git_repo}" mirror_dir = "#{mirror_dir}.git" unless @git_info[git_repo]['is_submodule'] - return old_pri + Math.cbrt(WEEK_SECONDS) unless File.directory?(mirror_dir) + return old_pri + Math.cbrt(MONTH_SECONDS) unless File.directory?(mirror_dir) return cal_priority(mirror_dir, old_pri) end def cal_priority(mirror_dir, old_pri) last_commit_time = %x(git -C #{mirror_dir} log --pretty=format:"%ct" -1 2>/dev/null).to_i - return old_pri + Math.cbrt(WEEK_SECONDS) if last_commit_time.zero? + return old_pri + Math.cbrt(MONTH_SECONDS) if last_commit_time.zero? t = Time.now.to_i interval = t - last_commit_time - return old_pri + Math.cbrt(WEEK_SECONDS) if interval <= 0 + return old_pri + Math.cbrt(MONTH_SECONDS) if interval <= 0 return old_pri + Math.cbrt(interval) end -- 2.23.0

2 2

[PATCH v2 compass-ci 2/2] lib/git_mirror.rb: add logs for kibana
by Li Yuanchao 07 Apr '21

07 Apr '21

Add logs when git clone fail, git fetch fail and clone a new repo. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 6340400..ad8847a 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -49,11 +49,18 @@ class GitMirror url = get_url(Array(url)[0]) 10.times do stderr = %x(git clone --mirror --depth 1 #{url} #{mirror_dir} 2>&1) - return true if File.directory?(mirror_dir) && File.exist?("#{mirror_dir}/config") + return 2 if File.directory?(mirror_dir) && File.exist?("#{mirror_dir}/config") url = "git://#{url.split('://')[1]}" if stderr_443?(stderr) end - return false + return -2 + end + + def fetch_443(mirror_dir, fetch_info) + return unless stderr_443?(fetch_info) + + url = %x(git -C #{mirror_dir} ls-remote --get-url origin).chomp + %x(git -C #{mirror_dir} remote set-url origin git://#{url.split('://')[1]}) end def git_fetch(mirror_dir) @@ -67,12 +74,10 @@ class GitMirror if fetch_info.include?(ERR_MESSAGE) && Dir.empty?(mirror_dir) FileUtils.rmdir(mirror_dir) end + fetch_443(mirror_dir, fetch_info) + return -1 if fetch_info.include?('fatal') - if stderr_443?(fetch_info) - url = %x(git -C #{mirror_dir} ls-remote --get-url origin).chomp - %x(git -C #{mirror_dir} remote set-url origin git://#{url.split('://')[1]}) - end - return fetch_info.include? '->' + return fetch_info.include?('->') ? 1 : 0 end def url_changed?(url, mirror_dir) @@ -196,8 +201,14 @@ class MirrorMain git_repo = feedback_info[:git_repo] return if check_submodule(git_repo) + # values of possible_new_refs: + # 2: git clone a new repo + # 1: git fetch and get new refs + # 0: git fetch and no new refs + # -1: git fetch fail + # -2: git clone fail update_fork_stat(git_repo, feedback_info[:possible_new_refs]) - return unless feedback_info[:possible_new_refs] + return if feedback_info[:possible_new_refs] < 1 handle_feedback_new_refs(git_repo, feedback_info) end @@ -335,7 +346,9 @@ class MirrorMain def update_fork_stat(git_repo, possible_new_refs) update_stat_fetch(git_repo) - update_stat_new_refs(git_repo) if possible_new_refs && last_commit_new?(git_repo) + git_fail_log(git_repo, possible_new_refs) if possible_new_refs.negative? + update_stat_new_refs(git_repo) if possible_new_refs.positive? && last_commit_new?(git_repo) + new_repo_log(git_repo) if possible_new_refs == 2 es_repo_update(git_repo) end end @@ -533,10 +546,24 @@ class MirrorMain end end + def new_repo_log(git_repo) + @log.info({ + msg: 'new repo', + repo: git_repo + }.to_json) + end + + def git_fail_log(git_repo, possible_new_refs) + msg = possible_new_refs == -1 ? 'git fetch fail' : 'git clone fail' + @log.info({ + msg: msg, + repo: git_repo + }.to_json) + end + def new_refs_log(git_repo, nr_new_branch) @log.info({ msg: 'new refs', - time: Time.now, repo: git_repo, nr_new_branch: nr_new_branch }.to_json) -- 2.23.0

1 0