- Compass-ci - mailweb.openeuler.org

[PATCH v2 lkp-tests 2/2] sbin/submit: optimize submit -r
by Li Ping 05 Mar '21

05 Mar '21

[why] submit -m -r iperf.yaml submit /home/liping/lkp-tests/jobs/iperf.yaml, got job id=z9.1183160 submit /home/liping/lkp-tests/jobs/iperf.yaml, got job id=z9.1183161 query=>{"job_id":["z9.1183160","z9.1183161"]} connect to ws://172.168.131.2:11310/filter {"level_num":1,"level":"INFO","time":"2021-03-05T11:31:36.649+0800","job_id":"z9.1183161","message":"","job_state":"submit","result_root":"/srv/result/iperf/2021-03-05/vm-2p8g/openeuler-20.03-aarch64/udp-300/z9.1183161","status_code":200,"method":"POST","resource":"/submit_job","elapsed":"24.68ms"} {"level_num":1,"level":"INFO","time":"2021-03-05T11:31:39.699+0800","job_id":"z9.1183160","result_root":"/srv/result/iperf/2021-03-05/vm-2p8g/openeuler-20.03-aarch64/tcp-300/z9.1183160","job_state":"set result root","status_code":200,"method":"GET","resource":"/boot.ipxe/mac/0a-9f-90-36-ae-6b","elapsed":"8.75ms"} {"level_num":1,"level":"INFO","time":"2021-03-05T11:31:39.735+0800","job_id":"z9.1183160","job_state":"boot","status_code":200,"method":"GET","resource":"/boot.ipxe/mac/0a-9f-90-36-ae-6b","elapsed":"44.08ms"} ==> only get one result [how] get result_root directly from the msg not from fluentd data Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/monitor.rb | 36 ++++++++++-------------------------- sbin/monitor | 8 ++++++++ sbin/submit | 4 +++- 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/lib/monitor.rb b/lib/monitor.rb index 78582be6a..24191fa8d 100755 --- a/lib/monitor.rb +++ b/lib/monitor.rb @@ -11,7 +11,7 @@ require 'eventmachine' require 'json' class Monitor - attr_accessor :monitor_url, :query, :overrides, :action, :job, :result, :stop_query + attr_accessor :monitor_url, :query, :overrides, :action, :job, :result, :result_roots, :stop_query def initialize(monitor_url = '', query = {}, action = {}) @monitor_url = monitor_url @@ -25,9 +25,9 @@ class Monitor @exit_status_code = 0 @defaults = {} @result = [] + @result_roots = [] @stop_query = {} @reason = nil - @result_root = nil end def load_default @@ -95,29 +95,13 @@ class Monitor exec cmd end - def set_result_root(data) - return unless @action['mirror_result'] - return unless data['log'] - - data = JSON.parse(data['log']) - return unless data['result_root'] - - @result_root = data['result_root'] - end - - def lftp_mirror - @result_root.delete_prefix!('/srv') - srv_http_host = job['SRV_HTTP_HOST'] || 'api.compass-ci.openeuler.org' - srv_http_port = job['SRV_HTTP_PORT'] || '11300' - url = "http://#{srv_http_host}:#{srv_http_port}#{@result_root}" - system "lftp -c mirror #{url} >/dev/null 2>&1" - end - - def mirror_result(data) - if @result_root - lftp_mirror - else - set_result_root(data) + def mirror_result + @result_roots.each do |res| + res.to_s.delete_prefix!('/srv') + srv_http_host = job['SRV_HTTP_HOST'] || 'api.compass-ci.openeuler.org' + srv_http_port = job['SRV_HTTP_PORT'] || '11300' + url = "http://#{srv_http_host}:#{srv_http_port}#{res}" + system "lftp -c mirror #{url} >/dev/null 2>&1" end end @@ -163,7 +147,7 @@ class Monitor output(data) connect(data, ws) - mirror_result(data) + mirror_result stop(data, ws) if @action['stop'] end diff --git a/sbin/monitor b/sbin/monitor index def47bf6a..a2e677fc0 100755 --- a/sbin/monitor +++ b/sbin/monitor @@ -12,6 +12,7 @@ require 'yaml' opt_set_key_value = {} opt_action = {'output' => true} opt_job = {} +opt_results = [] opt_monitor_url = nil opt_timeout = 0 opt_stop_query = {} @@ -35,6 +36,12 @@ options = OptionParser.new do |opts| opt_action[action] = true end end + + opts.on('-r result_roots', '--results results', 'set monitor result_roots') do |result_roots| + result_roots.split(',').each do |res| + opt_results << res + end + end opts.separator ' action: output, output log' opts.separator ' action: connect, connect to the host when get ip' opts.separator ' action: stop, exit monitor client when get log' @@ -79,6 +86,7 @@ monitor = Monitor.new monitor.monitor_url = opt_monitor_url if opt_monitor_url monitor.overrides = opt_set_key_value monitor.action = opt_action +monitor.result_roots = opt_results monitor.job = opt_job monitor.stop_query = opt_stop_query diff --git a/sbin/submit b/sbin/submit index a6f0cd387..44fd0a7db 100755 --- a/sbin/submit +++ b/sbin/submit @@ -19,6 +19,7 @@ opt_monitor = false opt_monitor_query = {} opt_my_queue = false actions = ['output', 'stop'] +result_roots = [] options = OptionParser.new do |opts| opts.banner = 'Usage: submit [options] job1.yaml job2.yaml ...' @@ -197,6 +198,7 @@ ARGV.each do |jobfile| messages = scheduler_client.submit_job(job_json) JSON.parse(messages).each do |msg| if msg['message'].empty? + result_roots << msg['result_root'] job_ids << msg['job_id'].to_s puts("submit #{jobfile}, got job id=#{msg['job_id']}") else @@ -210,6 +212,6 @@ end if opt_monitor job_hash_list[0].delete('define_files') opt_monitor_query.merge!({'job_id' => job_ids}) - cmd = "#{LKP_SRC}/sbin/monitor -f '#{opt_monitor_query.to_json}' -j '#{job_hash_list[0].to_json}' -a #{actions.join(',')} -s 'job_state: extract_finished'" + cmd = "#{LKP_SRC}/sbin/monitor -f '#{opt_monitor_query.to_json}' -j '#{job_hash_list[0].to_json}' -a #{actions.join(',')} -r #{result_roots.join(',')} -s 'job_state: extract_finished'" exec cmd end -- 2.23.0

1 0

[PATCH lkp-tests] sbin/submit: fix cmdline parameter parse float
by Wei Jihui 05 Mar '21

05 Mar '21

[error] input: submit job.yaml a=6.10 output at job.yaml: a: 6.1 [why] Yaml.load will load 6.10 as float to 6.1 Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- sbin/submit | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sbin/submit b/sbin/submit index a6f0cd38..e5e41828 100755 --- a/sbin/submit +++ b/sbin/submit @@ -28,7 +28,8 @@ options = OptionParser.new do |opts| opts.separator 'options:' opts.on("-s 'KEY: VALUE'", "--set 'KEY: VALUE'", 'add YAML hash to job') do |key_value| - opt_set_key_value.merge! YAML.load key_value + k, v = key_value.sub(' ', '').split(':') + opt_set_key_value[k] = v end opts.on('-o DIR', '--output DIR', 'save job yaml to DIR/') do |dir| @@ -58,9 +59,9 @@ options = OptionParser.new do |opts| opts.on('-m', '--monitor', "monitor job status: use -m 'KEY: VALUE' to add rule") do opt_monitor = true - filter = YAML.load ARGV[0] if ARGV[0] - if filter.class == Hash - opt_monitor_query.merge! filter + k, v = ARGV[0].sub(' ', '').split(':') if ARGV[0] + if (k && !k.empty?) && (v && !v.empty?) + opt_monitor_query[k] = v ARGV.shift end end @@ -77,10 +78,11 @@ options.parse!(ARGV) seen_job_yaml = false ARGV.delete_if do |arg| if arg.index '=' + k, v = arg.sub(' ', '').sub(/=/, ':').split(':') if seen_job_yaml - opt_set_key_value.merge! YAML.load arg.sub(/=/, ': ') + opt_set_key_value[k] = v else - opt_cmdline_defaults.merge! YAML.load arg.sub(/=/, ': ') + opt_cmdline_defaults[k] = v end true else -- 2.23.0

1 0

[PATCH lkp-tests] sbin/submit: optimize submit -r
by Li Ping 05 Mar '21

05 Mar '21

[why] submit -m -r iperf.yaml submit /home/liping/lkp-tests/jobs/iperf.yaml, got job id=z9.1183160 submit /home/liping/lkp-tests/jobs/iperf.yaml, got job id=z9.1183161 query=>{"job_id":["z9.1183160","z9.1183161"]} connect to ws://172.168.131.2:11310/filter {"level_num":1,"level":"INFO","time":"2021-03-05T11:31:36.649+0800","job_id":"z9.1183161","message":"","job_state":"submit","result_root":"/srv/result/iperf/2021-03-05/vm-2p8g/openeuler-20.03-aarch64/udp-300/z9.1183161","status_code":200,"method":"POST","resource":"/submit_job","elapsed":"24.68ms"} {"level_num":1,"level":"INFO","time":"2021-03-05T11:31:39.699+0800","job_id":"z9.1183160","result_root":"/srv/result/iperf/2021-03-05/vm-2p8g/openeuler-20.03-aarch64/tcp-300/z9.1183160","job_state":"set result root","status_code":200,"method":"GET","resource":"/boot.ipxe/mac/0a-9f-90-36-ae-6b","elapsed":"8.75ms"} {"level_num":1,"level":"INFO","time":"2021-03-05T11:31:39.735+0800","job_id":"z9.1183160","job_state":"boot","status_code":200,"method":"GET","resource":"/boot.ipxe/mac/0a-9f-90-36-ae-6b","elapsed":"44.08ms"} ==> only get one result [how] get result_root directly from the msg not from fluentd data Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/monitor.rb | 36 ++++++++++-------------------------- sbin/monitor | 8 ++++++++ sbin/submit | 4 +++- 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/lib/monitor.rb b/lib/monitor.rb index 78582be6a..996bec749 100755 --- a/lib/monitor.rb +++ b/lib/monitor.rb @@ -11,7 +11,7 @@ require 'eventmachine' require 'json' class Monitor - attr_accessor :monitor_url, :query, :overrides, :action, :job, :result, :stop_query + attr_accessor :monitor_url, :query, :overrides, :action, :job, :result, :result_roots, :stop_query def initialize(monitor_url = '', query = {}, action = {}) @monitor_url = monitor_url @@ -25,9 +25,9 @@ class Monitor @exit_status_code = 0 @defaults = {} @result = [] + @result_roots = result_roots @stop_query = {} @reason = nil - @result_root = nil end def load_default @@ -95,29 +95,13 @@ class Monitor exec cmd end - def set_result_root(data) - return unless @action['mirror_result'] - return unless data['log'] - - data = JSON.parse(data['log']) - return unless data['result_root'] - - @result_root = data['result_root'] - end - - def lftp_mirror - @result_root.delete_prefix!('/srv') - srv_http_host = job['SRV_HTTP_HOST'] || 'api.compass-ci.openeuler.org' - srv_http_port = job['SRV_HTTP_PORT'] || '11300' - url = "http://#{srv_http_host}:#{srv_http_port}#{@result_root}" - system "lftp -c mirror #{url} >/dev/null 2>&1" - end - - def mirror_result(data) - if @result_root - lftp_mirror - else - set_result_root(data) + def mirror_result + @result_roots.each do |res| + res.to_s.delete_prefix!('/srv') + srv_http_host = job['SRV_HTTP_HOST'] || 'api.compass-ci.openeuler.org' + srv_http_port = job['SRV_HTTP_PORT'] || '11300' + url = "http://#{srv_http_host}:#{srv_http_port}#{res}" + system "lftp -c mirror #{url} >/dev/null 2>&1" end end @@ -163,7 +147,7 @@ class Monitor output(data) connect(data, ws) - mirror_result(data) + mirror_result stop(data, ws) if @action['stop'] end diff --git a/sbin/monitor b/sbin/monitor index def47bf6a..a2e677fc0 100755 --- a/sbin/monitor +++ b/sbin/monitor @@ -12,6 +12,7 @@ require 'yaml' opt_set_key_value = {} opt_action = {'output' => true} opt_job = {} +opt_results = [] opt_monitor_url = nil opt_timeout = 0 opt_stop_query = {} @@ -35,6 +36,12 @@ options = OptionParser.new do |opts| opt_action[action] = true end end + + opts.on('-r result_roots', '--results results', 'set monitor result_roots') do |result_roots| + result_roots.split(',').each do |res| + opt_results << res + end + end opts.separator ' action: output, output log' opts.separator ' action: connect, connect to the host when get ip' opts.separator ' action: stop, exit monitor client when get log' @@ -79,6 +86,7 @@ monitor = Monitor.new monitor.monitor_url = opt_monitor_url if opt_monitor_url monitor.overrides = opt_set_key_value monitor.action = opt_action +monitor.result_roots = opt_results monitor.job = opt_job monitor.stop_query = opt_stop_query diff --git a/sbin/submit b/sbin/submit index a6f0cd387..44fd0a7db 100755 --- a/sbin/submit +++ b/sbin/submit @@ -19,6 +19,7 @@ opt_monitor = false opt_monitor_query = {} opt_my_queue = false actions = ['output', 'stop'] +result_roots = [] options = OptionParser.new do |opts| opts.banner = 'Usage: submit [options] job1.yaml job2.yaml ...' @@ -197,6 +198,7 @@ ARGV.each do |jobfile| messages = scheduler_client.submit_job(job_json) JSON.parse(messages).each do |msg| if msg['message'].empty? + result_roots << msg['result_root'] job_ids << msg['job_id'].to_s puts("submit #{jobfile}, got job id=#{msg['job_id']}") else @@ -210,6 +212,6 @@ end if opt_monitor job_hash_list[0].delete('define_files') opt_monitor_query.merge!({'job_id' => job_ids}) - cmd = "#{LKP_SRC}/sbin/monitor -f '#{opt_monitor_query.to_json}' -j '#{job_hash_list[0].to_json}' -a #{actions.join(',')} -s 'job_state: extract_finished'" + cmd = "#{LKP_SRC}/sbin/monitor -f '#{opt_monitor_query.to_json}' -j '#{job_hash_list[0].to_json}' -a #{actions.join(',')} -r #{result_roots.join(',')} -s 'job_state: extract_finished'" exec cmd end -- 2.23.0

1 0

[PATCH lkp-tests] /stats add proc-vmstat.rb
by Lu Weitao 05 Mar '21

05 Mar '21

[Why] convert stats/proc-vmstat to proc_vmstat() of proc-vmstat.rb Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- stats/proc-vmstat.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 stats/proc-vmstat.rb diff --git a/stats/proc-vmstat.rb b/stats/proc-vmstat.rb new file mode 100644 index 000000000..d49002801 --- /dev/null +++ b/stats/proc-vmstat.rb @@ -0,0 +1,10 @@ + +def proc_vmstat(log_lines) + result = Hash.new { |hash, key| hash[key] = [] } + log_lines.each do |line| + key, value = line.split + result[key.chomp(':')] << value.to_i + end + + result +end -- 2.23.0

2 3

[PATCH lkp-tests] add test script for dockerfile os migrating
by Bai Jing 05 Mar '21

05 Mar '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 139 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100755 tests/openeuler_docker diff --git a/tests/openeuler_docker b/tests/openeuler_docker new file mode 100755 index 000000000..b77b890a9 --- /dev/null +++ b/tests/openeuler_docker @@ -0,0 +1,139 @@ +#!/bin/sh +# - dockerfile_url + +# exit 1 if openEuler failed +# exit 0 if +# - no centos support +# - centos failed + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/log.sh + +[ -n "$dockerfile_url" ] || die "dockerfile_url is empty" + +cd $BENCHMARK_ROOT + +centos_one_ok=false + +fit_repo() +{ + yum reinstall -y git + git clone "$dockerfile_url" || exit 0 + file_dir=$(find ./ -type f -name "*Dockerfile*" | xargs grep -HE '^FROM centos:(centos|)[78]') + [ -n "$file_dir" ] || { + echo "current-repo not support centos" + exit 0 + } +} + +openeuler_ready() +{ + docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar +} + +centos_docker() +{ + flag=0 + i=1 + for file in $(find ./ -type f -name "*Dockerfile*" | xargs grep -HE '^FROM centos:(centos|)[78]' | awk -F: '{print $1}') + do + file_dir=${file%/*} + image=$(echo "$file" | awk -F '/' '{print $3}' | tr 'A-Z' 'a-z') + [ -n "$image" ] || image=${file#*/}_$i + if [ -f "$file_dir/build.sh" ] && [ -x "$file_dir/build.sh" ]; then + "$file_dir/build.sh" > /dev/null 2> /dev/null && { + centos_one_ok=true + handle_dockerfile + } + else + docker_build > /dev/null 2> /dev/null && { + centos_one_ok=true + handle_dockerfile + } + fi + let i++ + done + [ "$centos_one_ok" == "false" ] && { + echo "centos all failed" + exit 0 + } +} + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$file" + +} + +handle_epel() +{ + grep -qw "epel-release" "$file" && { + sed -i "s/epel-release/bash/g" "$file" + } +} + +handle_epelrpm() +{ + + grep -qw "https://.*epel-release.*rpm" "$file" && { + sed -i "s|https://.*epel-release.*rpm|bash|g" "$file" + + } +} + +handle_groupadd() +{ + grep -qw "groupadd*" "$file" && { + sed -i '/groupadd*/i\ yum -y install shadow && \\' "$file" + } + +} + +handle_epel_repo() +{ + grep -qwE '.* wget .*(epel|CentOS-Base)\.repo' "$file" && { + sed 's|^|#|g' "$file" + } +} + +handle_dockerfile() +{ + flag=1 + handle_FROM + handle_epel + handle_groupadd + docker_build "-openeuler" + [ $? -ne 0 ] || docker_run "-openeuler" +} + +docker_build() +{ + docker build -f "$file" -t "${image}$1" . + [ $? -ne 0 ] && { + [ "$flag" == "1" ] && echo "${image}$1.build.fail: 1" + return 1 + } + [ "$flag" == "1" ] && echo "${image}$1.build.success: 1" + flag=0 +} + +docker_run() +{ + docker run -d --name "${image}" "${image}$1" + docker_stats=$(docker inspect ${image} | grep Running | awk -F ' ' '{print $2}' | tr -d ',') + [ $docker_stats == false ] && { + echo "${image}.run.fail: 1" + return 1 + } + + echo "${image}.run.success: 1" +} + +docker_test() +{ + fit_repo + openeuler_ready + centos_docker +} + +docker_test -- 2.23.0

2 2

[PATCH v2 compass-ci] service/lifecycle: add lifecycle
by Wu Zhende 05 Mar '21

05 Mar '21

Function: processing timeout jobs/machines processing crash jobs/machines Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/lifecycle.cr | 317 +++++++++++++++++++++++++- src/lifecycle.cr | 17 +- src/lifecycle/constants.cr | 3 + src/lifecycle/lifecycle.cr | 19 +- src/scheduler/elasticsearch_client.cr | 18 +- 5 files changed, 339 insertions(+), 35 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index af5cd07..3a72906 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -1,19 +1,43 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +require "set" require "kemal" require "yaml" -require "./web_env" +require "./mq" +require "./scheduler_api" require "../scheduler/elasticsearch_client" +require "../lifecycle/constants" + +class String + def bigger_than?(time) + return false if self.empty? + + time = time.to_s + return true if time.empty? + + # Historical data contains time in the format of "%Y-%m-%d %H:%M:%S" + # Compatibility processing is required + time = time.gsub(/ /, "T") + time = Time.parse(time, "%Y-%m-%dT%H:%M:%S", Time.local.location) + self_time = Time.parse(self, "%Y-%m-%dT%H:%M:%S", Time.local.location) + + self_time > time + end +end class Lifecycle property es - def initialize(env : HTTP::Server::Context) + def initialize + @mq = MQClient.instance @es = Elasticsearch::Client.new - @env = env - @log = env.log.as(JSONLogger) + @scheduler_api = SchedulerAPI.new + @log = JSONLogger.new + @jobs = Hash(String, JSON::Any).new + @machines = Hash(String, JSON::Any).new + @match = Hash(String, Set(String)).new {|h, k| h[k] = Set(String).new} end def alive(version) @@ -22,18 +46,291 @@ class Lifecycle @log.warn(e) end - def get_running_testbox - size = @env.params.query["size"]? || 20 - from = @env.params.query["from"]? || 0 + def init_from_es + jobs = get_active_jobs + jobs.each do |result| + job_id = result["_id"].to_s + job = result["_source"].as_h + job.delete_if{|key, _| !JOB_KEYWORDS.includes?(key)} + + @jobs[job_id] = JSON.parse(job.to_json) + @match[job["testbox"].to_s] << job_id + end + + machines = get_active_machines + machines.each do |result| + testbox = result["_id"].to_s + machine = result["_source"].as_h + machine.delete("history") + + machine = JSON.parse(machine.to_json) + @machines[testbox] = machine + + deal_match_job(testbox, machine["job_id"].to_s) + end + end + + def deal_match_job(testbox, job_id) + @match[testbox].each do |id| + next if id == job_id + + msg = { + "job_id" => id, + "job_state" => "abnormal", + "testbox" => testbox + } + @mq.pushlish_confirm("job_mq", msg.to_json) + @match[testbox].delete(id) + end + end + + def get_active_jobs query = { - "size" => size, - "from" => from, + "size" => 10000, + "query" => { + "term" => { + "job_state" => "boot" + } + } + } + @es.search("jobs", query) + end + + def get_active_machines + query = { + "size" => 10000, "query" => { "terms" => { - "state" => ["booting", "running"] + "state" => ["booting", "running", "rebooting"] } } } @es.search("testbox", query) end + + def mq_event_loop + puts "deal job events" + q = @mq.ch.queue("job_mq") + q.subscribe(no_ack: false) do |msg| + event = JSON.parse(msg.body_io.to_s) + job_state = event["job_state"]? + + case job_state + when "boot" + on_job_boot(event) + when "close" + on_job_close(event) + when "abnormal" + on_abnormal_job(event) + else + on_other_job(event) + end + @mq.ch.basic_ack(msg.delivery_tag) + end + end + + def on_other_job(event) + event_job_id = event["job_id"].to_s + return if event_job_id.empty? + + update_cached_job(event_job_id, event) + + job = @jobs[event_job_id]? + return unless job + + testbox = job["testbox"].to_s + update_cached_machine(testbox, event) + end + + def update_cached_machine(testbox, event) + machine = @machines[testbox]? + return if machine && !event["time"].to_s.bigger_than?(machine["time"]?) + + update_es_machine_time(testbox, event) + end + + def update_es_machine_time(testbox, event) + machine = @es.get_tbox(testbox) + return unless machine + return unless event["time"].to_s.bigger_than?(machine["time"]?) + + machine.as_h.delete("history") + machine.as_h["time"] = event["time"] + machine.as_h["state"] = JSON::Any.new("booting") + @machines[testbox] = machine + @es.update_tbox(testbox, machine.as_h) + end + + def update_cached_job(job_id, event) + job = @jobs[job_id]? + if job + @jobs[job_id] = JSON.parse(job.as_h.merge!(event.as_h).to_json) + else + job = @es.get_job(job_id) + return unless job + return if JOB_CLOSE_STATE.includes?(job["job_state"]?) + + job = job.dump_to_json_any.as_h + job.delete_if{|key, _| !JOB_KEYWORDS.includes?(key)} + job["job_state"] = event["job_state"] + @jobs[job_id] = JSON.parse(job.to_json) + end + end + + def on_abnormal_job(event) + event_job_id = event["job_id"].to_s + return unless @jobs.has_key?(event_job_id) + + @jobs.delete(event_job_id) + spawn @scheduler_api.close_job(event_job_id, "abnormal", "lifecycle") + end + + def on_job_close(event) + event_job_id = event["job_id"].to_s + job = @jobs[event_job_id] + + return unless job + + @jobs.delete(event_job_id) + update_cached_machine(job["testbox"].to_s, event) + end + + def on_job_boot(event) + event_job_id = event["job_id"]?.to_s + @jobs[event_job_id] = event unless event_job_id.empty? + machine_info = @machines[event["testbox"]]? + deal_boot_machine(machine_info, event) + end + + def deal_boot_machine(machine_info, event) + event_job_id = event["job_id"]?.to_s + + unless machine_info + @machines[event["testbox"].to_s] = event + return + end + + machine_job_id = machine_info["job_id"].to_s + # The job is not updated + # No action is required + return if event_job_id == machine_job_id + + time = machine_info["time"]? + # Skip obsolete event + return unless event["time"].to_s.bigger_than?(time) + + @machines[event["testbox"].to_s] = event + + deal_match_job(event["testbox"].to_s, event_job_id) + + # No previous job to process + return if machine_job_id.empty? + return unless @jobs.has_key?(machine_job_id) + + @jobs.delete(machine_job_id) + spawn @scheduler_api.close_job(machine_job_id, "abnormal", "lifecycle") + end + + def max_time(times) + result = "" + times.each do |time| + result = time if time.to_s.bigger_than?(result) + end + return result + end + + def timeout_job_loop + dead_job_id = nil + loop do + close_job(dead_job_id, "timeout") if dead_job_id + deadline, dead_job_id = get_min_deadline + # deal timeout job + next if dead_job_id && deadline <= Time.local + + sleep_until(deadline) + end + end + + def timeout_machine_loop + dead_machine_name = nil + loop do + reboot_timeout_machine(dead_machine_name) if dead_machine_name + deadline, dead_machine_name = get_min_deadline_machine + next if dead_machine_name && deadline <= Time.local + + sleep_until(deadline) + end + end + + def sleep_until(deadline) + s = (deadline - Time.local).total_seconds + sleep(s) + end + + def get_min_deadline + deadline = (Time.local + 60.second) + dead_job_id = nil + @jobs.each do |id, job| + next unless job["deadline"]? + job_deadline = Time.parse(job["deadline"].to_s, "%Y-%m-%dT%H:%M:%S", Time.local.location) + return job_deadline, id if Time.local >= job_deadline + next unless deadline > job_deadline + + deadline = job_deadline + dead_job_id = id + end + return deadline, dead_job_id + end + + def get_min_deadline_machine + deadline = (Time.local + 60.second) + dead_machine_name = nil + @machines.each do |name, machine| + next if machine["deadline"]?.to_s.empty? + + machine_deadline = Time.parse(machine["deadline"].to_s, "%Y-%m-%dT%H:%M:%S", Time.local.location) + return machine_deadline, name if Time.local >= machine_deadline + next unless deadline > machine_deadline + + deadline = machine_deadline + dead_machine_name = name + end + return deadline, dead_machine_name + end + + def close_job(job_id, reason) + @jobs.delete(job_id) + spawn @scheduler_api.close_job(job_id, reason, "lifecycle") + end + + def reboot_timeout_machine(testbox) + @machines.delete(testbox) + machine = @es.get_tbox(testbox) + return unless machine + return if MACHINE_CLOSE_STATE.includes?(machine["state"]) + + deadline = machine["deadline"]? + return unless deadline + + deadline = Time.parse(deadline.to_s, "%Y-%m-%dT%H:%M:%S", Time.local.location) + return if Time.local < deadline + + mq_queue = get_machine_reboot_queue(testbox) + @mq.pushlish_confirm(mq_queue, machine.to_json) + + machine["state"] = "rebooting_queue" + machine["time"] = Time.local.to_s("%Y-%m-%dT%H:%M:%S+0800") + @es.update_tbox(testbox, machine.as_h) + end + + def get_machine_reboot_queue(testbox) + if testbox.includes?(".") + testbox =~ /.*\.(.*)-\d+$/ + else + testbox =~ /(.*)--.*/ + end + $1 + rescue + testbox + end end diff --git a/src/lifecycle.cr b/src/lifecycle.cr index a864621..f6b80a8 100644 --- a/src/lifecycle.cr +++ b/src/lifecycle.cr @@ -2,15 +2,20 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. require "lifecycle/lifecycle" -require "./lifecycle/constants.cr" +require "./lifecycle/constants" require "./lib/json_logger" +require "./lib/lifecycle" module Cycle log = JSONLogger.new + lifecycle = Lifecycle.new - begin - Kemal.run(LIFECYCLE_PORT) - rescue e - log.error(e) - end + # init @jobs and @machines + lifecycle.init_from_es + lifecycle.mq_event_loop + + spawn lifecycle.timeout_job_loop + spawn lifecycle.timeout_machine_loop + + Kemal.run(LIFECYCLE_PORT) end diff --git a/src/lifecycle/constants.cr b/src/lifecycle/constants.cr index 137b7f9..694a407 100644 --- a/src/lifecycle/constants.cr +++ b/src/lifecycle/constants.cr @@ -2,3 +2,6 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. LIFECYCLE_PORT = (ENV.has_key?("LIFECYCLE_PORT") ? ENV["LIFECYCLE_PORT"] : 11312).to_i32 +JOB_CLOSE_STATE = ["abnormal", "close", "failed", "finished", "timeout", "crash"] +JOB_KEYWORDS = ["testbox", "job_state", "deadline"] +MACHINE_CLOSE_STATE = ["real_rebooting", "rebooting_queue"] diff --git a/src/lifecycle/lifecycle.cr b/src/lifecycle/lifecycle.cr index da42fda..c318721 100644 --- a/src/lifecycle/lifecycle.cr +++ b/src/lifecycle/lifecycle.cr @@ -9,24 +9,7 @@ require "../lib/json_logger" module Cycle VERSION = "0.1.0" - - add_context_storage_type(Time::Span) - - before_all do |env| - env.set "start_time", Time.monotonic - env.response.headers["Connection"] = "close" - env.create_log - env.create_lifecycle - end - - # echo alive get "/" do |env| - env.lifecycle.alive(VERSION) - end - - # find the testbox that are performing jobs - # curl http://localhost:11312/get_running_testbox?size=10&from=0 - get "/get_running_testbox" do |env| - env.lifecycle.get_running_testbox.to_json + "done" end end diff --git a/src/scheduler/elasticsearch_client.cr b/src/scheduler/elasticsearch_client.cr index 91fb84a..f6ff618 100644 --- a/src/scheduler/elasticsearch_client.cr +++ b/src/scheduler/elasticsearch_client.cr @@ -79,7 +79,10 @@ class Elasticsearch::Client end def search(index, query) - @client.search({:index => index, :body => query}) + results = @client.search({:index => index, :body => query}) + raise results unless results.is_a?(JSON::Any) + + results["hits"]["hits"].as_a end def update_account(account_content : JSON::Any, my_email : String) @@ -92,6 +95,19 @@ class Elasticsearch::Client ) end + def get_tbox(testbox) + query = {:index => "testbox", :type => "_doc", :id => testbox} + return nil unless @client.exists(query) + + response = @client.get_source(query) + case response + when JSON::Any + return response + else + return nil + end + end + def update_tbox(testbox : String, wtmp_hash : Hash) query = {:index => "testbox", :type => "_doc", :id => testbox} if @client.exists(query) -- 2.23.0

1 0

[PATCH v4 compass-ci] container/submit: add attach lkp-tests
by Luan Shengde 05 Mar '21

05 Mar '21

add attach lkp-tests to the container when run the container to submit jobs. [why] enable user edit code or job file(s). user can edit the code or job file(s) according to their requires. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/submit/submit | 1 + 1 file changed, 1 insertion(+) diff --git a/container/submit/submit b/container/submit/submit index 6236892..724e04e 100755 --- a/container/submit/submit +++ b/container/submit/submit @@ -13,6 +13,7 @@ cmd=( --name=submit-$USER-$data_suffix -it -v /etc/compass-ci:/etc/compass-ci:ro + -v $LKP_SRC:/srv/lkp-tests:ro -v $HOME/.config:/root/.config:ro -v $HOME/.ssh:/root/.ssh:rw submit -- 2.23.0

1 0

[PATCH compass-ci 1/2] sbin/result2stats: support call function from stats/script
by Lu Weitao 05 Mar '21

05 Mar '21

[Why] old logical: lkp-tests/stats/srcipt < script_log > tmpfile lkp-tests/sbin/dump-stat < tmpfile > $script.json this flow will escape special characters [How] use return of function to keep original characters lkp-tests/stats/srcipt.rb func(read(script_log)) --> hash compass-ci/lib/dump-stats.rb hash --> dump_stats() --> $script.json Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/stats.rb | 19 ++++++++++++++----- lib/stats_wrapper.rb | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/lib/stats.rb b/lib/stats.rb index c01d38d..94e370a 100644 --- a/lib/stats.rb +++ b/lib/stats.rb @@ -14,11 +14,16 @@ end def available_stats vailable_stats = [] + available_stats_func = [] Dir.open(LKP_SRC + '/stats').each do |file| - vailable_stats << file unless file.start_with?('.') + if file.end_with?('.rb') + available_stats_func << file.chomp('.rb') + else + vailable_stats << file unless file.start_with?('.') + end end - vailable_stats + [vailable_stats, available_stats_func] end def assign_default_stats(stats_list) @@ -32,13 +37,17 @@ class Stats def initialize(result_root) @result_root = result_root @job = load_yaml(File.join(result_root, 'job.yaml')) - @available_stats = available_stats + @available_stats, @available_stats_func = available_stats end def extract_stats stats_list = assign_stats_list stats_list.each do |stat| - StatsWrapper.wrapper(*stat) + if @available_stats_func.include?(stat) + StatsWrapper.wrapper_func(*stat) + else + StatsWrapper.wrapper(*stat) + end end end @@ -48,7 +57,7 @@ class Stats stats_list << ['time', @job['suite'] + '.time'] stats_list << 'stderr' @job.each_key do |k| - if @available_stats.include?(k) + if @available_stats.include?(k) || @available_stats_func.include?(k) stats_list << k end end diff --git a/lib/stats_wrapper.rb b/lib/stats_wrapper.rb index 8a6d6a9..09662be 100644 --- a/lib/stats_wrapper.rb +++ b/lib/stats_wrapper.rb @@ -6,6 +6,11 @@ LKP_SRC ||= ENV['LKP_SRC'] || '/c/lkp-tests' require "#{LKP_SRC}/lib/log.rb" require 'tempfile' require 'English' +require_relative './dump_stat' + +Dir[File.expand_path("#{LKP_SRC}/stats/*.rb")].uniq.each do |file| + require file +end PROGRAM_DIR = "#{LKP_SRC}/stats" @@ -35,6 +40,26 @@ module StatsWrapper delete_log_package end + def self.wrapper_func(program, program_time = nil) + @program = program + @stats_group = program_time || program + @log = "#{RESULT_ROOT}/#{@stats_group}" + + return unless File.exist?("#{(a)log}.yaml") || pretreatment + + if File.exist?("#{(a)log}.yaml") + stat_result = YAML.safe_load("#{(a)log}.yaml") + else + log_lines = read_log(@log) + call_func_cmd = "#{(a)program.gsub('-', '_')}(log_lines)" # eg: proc_vmstats(log_lines) + stat_result = eval(call_func_cmd) + end + return unless DumpStat.dump_stat(@stats_group, stat_result) + + check_empty_json + delete_log_package + end + def self.pretreatment return unless available_program? return unless unzip_log @@ -261,3 +286,11 @@ module StatsWrapper File.delete(@tmpfile) if File.exist?(@tmpfile) end end + +# read line of log +# return Array(String) +def read_log(log_path) + return nil unless File.exist?(log_path) + + File.readlines(log_path) +end -- 2.23.0

2 2

[PATCH v2 compass-ci] src/lib/job.cr: delete os_mount in common_dir for upload rpm
by Li Ping 04 Mar '21

04 Mar '21

[why] rpm has no much to do with os_mount. Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/job.cr | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0b4bee4..a30c969 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -274,7 +274,11 @@ class Job tmp_os_mount = pkg_style["os_mount"]? == nil ? "#{os_mount}" : pkg_style["os_mount"] mount_type = tmp_os_mount == "cifs" ? "nfs" : tmp_os_mount.dup - common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + if @hash["rpmbuild-pkg"]? + common_dir = "#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + else + common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + end return common_dir end -- 2.23.0

2 2

[PATCH compass-ci] lib/git_mirror.rb: Add new_refs_log for kibana
by Li Yuanchao 04 Mar '21

04 Mar '21

The logs can be shown as chart by kibana. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index bb0c9f9..92a3830 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -192,6 +192,7 @@ class MirrorMain feedback_info[:new_refs] = new_refs send_message(feedback_info) + new_refs_log(git_repo, new_refs) end def do_push(fork_key) @@ -509,4 +510,15 @@ class MirrorMain traverse_repodir(repodir, belong) end end + + def new_refs_log(git_repo, new_refs) + log = { + msg: 'new refs', + date: Date.today, + time: Time.now, + repo: git_repo, + nr_new_branch: new_refs[:heads].length + }.to_json + puts log + end end -- 2.23.0

1 0