March 2021 - Compass-ci - mailweb.openeuler.org

[PATCH v3 compass-ci 2/2] src/lib/web_backend.rb: implement for group jobs stats count
by Li Yuanchao 19 Mar '21

19 Mar '21

Query jobs by conditions such as group id or suite, and count the number of passed cases and failed cases. output is like: { "kezhiming" : { "nr_all" : $nr_all, "nr_pass" : $nr_pass, "nr_fail" : $nr_fail }, "chenqun" : { "nr_all" : $nr_all, "nr_pass" : $nr_pass, "nr_fail" : $nr_fail }, ... } Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 68 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 89c0f3c..04bccdb 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -121,6 +121,8 @@ end def get_dimension_conditions(params) dimension = params.key?(:dimension) ? [params.delete(:dimension)] : [] + dimension = params.key?(:GROUP_BY) ? [params.delete(:GROUP_BY)] : [] if dimension.empty? + conditions = {} FIELDS.each do |f| v = params[f] @@ -548,3 +550,69 @@ def new_refs_statistics(params) end [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] end + +def single_count(stats) + fail_count = 0 + pass_count = 0 + single_nr_fail = 0 + single_nr_pass = 0 + stats.each do |stat, value| + fail_count += 1 if stat.match(/\.fail$/i) + pass_count += 1 if stat.match(/\.pass$/i) + single_nr_fail = value if stat.match(/\.nr_fail$/i) + single_nr_pass = value if stat.match(/\.nr_pass$/i) + end + fail_count = single_nr_fail.zero? ? fail_count : single_nr_fail + pass_count = single_nr_pass.zero? ? pass_count : single_nr_pass + [fail_count, pass_count, fail_count + pass_count] +end + +def count_stats(job_list) + nr_fail = 0 + nr_pass = 0 + nr_all = 0 + job_list.each do |job| + next unless job['_source']['stats'] + + fail_count, pass_count, all_count = single_count(job['_source']['stats']) + nr_fail += fail_count + nr_pass += pass_count + nr_all += all_count + end + { 'nr_fail' => nr_fail, 'nr_pass' => nr_pass, 'nr_all' => nr_all } +end + +def get_jobs_stats_count(dimension, must, size, from) + dimension_list = get_dimension_list(dimension) + stats_count = {} + dimension_list.each do |dim| + job_list = query_dimension(dimension[0], dim, must, size, from) + stats_count[dim] = count_stats(job_list) + end + stats_count.to_json +end + +def get_stats_by_dimension(conditions, dimension, must, size, from) + must += build_multi_field_subquery_body(conditions) + count_query = { query: { bool: { must: must } } } + total = es_count(count_query) + return {} if total < 1 + + get_jobs_stats_count(dimension, must, size, from) +end + +def get_jobs_stats(params) + dimension, conditions = get_dimension_conditions(params) + must = get_es_must(params) + get_stats_by_dimension(conditions, dimension, must, 1000, 0) +end + +def group_jobs_stats(params) + begin + body = get_jobs_stats(params) + rescue StandardError => e + warn e.message + return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'group jobs table error'] + end + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] +end -- 2.23.0

1 0

[PATCH compass-ci] scheduler/elasticsearch_client.cr: fix search error
by Wu Zhende 19 Mar '21

19 Mar '21

Unhandled exception: Missing hash key: "hits" (KeyError) from /usr/lib/crystal/core/hash.cr:0:9 in '[]' from /usr/lib/crystal/core/json/any.cr:102:7 in '[]' from /c/cci/src/scheduler/elasticsearch_client.cr:85:5 in 'search' from /c/cci/src/lib/lifecycle.cr:109:5 in 'get_active_machines' from /c/cci/src/lib/lifecycle.cr:60:5 in 'init_from_es' from /c/cci/src/lifecycle.cr:14:3 in '__crystal_main' from /usr/lib/crystal/core/crystal/main.cr:97:5 in 'main_user_code' from /usr/lib/crystal/core/crystal/main.cr:86:7 in 'main' from /usr/lib/crystal/core/crystal/main.cr:106:3 in 'main' from ??? Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/scheduler/elasticsearch_client.cr | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/scheduler/elasticsearch_client.cr b/src/scheduler/elasticsearch_client.cr index f6ff618..219cf3d 100644 --- a/src/scheduler/elasticsearch_client.cr +++ b/src/scheduler/elasticsearch_client.cr @@ -82,7 +82,10 @@ class Elasticsearch::Client results = @client.search({:index => index, :body => query}) raise results unless results.is_a?(JSON::Any) - results["hits"]["hits"].as_a + return results["hits"]["hits"].as_a unless results.as_h.has_key?("error") + + puts results + Array(JSON::Any).new end def update_account(account_content : JSON::Any, my_email : String) -- 2.23.0

1 0

[PATCH v3 compass-ci] container/web-backend: add API /get_table_error
by Lu Weitao 19 Mar '21

19 Mar '21

[Example] curl -X GET 'localhost:30000/get_job_error?group_id=wcl_ansible-openeuler-03-10' return: { "filter":{"group_id":"wcl_ansible-openeuler-03-10"}, "table_fields":["job_id","error_id","error_message","link to result"], "data_table":[ [ "crystal.1354921", "ansible_test.error.Unable-to-start-service-httpd-Job-for-httpdservice-xxx.fail", "{\"changed\": false, \"msg\": \"Unable to start service httpd: Job for httpd.service failed because a timeout was exceeded.}", "http://172.17.0.1:11300/result/ansible_test/2021-03-10/vm-2p16g--wcl1/opene… ate/crystal.1354921" ], [ "crystal.1354929", ... ] ... ] } Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend/web-backend | 19 +++++++++ src/lib/web_backend.rb | 68 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 33a3fae..284e7a3 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -146,3 +146,22 @@ end get '/get_repo_statistics' do new_refs_statistics(params) end + +# GET /get_job_error?suite=virttest&tbox_group=vm-2p8g +# Response +# { +# "filter":{"group_id":"wcl_ansible-openeuler-03-10"}, +# "table_fields":["job_id","error_id","error_message","link to result"], +# "data_table":[ +# [ +# "crystal.1354921", +# "ansible_test.error.Unable-to-start-service-httpd-Job-for-httpdservice-failed.fail", +# "{\"changed\": false, \"msg\": \"Unable to start service httpd: Job for httpd.service failed because a timeout was exceeded.} +# "http://172.17.0.1:11300/result/ansible_test/2021-03-10/vm-2p16g--wcl1/opene…" +# ], +# ... +# ] +# } +get '/get_job_error' do + get_job_error(params) +end diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 89c0f3c..48b7ef2 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -12,6 +12,7 @@ require "#{CCI_SRC}/lib/compare.rb" require "#{CCI_SRC}/lib/constants.rb" require "#{CCI_SRC}/lib/es_query.rb" require "#{CCI_SRC}/lib/matrix2.rb" +require "#{CCI_SRC}/lib/params_group.rb" require "#{CCI_SRC}/lib/compare_data_format.rb" UPSTREAM_REPOS_PATH = ENV['UPSTREAM_REPOS_PATH'] || '/c/upstream-repos' @@ -548,3 +549,70 @@ def new_refs_statistics(params) end [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] end + + +# ------------------------------------------------------------------------------------------- +# job error table like: +# job_id error_id error_message link to result +# ------------------------------------------------------------------------------------- +# crystal.630608 "stderr.xxx" "messag:xxxx" https://$host:$port/$result_root +# ... +# ------------------------------------------------------------------------------------------- + +def get_job_error(params) + begin + body = job_error_body(params) + rescue StandardError => e + warn e.message + return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'get error table error'] + end + + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] +end + +def job_error_body(params) + error_table = get_error_table(params) + { + filter: params, + table_fields: ["job_id", "error_id", "error_message", "link to result"], + data_table: error_table, + }.to_json +end + +def get_error_table(filter_items) + error_table = [] + + job_list = get_job_list(filter_items) + job_list.each do |job| + error_table += get_error_from_job(job) + end + + error_table +end + +def get_job_list(items) + es = ESQuery.new(ES_HOST, ES_PORT) + query_results = es.multi_field_query(items) + + extract_jobs_list(query_results['hits']['hits']) +end + +# get all error_id from one job +def get_error_from_job(job) + job_error = [] + job['stats'].each do |metric, value| + next unless metric.end_with?('.message') + + result_host = ENV['SRV_HTTP_RESULT_HOST'] || SRV_HTTP_RESULT_HOST + result_port = ENV['SRV_HTTP_RESULT_PORT'] || SRV_HTTP_RESULT_PORT + error_id = metric.sub('.message', '.fail') + job_error << [ + job['id'], + error_id, + value, + "http://#{result_host}:#{result_port}#{job['result_root']}" + ] + end + + job_error +end -- 2.23.0

2 4

[PATCH lkp-tests] lib/monitor: add mirror_result action judgement
by Li Ping 19 Mar '21

19 Mar '21

[why] submit -m would trigger mirror_result, even if i have not use the option "-r" Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/monitor.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/monitor.rb b/lib/monitor.rb index 24191fa8d..f6dcef7fd 100755 --- a/lib/monitor.rb +++ b/lib/monitor.rb @@ -96,6 +96,8 @@ class Monitor end def mirror_result + return unless @action['mirror_result'] + @result_roots.each do |res| res.to_s.delete_prefix!('/srv') srv_http_host = job['SRV_HTTP_HOST'] || 'api.compass-ci.openeuler.org' -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] container/web-backend: add api for group jobs stats count
by Li Yuanchao 19 Mar '21

19 Mar '21

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/web-backend/web-backend | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 33a3fae..2971719 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -146,3 +146,28 @@ end get '/get_repo_statistics' do new_refs_statistics(params) end + +# GET /get_jobs_summary?suite=iperf&dimension=my_name +# must: +# - query_conditions +# - suite / group_id / ... +# - dimension +# - group_id / my_email / my_name +# +# Response like: +# - { +# "kezhiming" : { +# "all" : $all, +# "nr_pass" : $nr_pass, +# "nr_fail" : $nr_fali +# }, +# "chenqun" : { +# "all" : $all, +# "nr_pass" : $nr_pass, +# "nr_fail" : $nr_fali +# }, +# ... +# } +get '/get_jobs_summary' do + group_jobs_stats(params) +end -- 2.23.0

2 2

[PATCH compass-ci] dracut-initrd: add support of custom bootstrap-$(arch).cgz
by Yu Chuan 19 Mar '21

19 Mar '21

[Why] When user want run job with their own custom bootstrap instead of lkp-tests framework, they will generate a bootstrap-$(arch).cgz firstly, and scheduler will give the bootstrap-$(arch).cgz path to testbox, then testbox download the bootstrap-$(arch).cgz and use it as part of initrd. So we add a step to handle bootstrap-$(arch).cgz, extract all of it to $NEWROOT and boot from $NEWROOT. [Addition] The root dir in bootstrap-$(arch).cgz is /custom_bootstrap. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- container/dracut-initrd/bin/overlay-lkp.sh | 5 +++++ container/dracut-initrd/modules.d/90lkp/lkp-deploy.sh | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/container/dracut-initrd/bin/overlay-lkp.sh b/container/dracut-initrd/bin/overlay-lkp.sh index 9a2800e532f9..c2165baa6eea 100644 --- a/container/dracut-initrd/bin/overlay-lkp.sh +++ b/container/dracut-initrd/bin/overlay-lkp.sh @@ -1,6 +1,11 @@ #!/bin/bash # SPDX-License-Identifier: GPL-2.0 +# transfer custom bootstrap +[ -d /custom_bootstrap ] && { + cp -a /custom_bootstrap/* "$NEWROOT"/ +} + # transfer LKP dirs [ -d /lkp ] || return 0 diff --git a/container/dracut-initrd/modules.d/90lkp/lkp-deploy.sh b/container/dracut-initrd/modules.d/90lkp/lkp-deploy.sh index 44f994e1475e..8597f3086b6a 100755 --- a/container/dracut-initrd/modules.d/90lkp/lkp-deploy.sh +++ b/container/dracut-initrd/modules.d/90lkp/lkp-deploy.sh @@ -7,6 +7,11 @@ if ! getargbool 0 local; then return fi +# transfer custom bootstrap +[ -d /custom_bootstrap ] && { + cp -a /custom_bootstrap/* "$NEWROOT"/ +} + # transfer LKP dirs [ -d /lkp ] || return 0 -- 2.23.0

1 0

[PATCH compass-ci] scheduler: add custom_bootstrap logic
by Cao Xueliang 19 Mar '21

19 Mar '21

If the custom_bootstrap field in job yaml, need runtime field too, then just return the custom_bootstrap.cgz to the testbox. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/lib/job.cr | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/lib/job.cr b/src/lib/job.cr index 40aaa62..b96cf15 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -514,6 +514,15 @@ class Job private def common_initrds temp_initrds = [] of String + if @hash.has_key?("custom_bootstrap") + raise "need runtime field in the job yaml." unless @hash.has_key?("runtime") + + temp_initrds << "#{INITRD_HTTP_PREFIX}" + + JobHelper.service_path("#{SRV_INITRD}/custom_bootstrap/#{@hash["my_email"]}/bootstrap-#{os_arch}.cgz") + + return temp_initrds + end + temp_initrds << "#{INITRD_HTTP_PREFIX}" + JobHelper.service_path("#{SRV_INITRD}/lkp/#{lkp_initrd_user}/lkp-#{os_arch}.cgz") temp_initrds << "#{SCHED_HTTP_PREFIX}/job_initrd_tmpfs/#{id}/job.cgz" -- 2.23.0

1 0

[PATCH v2 compass-ci] providers/qemu/kvm.sh: added QEMU multiple network card functionality
by Wang Chenglong 19 Mar '21

19 Mar '21

[How] Use the 'nr_nic' to set the number of network cards in yaml. nr_nic: 2 root@vm-2p8g ~# ifconfig enp0s1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 172.18.42.60 netmask 255.255.0.0 broadcast 172.18.255.255 inet6 fe80::843:24ff:fef2:ec17 prefixlen 64 scopeid 0x20<link> ether 0a:43:24:f2:ec:17 txqueuelen 1000 (Ethernet) RX packets 155492 bytes 1172898602 (1.0 GiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 75 bytes 11207 (10.9 KiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 enp0s2: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 172.18.88.228 netmask 255.255.0.0 broadcast 172.18.255.255 inet6 fe80::820:70ff:fe34:d27b prefixlen 64 scopeid 0x20<link> ether 0a:20:70:34:d2:7b txqueuelen 1000 (Ethernet) RX packets 168 bytes 9079 (8.8 KiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 199208 bytes 32959935 (31.4 MiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- providers/qemu/kvm.sh | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index dcab81a..a1c0046 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -119,9 +119,34 @@ add_disk() done } +set_mac() +{ + job_id=$(awk -F'/' '/job_initrd_tmpfs/{print $(NF-1)}' $ipxe_script) + + if [ $(command -v es-find) ]; then + nr_nic=$(es-find id=$job_id | awk '/nr_nic/{print $2}' | tr -d ,) + else + echo "command not found: es-find. set nr_nic=1" + sleep 1 + fi + + mac_arr[1]=$mac + nr_nic=${nr_nic:-1} + + [ "$nr_nic" -ge "2" ] || return + for i in $(seq 2 $nr_nic) + do + mac=$(echo $hostname$i | md5sum | sed 's/^$..$$..$$..$$..$$..$.*$/0a-\1-\2-\3-\4-\5/') + mac_arr[$i]=$mac + done +} + set_nic() { - nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" + for i in $(seq 1 $nr_nic) + do + nic[$i]="-nic tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac_arr[$i]}" + done } set_device() @@ -190,7 +215,7 @@ individual_option() -machine virt-4.0,accel=kvm,gic-version=3 -cpu Kunpeng-920 -bios $bios - -nic $nic + ${nic[@]} ) ;; qemu-kvm) @@ -198,17 +223,17 @@ individual_option() -machine virt-4.0,accel=kvm,gic-version=3 -cpu Kunpeng-920 -bios $bios - -nic $nic + ${nic[@]} ) [ "$(arch)" == "x86_64" ] && arch_option=( -bios $bios - -nic $nic + ${nic[@]} ) ;; qemu-system-x86_64) arch_option=( -bios $bios - -nic $nic + ${nic[@]} ) ;; qemu-system-riscv64) @@ -234,6 +259,7 @@ set_options() { set_bios set_helper + set_mac set_nic set_device set_netdev -- 2.23.0

1 0

[PATCH compass-ci] providers/qemu/kvm.sh: added QEMU multiple network card functionality
by Wang Chenglong 19 Mar '21

19 Mar '21

[How] Use the 'nr_nic' to set the number of network cards in yaml. nr_nic: 2 root@vm-2p8g ~# ifconfig enp0s1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 172.18.42.60 netmask 255.255.0.0 broadcast 172.18.255.255 inet6 fe80::843:24ff:fef2:ec17 prefixlen 64 scopeid 0x20<link> ether 0a:43:24:f2:ec:17 txqueuelen 1000 (Ethernet) RX packets 155492 bytes 1172898602 (1.0 GiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 75 bytes 11207 (10.9 KiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 enp0s2: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 172.18.88.228 netmask 255.255.0.0 broadcast 172.18.255.255 inet6 fe80::820:70ff:fe34:d27b prefixlen 64 scopeid 0x20<link> ether 0a:20:70:34:d2:7b txqueuelen 1000 (Ethernet) RX packets 168 bytes 9079 (8.8 KiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 199208 bytes 32959935 (31.4 MiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- providers/qemu/kvm.sh | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index dcab81a..bbd5cd5 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -119,9 +119,35 @@ add_disk() done } +set_mac() +{ + job_id=$(cat ipxe_script|awk -F'/' '/job_initrd_tmpfs/{print $(NF-1)}') + + if [ $(command -v es-find) ]; then + nr_nic=$(es-find id=$job_id | awk '/nr_nic/{print $2}' | tr -d ,) + else + echo "command not found: es-find. set nr_nic=1" + sleep 1 + fi + + mac_arr[1]=$mac + nr_nic=${nr_nic:-1} + + if [ "$nr_nic" -ge "2" ]; then + for i in $(seq 2 $nr_nic) + do + mac=$(echo $hostname$i | md5sum | sed 's/^$..$$..$$..$$..$$..$.*$/0a-\1-\2-\3-\4-\5/') + mac_arr[$i]=$mac + done + fi +} + set_nic() { - nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" + for i in $(seq 1 $nr_nic) + do + nic[$i]="-nic tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac_arr[$i]}" + done } set_device() @@ -190,7 +216,7 @@ individual_option() -machine virt-4.0,accel=kvm,gic-version=3 -cpu Kunpeng-920 -bios $bios - -nic $nic + ${nic[@]} ) ;; qemu-kvm) @@ -198,17 +224,17 @@ individual_option() -machine virt-4.0,accel=kvm,gic-version=3 -cpu Kunpeng-920 -bios $bios - -nic $nic + ${nic[@]} ) [ "$(arch)" == "x86_64" ] && arch_option=( -bios $bios - -nic $nic + ${nic[@]} ) ;; qemu-system-x86_64) arch_option=( -bios $bios - -nic $nic + ${nic[@]} ) ;; qemu-system-riscv64) @@ -234,6 +260,7 @@ set_options() { set_bios set_helper + set_mac set_nic set_device set_netdev -- 2.23.0

3 4

[PATCH v2 compass-ci] sbin/es-jobs: add -e to get fail result
by Wu Zhende 19 Mar '21

19 Mar '21

es-jobs -e group_id=xxxx error.result.path=/srv/result/openeuler_docker/xxxx: ["Error: unable to find a match"] Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/es_jobs.rb | 24 +++++++++++++++++++++++- sbin/es-jobs | 7 ++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index b0d0fb2..e4e9530 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -12,13 +12,14 @@ require_relative './es_query' # deal jobs search from es class ESJobs - def initialize(es_query, my_refine = [], fields = [], stats_filter = []) + def initialize(es_query, my_refine = [], fields = [], stats_filter = [], error_result = false) @es_query = es_query @jobs = query_jobs_from_es @refine = my_refine @fields = fields @stats_filter = stats_filter @refine_jobs = [] + @error_result = error_result set_jobs_summary end @@ -98,6 +99,7 @@ class ESJobs 'max.stats' => {}, 'min.stats' => {} } + result['error.result'] = {} if @error_result metrics.each { |metric| result['raw.stats'][metric] = [] } result end @@ -119,7 +121,27 @@ class ESJobs result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 result['raw.id']["[#{keyword}=#{job[keyword]}]"] ||= [] result['raw.id']["[#{keyword}=#{job[keyword]}]"] << job['id'] + + next unless @error_result && job[keyword] == "failed" + + get_failed_result(result, job) + end + end + + def get_failed_result(result, job) + path = "/srv#{job['result_root']}/#{job['suite']}" + res = find_error_lines(path) + result['error.result']["path=#{path}"] = res + end + + def find_error_lines(path) + res = [] + return res unless File.exist?(path) + + File.open(path).each do |line| + res << line.to_s if line.match(/error:/i) end + res end def stats_count(result) diff --git a/sbin/es-jobs b/sbin/es-jobs index ff0d540..3804dc9 100755 --- a/sbin/es-jobs +++ b/sbin/es-jobs @@ -22,6 +22,7 @@ end opt_refine = [-1] opt_fields = [] opt_stats_filter = [] +opt_error_result = false opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: es-jobs [options] search_key1=val1[,val2..] ..' @@ -45,11 +46,15 @@ opt_parser = OptionParser.new do |opts| opts.on('-s fields', '--stats-filter fields', 'return data contains fields in stats') do |fields| opt_stats_filter = fields.split(',') end + + opts.on('-e', '--error.result', 'obtain the error information about job failure') do + opt_error_result = true + end end opt_parser.parse!(ARGV) items = parse_argv raise 'Please enter a query' if items.empty? -es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter) +es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter, opt_error_result) es_jobs.output -- 2.23.0

1 0