[why] Reduce the nesting of results and output yaml [example] es-jobs os=archLinux [output] kvcount.arch=aarch64: 43 kvcount.arch=x86_64: 2 kvcount.category=benchmark: 12 kvcount.category=functional: 33 kvcount.job_state=failed: 20 kvcount.job_state=finished: 25 kvcount.os=archLinux: 45 kvcount.suite=cci-depends: 28 kvcount.suite=cci-makepkg: 5 kvcount.suite=iperf: 3 kvcount.suite=iperf-walk-os-test: 9 kvcount.tbox_group=vm-2p16g--wangyong: 1 kvcount.tbox_group=vm-2p16g--wcl: 1 kvcount.tbox_group=vm-2p16g.wangyong: 34 kvcount.tbox_group=vm-2p8g: 9 sum.stats.stderr./lkp/lkp/src/monitors/perf-stat:#:main: 12 sum.stats.stderr.Can_not_find_perf_command: 12 sum.stats.stderr./lkp/lkp/src/tests/wrapper:line#:which:command_not_found: 10 sum.stats.last_state.test.iperf.exit_code.127: 12 ... raw.stats.sched_debug.cfs_rq:/.load.stddev: [524288.0, null, null, null, null, null, null, null, null, 524288.0, 524288.0, null, null, null, null, null, 516608.0, null, 1048576.0, null, null, null, null, null, null, null, 524288.0, null, null, null, null, 2104832.0, 1572864.0, null, null, 2097152.0, null, null, null, null, null, null, null, null, null] ... avg.stats.sched_debug.cfs_rq:/.load.stddev: 1048576.0 avg.stats.softirqs.CPU1.NET_RX: 2.5833333333333335 avg.stats.slabinfo.kmalloc-512.active_objs: 1372.75 ... crystal.122454.summary.success: 1 crystal.122454.summary.result: success crystal.122690.summary.any_warning: 1 crystal.122690.summary.any_stderr: 1 crystal.122690.summary.any_error: 1 crystal.122690.summary.result: error ... Signed-off-by: Lu Kaiyi <2392863668@qq.com> --- lib/es_jobs.rb | 126 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 119 insertions(+), 7 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..c8a5b6c 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -2,7 +2,9 @@ # frozen_string_literal: true LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' +KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo].freeze +require 'yaml' require "#{LKP_SRC}/lib/stats" require_relative './es_query' @@ -123,17 +125,127 @@ class ESJobs return 0 end - def output + def get_all_metrics(jobs) + metrics = [] + jobs.each do |job| + stats = job['stats'] + next unless stats + + metrics.concat(stats.keys) + end + metrics.uniq! + end + + def initialize_result_hash(metrics) result = { - 'stats.count' => @stats['stats.count'] + 'kvcount' => {}, + 'sum.stats' => {}, + 'raw.stats' => {}, + 'avg.stats' => {}, + 'max.stats' => {}, + 'min.stats' => {} } + metrics.each { |metric| result['raw.stats'][metric] = [] } + result + end + + def set_default_value(result, stats, metrics) + left_metrics = metrics - stats.keys + left_metrics.each { |metric| result['raw.stats'][metric] << nil } - @stats.each do |key, value| - result[key] = value if @fields.include?(key) + stats.each do |key, value| + result['raw.stats'][key] << value end + end - @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? - @result.merge!(result) - puts JSON.pretty_generate(@result) + def kvcount(result, job) + KEYWORD.each do |keyword| + next unless job[keyword] + + result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0 + result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 + end + end + + def set_jobs_stat(result, stats, job) + summary_result = '' + if stats.keys.any? { |stat| stat.match(/warn/i) } + result["#{job['id']}.summary.any_warning"] = 1 + summary_result = 'warning' + end + if stats.keys.any? { |stat| stat.match(/stderr/i) } + result["#{job['id']}.summary.any_stderr"] = 1 + summary_result = 'stderr' + end + if stats.keys.any? { |stat| stat.match(/error|nr_fail=0/i) } + result["#{job['id']}.summary.any_error"] = 1 + summary_result = 'error' + end + if stats.keys.any? { |stat| stat.match(/fail/i) } && + stats.keys.all? { |stat| !stat.match(/nr_fail=0/i) } + result["#{job['id']}.summary.any_fail"] = 1 + summary_result = 'fail' + end + + if summary_result.empty? + result["#{job['id']}.summary.success"] = 1 + result["#{job['id']}.summary.result"] = 'success' + else + result["#{job['id']}.summary.result"] = summary_result + end + end + + def stats_count(result) + result['raw.stats'].each do |key, value| + if function_stat?(key) + result['sum.stats'][key] = value.compact.size + else + result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f + result['max.stats'][key] = value.compact.max + result['min.stats'][key] = value.compact.min + end + result['raw.stats'][key] = '---' + value.inspect + '---' + end + end + + def query_jobs_state(jobs) + metrics = get_all_metrics(jobs) + result = initialize_result_hash(metrics) + jobs.each do |job| + stats = job['stats'] + next unless stats + + set_jobs_stat(result, stats, job) + set_default_value(result, stats, metrics) + kvcount(result, job) + end + + stats_count(result) + result + end + + def compact_hash(prefix, result) + result.each do |key, value| + if prefix.empty? + prefix_key = "#{key}" + else + prefix_key = "#{prefix}.#{key}" + end + + if value.is_a? Hash + compact_hash(prefix_key, value) + else + @results[prefix_key] = value + end + end + end + + def output + @result = query_jobs_state(@jobs.values) + @result['kvcount'] = @result['kvcount'].to_a.sort.to_h + @results = {} + compact_hash('', @result) + yaml = @results.to_yaml.delete_prefix!("---\n") + puts yaml.gsub('nil', 'null').gsub('"---', '').gsub('---"', '') end end -- 2.23.0