[why]
Reduce the nesting of results and output yaml
[example]
es-jobs os=archLinux
[output]
kvcount.arch=aarch64: 43
kvcount.arch=x86_64: 2
kvcount.category=benchmark: 12
kvcount.category=functional: 33
kvcount.job_state=failed: 20
kvcount.job_state=finished: 25
kvcount.os=archLinux: 45
kvcount.suite=cci-depends: 28
kvcount.suite=cci-makepkg: 5
kvcount.suite=iperf: 3
kvcount.suite=iperf-walk-os-test: 9
kvcount.tbox_group=vm-2p16g--wangyong: 1
kvcount.tbox_group=vm-2p16g--wcl: 1
kvcount.tbox_group=vm-2p16g.wangyong: 34
kvcount.tbox_group=vm-2p8g: 9
sum.stats.stderr./lkp/lkp/src/monitors/perf-stat:#:main: 12
sum.stats.stderr.Can_not_find_perf_command: 12
sum.stats.stderr./lkp/lkp/src/tests/wrapper:line#:which:command_not_found: 10
sum.stats.last_state.test.iperf.exit_code.127: 12
...
raw.stats.sched_debug.cfs_rq:/.load.stddev: [524288.0, null, null, null, null, null,
null, null, null, 524288.0, 524288.0, null, null, null, null, null, 516608.0, null, 1048576.0,
null, null, null, null, null, null, null, 524288.0, null, null, null, null, 2104832.0, 1572864.0,
null, null, 2097152.0, null, null, null, null, null, null, null, null, null]
...
avg.stats.sched_debug.cfs_rq:/.load.stddev: 1048576.0
avg.stats.softirqs.CPU1.NET_RX: 2.5833333333333335
avg.stats.slabinfo.kmalloc-512.active_objs: 1372.75
...
crystal.122454.summary.success: 1
crystal.122454.summary.result: success
crystal.122690.summary.any_warning: 1
crystal.122690.summary.any_stderr: 1
crystal.122690.summary.any_error: 1
crystal.122690.summary.result: error
...
Signed-off-by: Lu Kaiyi <2392863668(a)qq.com>
---
lib/es_jobs.rb | 126 ++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 119 insertions(+), 7 deletions(-)
diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb
index a90b1ba..c8a5b6c 100644
--- a/lib/es_jobs.rb
+++ b/lib/es_jobs.rb
@@ -2,7 +2,9 @@
# frozen_string_literal: true
LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests'
+KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo].freeze
+require 'yaml'
require "#{LKP_SRC}/lib/stats"
require_relative './es_query'
@@ -123,17 +125,127 @@ class ESJobs
return 0
end
- def output
+ def get_all_metrics(jobs)
+ metrics = []
+ jobs.each do |job|
+ stats = job['stats']
+ next unless stats
+
+ metrics.concat(stats.keys)
+ end
+ metrics.uniq!
+ end
+
+ def initialize_result_hash(metrics)
result = {
- 'stats.count' => @stats['stats.count']
+ 'kvcount' => {},
+ 'sum.stats' => {},
+ 'raw.stats' => {},
+ 'avg.stats' => {},
+ 'max.stats' => {},
+ 'min.stats' => {}
}
+ metrics.each { |metric| result['raw.stats'][metric] = [] }
+ result
+ end
+
+ def set_default_value(result, stats, metrics)
+ left_metrics = metrics - stats.keys
+ left_metrics.each { |metric| result['raw.stats'][metric] << nil }
- @stats.each do |key, value|
- result[key] = value if @fields.include?(key)
+ stats.each do |key, value|
+ result['raw.stats'][key] << value
end
+ end
- @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty?
- @result.merge!(result)
- puts JSON.pretty_generate(@result)
+ def kvcount(result, job)
+ KEYWORD.each do |keyword|
+ next unless job[keyword]
+
+ result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0
+ result['kvcount']["#{keyword}=#{job[keyword]}"] += 1
+ end
+ end
+
+ def set_jobs_stat(result, stats, job)
+ summary_result = ''
+ if stats.keys.any? { |stat| stat.match(/warn/i) }
+ result["#{job['id']}.summary.any_warning"] = 1
+ summary_result = 'warning'
+ end
+ if stats.keys.any? { |stat| stat.match(/stderr/i) }
+ result["#{job['id']}.summary.any_stderr"] = 1
+ summary_result = 'stderr'
+ end
+ if stats.keys.any? { |stat| stat.match(/error|nr_fail=0/i) }
+ result["#{job['id']}.summary.any_error"] = 1
+ summary_result = 'error'
+ end
+ if stats.keys.any? { |stat| stat.match(/fail/i) } &&
+ stats.keys.all? { |stat| !stat.match(/nr_fail=0/i) }
+ result["#{job['id']}.summary.any_fail"] = 1
+ summary_result = 'fail'
+ end
+
+ if summary_result.empty?
+ result["#{job['id']}.summary.success"] = 1
+ result["#{job['id']}.summary.result"] = 'success'
+ else
+ result["#{job['id']}.summary.result"] = summary_result
+ end
+ end
+
+ def stats_count(result)
+ result['raw.stats'].each do |key, value|
+ if function_stat?(key)
+ result['sum.stats'][key] = value.compact.size
+ else
+ result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f
+ result['max.stats'][key] = value.compact.max
+ result['min.stats'][key] = value.compact.min
+ end
+ result['raw.stats'][key] = '---' + value.inspect + '---'
+ end
+ end
+
+ def query_jobs_state(jobs)
+ metrics = get_all_metrics(jobs)
+ result = initialize_result_hash(metrics)
+ jobs.each do |job|
+ stats = job['stats']
+ next unless stats
+
+ set_jobs_stat(result, stats, job)
+ set_default_value(result, stats, metrics)
+ kvcount(result, job)
+ end
+
+ stats_count(result)
+ result
+ end
+
+ def compact_hash(prefix, result)
+ result.each do |key, value|
+ if prefix.empty?
+ prefix_key = "#{key}"
+ else
+ prefix_key = "#{prefix}.#{key}"
+ end
+
+ if value.is_a? Hash
+ compact_hash(prefix_key, value)
+ else
+ @results[prefix_key] = value
+ end
+ end
+ end
+
+ def output
+ @result = query_jobs_state((a)jobs.values)
+ @result['kvcount'] = @result['kvcount'].to_a.sort.to_h
+ @results = {}
+ compact_hash('', @result)
+ yaml = @results.to_yaml.delete_prefix!("---\n")
+ puts yaml.gsub('nil', 'null').gsub('"---', '').gsub('---"', '')
end
end
--
2.23.0