[why] Reduce the nesting of results and output yaml
[example] es-jobs os=archlinux
[output]
kvcount.arch=aarch64: 43 kvcount.arch=x86_64: 2 kvcount.category=benchmark: 12 kvcount.category=functional: 33 kvcount.job_state=failed: 20 kvcount.job_state=finished: 25 kvcount.os=archlinux: 45 kvcount.suite=cci-depends: 28 kvcount.suite=cci-makepkg: 5 kvcount.suite=iperf: 3 kvcount.suite=iperf-walk-os-test: 9 kvcount.summary.any_error=1: 40 kvcount.summary.any_stderr=1: 44 kvcount.summary.any_warning=1: 23 kvcount.summary.success=1: 1 kvcount.tbox_group=vm-2p16g--wangyong: 1 kvcount.tbox_group=vm-2p16g--wcl: 1 kvcount.tbox_group=vm-2p16g.wangyong: 34 kvcount.tbox_group=vm-2p8g: 9 ... raw.id.[job_state=failed]: ["crystal.122405","crystal.122406","crystal.122331","crystal.122365","crystal.122404","crystal.122438","crystal.122607","crystal.122370","crystal.122375","crystal.122410","crystal.122900","crystal.122906","crystal.122271","crystal.122451","crystal.122835","crystal.122834","crystal.122232","crystal.122403","crystal.122453","crystal.122886"] ... raw.id.[summary.any_error=1]: ["crystal.122436","crystal.122476","crystal.122803","crystal.122883","crystal.122910","crystal.122438","crystal.122452","crystal.122895","crystal.122446","crystal.122685","crystal.122687","crystal.122833","crystal.122451","crystal.122690","crystal.122719","crystal.122882","crystal.122890","crystal.122453","crystal.122443","crystal.122435","crystal.122432","crystal.122688","crystal.122689","crystal.122886"] ... sum.stats.last_state.test.iperf.exit_code.127: 12 sum.stats.stderr./lkp/lkp/src/lib/env.sh:#:set_perf_path: 12 sum.stats.last_state.is_incomplete_run: 12 sum.stats.kmsg.timestamp:last: 45 sum.stats.last_state.exit_fail: 20 sum.stats.stderr.has_stderr: 44 ... raw.stats.sched_debug.cfs_rq:/.load.stddev: [524288.0,null,null,null,null,null,null,null,null,524288.0,524288.0,null,null,null,null,null,516608.0,null,1048576.0,null,null,null,null,null,null,null,524288.0,null,null,null,null,2104832.0,1572864.0,null,null,2097152.0,null,null,null,null,null,null,null,null,null] ... avg.stats.sched_debug.cfs_rq:/.load.stddev: 1048576.0 avg.stats.softirqs.CPU1.NET_RX: 2.5833333333333335 avg.stats.slabinfo.kmalloc-512.active_objs: 1372.75 ... max.stats.sched_debug.cfs_rq:/.load.stddev: 2104832.0 max.stats.softirqs.CPU1.NET_RX: 6 max.stats.slabinfo.kmalloc-512.active_objs: 1616 ...
Signed-off-by: Lu Kaiyi 2392863668@qq.com --- lib/es_jobs.rb | 213 ++++++++++++++++++++++++++++--------------------- 1 file changed, 123 insertions(+), 90 deletions(-)
diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..c35b182 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -3,137 +3,170 @@
LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests'
+KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo summary.success + summary.any_fail summary.any_error summary.any_stderr summary.any_warning].freeze + +require 'json' require "#{LKP_SRC}/lib/stats" require_relative './es_query'
# deal jobs search from es class ESJobs def initialize(es_query, my_refine = [], fields = [], stats_filter = []) - @es_query = es_query - @es = ESQuery.new(ES_HOST, ES_PORT) + @jobs = query_jobs_from_es(es_query) @refine = my_refine @fields = fields @stats_filter = stats_filter - @stats_filter_result = {} @refine_jobs = [] - @jobs = {} - @stats_level = { - 0 => 'stats.success', - 1 => 'stats.unknown', - 2 => 'stats.warning', - 3 => 'stats.has_error' - } - set_defaults - deal_jobs + set_jobs_summary end
- def set_defaults - query_result = @es.multi_field_query(@es_query) - query_result['hits']['hits'].each do |job| - @jobs[job['_id']] = job['_source'] - end - - @stats = { - 'stats.count' => Hash.new(0), - 'stats.sum' => Hash.new(0), - 'stats.avg' => Hash.new(0) - } - @result = {} - @fields.each do |field| - @result[field] = [] - end + def query_jobs_from_es(items) + es = ESQuery.new(ES_HOST, ES_PORT) + result = es.multi_field_query items + jobs = result['hits']['hits'] + jobs.map! { |job| job['_source'] } + return jobs end
- def add_result_fields(job, level) - return unless @refine.include?(level) || @refine.include?(-1) - - @refine_jobs << job['id'] - @fields.each do |field| - value = job[field] - if value - value = job['id'] + '.' + value if field == 'job_state' - @result[field] << value + def set_job_summary(stats, job) + summary_result = '' + stats.each_key do |stat| + # "stderr.warning:rubygems_will_be_installed_before_its_ruby_dependency": 1, + # "stderr._#warning_FORTIFY_SOURCE_requires_compiling_with_optimization(-O)": 1, + # "stderr.disk_src.c:#:#:warning:incompatible_implicit_declaration_of_built-in_function'strcpy'": 1, + if stat.match(/warning/i) + job['summary.any_warning'] = 1 + summary_result = 'warning' + end + # "stderr.linux-perf": 1, + # "stderr.error:target_not_found:ruby-dev": 1, + # "stderr.error:could_not_open_file/var/lib/pacman/local/ldb-#:#-#/desc:Not_a_directory": 1, + if stat.match(/stderr./i) + job['summary.any_stderr'] = 1 + summary_result = 'stderr' + end + # "stderr.::Proceed_with_installation?[Y/n]error:target_not_found:liblzma-dev": 1, + # "stderr.==>ERROR:Failure_while_downloading_apache-cassandra": 1, + # "stderr.ftq.h:#:#:error:unknown_type_name'ticks'": 1, + # "last_state.test.iperf.exit_code.127": 1, + # "last_state.test.cci-makepkg.exit_code.1": 1, + if stat.match(/error|.exit_code./i) + job['summary.any_error'] = 1 + summary_result = 'error' + end + if stat.match(/.fail$/i) + job['summary.any_fail'] = 1 + summary_result = 'fail' end - - next unless job['stats'] - - @result[field] << job['stats'][field] if job['stats'][field] end - end - - def deal_jobs - stats_count = Hash.new(0) - stats_jobs = {} + return unless summary_result.empty?
- @jobs.each do |job_id, job| - level = deal_stats(job) - add_result_fields(job, level) + job['summary.success'] = 1 + end
- stat_key = @stats_level[level] - stat_jobs_key = stat_key + '_jobs' + # set jobs summary fields information in place + def set_jobs_summary + @jobs.each do |job| + stats = job['stats'] + next unless stats
- stats_count[stat_key] += 1 - stats_jobs[stat_jobs_key] ||= [] - stats_jobs[stat_jobs_key] << job_id + set_job_summary(stats, job) end - - @stats['stats.count'].merge!(stats_count) - @stats['stats.count'].merge!(stats_jobs) end
- def deal_stats(job, level = 0) - return 1 unless job['stats'] + def get_all_metrics(jobs) + metrics = [] + jobs.each do |job| + stats = job['stats'] + next unless stats
- job['stats'].each do |key, value| - match_stats_filter(key, value, job['id']) - calculate_stat(key, value) - level = get_stat_level(key, level) + metrics.concat(stats.keys) end - return level + metrics.uniq! end
- def match_stats_filter(key, value, job_id) - @stats_filter.each do |filter| - next unless key.include?(filter) + def initialize_result_hash(metrics) + result = { + 'kvcount' => {}, + 'raw.id' => {}, + 'sum.stats' => {}, + 'raw.stats' => {}, + 'avg.stats' => {}, + 'max.stats' => {}, + 'min.stats' => {} + } + metrics.each { |metric| result['raw.stats'][metric] = [] } + result + end
- key = job_id + '.' + key - @stats_filter_result[key] = value + def set_default_value(result, stats, metrics) + left_metrics = metrics - stats.keys + left_metrics.each { |metric| result['raw.stats'][metric] << nil }
- break + stats.each do |key, value| + result['raw.stats'][key] << value end end
- def calculate_stat(key, value) - if function_stat?(key) - return unless @fields.include?('stats.sum') + def kvcount(result, job) + KEYWORD.each do |keyword| + next unless job[keyword]
- @stats['stats.sum'][key] += value - else - return unless @fields.include?('stats.avg') + result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0 + result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 + result['raw.id']["[#{keyword}=#{job[keyword]}]"] ||= [] + result['raw.id']["[#{keyword}=#{job[keyword]}]"] << job['id'] + end + end
- @stats['stats.avg'][key] = (@stats['stats.avg'][key] + value) / 2 + def stats_count(result) + result['raw.stats'].each do |key, value| + if function_stat?(key) + result['sum.stats'][key] = value.compact.size + else + result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f + result['max.stats'][key] = value.compact.max + result['min.stats'][key] = value.compact.min + end end end
- def get_stat_level(stat, level) - return level if level >= 3 - return 3 if stat.match(/error|fail/i) - return 2 if stat.match(/warn/i) + def query_jobs_state(jobs) + metrics = get_all_metrics(jobs) + result = initialize_result_hash(metrics) + jobs.each do |job| + stats = job['stats'] + next unless stats + + set_default_value(result, stats, metrics) + kvcount(result, job) + end
- return 0 + stats_count(result) + result end
- def output - result = { - 'stats.count' => @stats['stats.count'] - } + def output_yaml(prefix, result) + result.each do |key, value| + if prefix.empty? + prefix_key = "#{key}" + else + prefix_key = "#{prefix}.#{key}" + end
- @stats.each do |key, value| - result[key] = value if @fields.include?(key) + if value.is_a? Hash + output_yaml(prefix_key, value) + else + puts "#{prefix_key}: #{value.to_json}" + end end + end
- @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? - @result.merge!(result) - puts JSON.pretty_generate(@result) + def output + @result = query_jobs_state(@jobs) + @result['kvcount'] = @result['kvcount'].sort.to_h + @result['raw.id'] = @result['raw.id'].sort.to_h + output_yaml('', @result) end end
- stats.each_key do |stat|
I'd suggest this:
The fail/error/warning cases should not take input from stderr.* stderr is rather chaotic -- let it stay in any_stderr.
# "stderr.warning:rubygems_will_be_installed_before_its_ruby_dependency": 1,
# "stderr._#warning_FORTIFY_SOURCE_requires_compiling_with_optimization(-O)": 1,
# "stderr.disk_src.c:#:#:warning:incompatible_implicit_declaration_of_built-in_function'strcpy'": 1,
if stat.match(/warning/i)
The warning match should be more exact. Avoid matching nr_warning. So does error. Don't use stderr.* as examples for them.
job['summary.any_warning'] = 1
summary_result = 'warning'
end
# "stderr.linux-perf": 1,
# "stderr.error:target_not_found:ruby-dev": 1,
# "stderr.error:could_not_open_file/var/lib/pacman/local/ldb-#:#-#/desc:Not_a_directory": 1,
if stat.match(/stderr\./i)
job['summary.any_stderr'] = 1
summary_result = 'stderr'
end
# "stderr.::Proceed_with_installation?[Y/n]error:target_not_found:liblzma-dev": 1,
# "stderr.==>ERROR:Failure_while_downloading_apache-cassandra": 1,
# "stderr.ftq.h:#:#:error:unknown_type_name'ticks'": 1,
# "last_state.test.iperf.exit_code.127": 1,
# "last_state.test.cci-makepkg.exit_code.1": 1,
if stat.match(/error|\.exit_code\./i)
job['summary.any_error'] = 1
summary_result = 'error'
end
Thanks, Fengguang
if stat.match(/\.fail$/i)
job['summary.any_fail'] = 1
summary_result = 'fail' end
next unless job['stats']
end@result[field] << job['stats'][field] if job['stats'][field]
- end
- def deal_jobs
- stats_count = Hash.new(0)
- stats_jobs = {}
- return unless summary_result.empty?
- @jobs.each do |job_id, job|
level = deal_stats(job)
add_result_fields(job, level)
- job['summary.success'] = 1
- end
stat_key = @stats_level[level]
stat_jobs_key = stat_key + '_jobs'
- # set jobs summary fields information in place
- def set_jobs_summary
- @jobs.each do |job|
stats = job['stats']
next unless stats
stats_count[stat_key] += 1
stats_jobs[stat_jobs_key] ||= []
stats_jobs[stat_jobs_key] << job_id
endset_job_summary(stats, job)
@stats['stats.count'].merge!(stats_count)
@stats['stats.count'].merge!(stats_jobs) end
def deal_stats(job, level = 0)
return 1 unless job['stats']
- def get_all_metrics(jobs)
- metrics = []
- jobs.each do |job|
stats = job['stats']
next unless stats
- job['stats'].each do |key, value|
match_stats_filter(key, value, job['id'])
calculate_stat(key, value)
level = get_stat_level(key, level)
endmetrics.concat(stats.keys)
- return level
- metrics.uniq! end
- def match_stats_filter(key, value, job_id)
- @stats_filter.each do |filter|
next unless key.include?(filter)
- def initialize_result_hash(metrics)
- result = {
'kvcount' => {},
'raw.id' => {},
'sum.stats' => {},
'raw.stats' => {},
'avg.stats' => {},
'max.stats' => {},
'min.stats' => {}
- }
- metrics.each { |metric| result['raw.stats'][metric] = [] }
- result
- end
key = job_id + '.' + key
@stats_filter_result[key] = value
- def set_default_value(result, stats, metrics)
- left_metrics = metrics - stats.keys
- left_metrics.each { |metric| result['raw.stats'][metric] << nil }
break
- stats.each do |key, value|
end endresult['raw.stats'][key] << value
- def calculate_stat(key, value)
- if function_stat?(key)
return unless @fields.include?('stats.sum')
- def kvcount(result, job)
- KEYWORD.each do |keyword|
next unless job[keyword]
@stats['stats.sum'][key] += value
- else
return unless @fields.include?('stats.avg')
result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0
result['kvcount']["#{keyword}=#{job[keyword]}"] += 1
result['raw.id']["[#{keyword}=#{job[keyword]}]"] ||= []
result['raw.id']["[#{keyword}=#{job[keyword]}]"] << job['id']
- end
- end
@stats['stats.avg'][key] = (@stats['stats.avg'][key] + value) / 2
- def stats_count(result)
- result['raw.stats'].each do |key, value|
if function_stat?(key)
result['sum.stats'][key] = value.compact.size
else
result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f
result['max.stats'][key] = value.compact.max
result['min.stats'][key] = value.compact.min
end endend
- def get_stat_level(stat, level)
- return level if level >= 3
- return 3 if stat.match(/error|fail/i)
- return 2 if stat.match(/warn/i)
- def query_jobs_state(jobs)
- metrics = get_all_metrics(jobs)
- result = initialize_result_hash(metrics)
- jobs.each do |job|
stats = job['stats']
next unless stats
set_default_value(result, stats, metrics)
kvcount(result, job)
- end
- return 0
- stats_count(result)
- result end
- def output
- result = {
'stats.count' => @stats['stats.count']
- }
- def output_yaml(prefix, result)
- result.each do |key, value|
if prefix.empty?
prefix_key = "#{key}"
else
prefix_key = "#{prefix}.#{key}"
end
- @stats.each do |key, value|
result[key] = value if @fields.include?(key)
if value.is_a? Hash
output_yaml(prefix_key, value)
else
puts "#{prefix_key}: #{value.to_json}"
endend
- end
- @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty?
- @result.merge!(result)
- puts JSON.pretty_generate(@result)
- def output
- @result = query_jobs_state(@jobs)
- @result['kvcount'] = @result['kvcount'].sort.to_h
- @result['raw.id'] = @result['raw.id'].sort.to_h
- output_yaml('', @result) end
end
2.23.0
On Tue, Jan 26, 2021 at 03:23:52PM +0800, Wu Fengguang wrote:
- stats.each_key do |stat|
I'd suggest this:
The fail/error/warning cases should not take input from stderr.* stderr is rather chaotic -- let it stay in any_stderr.
BTW we used blacklist for stderr stats
/c/lkp-tests/etc/ignore-stderr/
which turns out hard to maintain in increasing tests -- can easily add noisy stderr stats that are just informative messages instead of error conditions.
I'd suggest to add another white list, mainly from the comment strings in
/c/linux/include/uapi/asm-generic/errno-base.h #define EPERM 1 /* Operation not permitted */ #define ENOENT 2 /* No such file or directory */ #define ESRCH 3 /* No such process */
to only convert the well known error messages to stats.stderr.*
Thanks, Fengguang