[why] Reduce the nesting of results and output yaml
[example] es-jobs os=archLinux
[output]
--- kvcount.arch=aarch64: 43 kvcount.arch=x86_64: 2 kvcount.category=benchmark: 12 kvcount.category=functional: 33 kvcount.job_state=failed: 20 kvcount.job_state=finished: 25 kvcount.os=archLinux: 45 kvcount.suite=cci-depends: 28 kvcount.suite=cci-makepkg: 5 kvcount.suite=iperf: 3 kvcount.suite=iperf-walk-os-test: 9 kvcount.tbox_group=vm-2p16g--wangyong: 1 kvcount.tbox_group=vm-2p16g--wcl: 1 kvcount.tbox_group=vm-2p16g.wangyong: 34 kvcount.tbox_group=vm-2p8g: 9 sum.stats.stderr./lkp/lkp/src/monitors/perf-stat:#:main: 12 sum.stats.stderr.Can_not_find_perf_command: 12 sum.stats.stderr./lkp/lkp/src/tests/wrapper:line#:which:command_not_found: 10 sum.stats.last_state.test.iperf.exit_code.127: 12 ... raw.stats.sched_debug.cfs_rq:/.load.stddev: "[524288.0, nil, nil, nil, nil, nil, nil, nil, nil, 524288.0, 524288.0, nil, nil, nil, nil, nil, 516608.0, nil, 1048576.0, nil, nil, nil, nil, nil, nil, nil, 524288.0, nil, nil, nil, nil, 2104832.0, 1572864.0, nil, nil, 2097152.0, nil, nil, nil, nil, nil, nil, nil, nil, nil]" ... avg.stats.sched_debug.cfs_rq:/.load.stddev: 1048576.0 avg.stats.softirqs.CPU1.NET_RX: 2.5833333333333335 avg.stats.slabinfo.kmalloc-512.active_objs: 1372.75 ...
Signed-off-by: Lu Kaiyi 2392863668@qq.com --- lib/es_jobs.rb | 105 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 7 deletions(-)
diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..90e4302 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -2,7 +2,9 @@ # frozen_string_literal: true
LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' +KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo submit_id group_id]
+require 'yaml' require "#{LKP_SRC}/lib/stats" require_relative './es_query'
@@ -123,17 +125,106 @@ class ESJobs return 0 end
- def output + def query_jobs_from_es(items) + es = ESQuery.new(ES_HOST, ES_PORT) + result = es.multi_field_query items + jobs = result['hits']['hits'] + jobs.map! { |job| job['_source'] } + return jobs + end + + def get_all_metrics(jobs) + metrics = [] + jobs.each do |job| + stats = job['stats'] + next unless stats + + metrics.concat(stats.keys) + end + metrics.uniq! + end + + def initialize_result_hash(jobs, metrics) result = { - 'stats.count' => @stats['stats.count'] + 'kvcount' => {}, + 'sum.stats' => {}, + 'raw.stats' => {}, + 'avg.stats' => {}, + 'max.stats' => {}, + 'min.stats' => {} } + metrics.each { |metric| result['raw.stats'][metric] = [] } + result + end + + def set_default_value(result, stats, metrics) + left_metrics = metrics - stats.keys + left_metrics.each { |metric| result['raw.stats'][metric] << nil } + + stats.each do |key, value| + result['raw.stats'][key] << value + end + end + + def kvcount(result, job) + KEYWORD.each do |keyword| + next unless job[keyword] + + result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0 + result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 + end + end + + def stats_count(result) + result['raw.stats'].each do |key, value| + if function_stat?(key) + result['sum.stats'][key] = value.compact.size + else + result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f + result['max.stats'][key] = value.compact.max + result['min.stats'][key] = value.compact.min + end + result['raw.stats'][key] = value.to_s + end + end + + def query_jobs_state(jobs) + metrics = get_all_metrics(jobs) + result = initialize_result_hash(jobs, metrics) + jobs.each do |job| + stats = job['stats'] + next unless stats
- @stats.each do |key, value| - result[key] = value if @fields.include?(key) + set_default_value(result, stats, metrics) + kvcount(result, job) end
- @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? - @result.merge!(result) - puts JSON.pretty_generate(@result) + stats_count(result) + result + end + + def compact_hash(prefix, result) + result.each do |key, value| + if prefix.empty? + prefix_key = "#{key}" + else + prefix_key = "#{prefix}.#{key}" + end + + if value.is_a? Hash + compact_hash(prefix_key, value) + else + @results[prefix_key] = value + end + end + end + + def output + jobs = query_jobs_from_es(@es_query) + @result = query_jobs_state(jobs) + @result['kvcount'] = @result['kvcount'].to_a.sort.to_h + @results = {} + compact_hash('', @result) + puts @results.to_yaml end end
On Fri, Jan 15, 2021 at 12:21:41PM +0800, Lu Kaiyi wrote:
[why] Reduce the nesting of results and output yaml
[example] es-jobs os=archLinux
[output]
上面一行去掉
kvcount.arch=aarch64: 43 kvcount.arch=x86_64: 2 kvcount.category=benchmark: 12 kvcount.category=functional: 33 kvcount.job_state=failed: 20 kvcount.job_state=finished: 25 kvcount.os=archLinux: 45 kvcount.suite=cci-depends: 28 kvcount.suite=cci-makepkg: 5 kvcount.suite=iperf: 3 kvcount.suite=iperf-walk-os-test: 9 kvcount.tbox_group=vm-2p16g--wangyong: 1 kvcount.tbox_group=vm-2p16g--wcl: 1 kvcount.tbox_group=vm-2p16g.wangyong: 34 kvcount.tbox_group=vm-2p8g: 9 sum.stats.stderr./lkp/lkp/src/monitors/perf-stat:#:main: 12 sum.stats.stderr.Can_not_find_perf_command: 12 sum.stats.stderr./lkp/lkp/src/tests/wrapper:line#:which:command_not_found: 10 sum.stats.last_state.test.iperf.exit_code.127: 12 ... raw.stats.sched_debug.cfs_rq:/.load.stddev: "[524288.0, nil, nil, nil, nil, nil, nil, nil, nil, 524288.0, 524288.0, nil, nil, nil, nil, nil, 516608.0, nil, 1048576.0, nil, nil, nil, nil, nil, nil, nil, 524288.0, nil, nil, nil, nil, 2104832.0, 1572864.0, nil, nil, 2097152.0, nil, nil, nil, nil, nil, nil, nil, nil, nil]"
上面的值是字符串,不是数组。
Thanks, Fengguang
... avg.stats.sched_debug.cfs_rq:/.load.stddev: 1048576.0 avg.stats.softirqs.CPU1.NET_RX: 2.5833333333333335 avg.stats.slabinfo.kmalloc-512.active_objs: 1372.75 ...
Signed-off-by: Lu Kaiyi 2392863668@qq.com
lib/es_jobs.rb | 105 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 7 deletions(-)
diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..90e4302 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -2,7 +2,9 @@ # frozen_string_literal: true
LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' +KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo submit_id group_id]
+require 'yaml' require "#{LKP_SRC}/lib/stats" require_relative './es_query'
@@ -123,17 +125,106 @@ class ESJobs return 0 end
- def output
- def query_jobs_from_es(items)
- es = ESQuery.new(ES_HOST, ES_PORT)
- result = es.multi_field_query items
- jobs = result['hits']['hits']
- jobs.map! { |job| job['_source'] }
- return jobs
- end
- def get_all_metrics(jobs)
- metrics = []
- jobs.each do |job|
stats = job['stats']
next unless stats
metrics.concat(stats.keys)
- end
- metrics.uniq!
- end
- def initialize_result_hash(jobs, metrics) result = {
'stats.count' => @stats['stats.count']
'kvcount' => {},
'sum.stats' => {},
'raw.stats' => {},
'avg.stats' => {},
'max.stats' => {},
}'min.stats' => {}
- metrics.each { |metric| result['raw.stats'][metric] = [] }
- result
- end
- def set_default_value(result, stats, metrics)
- left_metrics = metrics - stats.keys
- left_metrics.each { |metric| result['raw.stats'][metric] << nil }
- stats.each do |key, value|
result['raw.stats'][key] << value
- end
- end
- def kvcount(result, job)
- KEYWORD.each do |keyword|
next unless job[keyword]
result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0
result['kvcount']["#{keyword}=#{job[keyword]}"] += 1
- end
- end
- def stats_count(result)
- result['raw.stats'].each do |key, value|
if function_stat?(key)
result['sum.stats'][key] = value.compact.size
else
result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f
result['max.stats'][key] = value.compact.max
result['min.stats'][key] = value.compact.min
end
result['raw.stats'][key] = value.to_s
- end
- end
- def query_jobs_state(jobs)
- metrics = get_all_metrics(jobs)
- result = initialize_result_hash(jobs, metrics)
- jobs.each do |job|
stats = job['stats']
next unless stats
- @stats.each do |key, value|
result[key] = value if @fields.include?(key)
set_default_value(result, stats, metrics)
endkvcount(result, job)
- @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty?
- @result.merge!(result)
- puts JSON.pretty_generate(@result)
- stats_count(result)
- result
- end
- def compact_hash(prefix, result)
- result.each do |key, value|
if prefix.empty?
prefix_key = "#{key}"
else
prefix_key = "#{prefix}.#{key}"
end
if value.is_a? Hash
compact_hash(prefix_key, value)
else
@results[prefix_key] = value
end
- end
- end
- def output
- jobs = query_jobs_from_es(@es_query)
- @result = query_jobs_state(jobs)
- @result['kvcount'] = @result['kvcount'].to_a.sort.to_h
- @results = {}
- compact_hash('', @result)
- puts @results.to_yaml end
end
2.23.0