[Compass-ci] Re: [PATCH v4 compass-ci] lib/es_jobs.rb: compact es-jobs output results

On Fri, Jan 15, 2021 at 12:21:41PM +0800, Lu Kaiyi wrote:

[why] Reduce the nesting of results and output yaml

[example] es-jobs os=archLinux

[output]

---

上面一行去掉

kvcount.arch=aarch64: 43 kvcount.arch=x86_64: 2 kvcount.category=benchmark: 12 kvcount.category=functional: 33 kvcount.job_state=failed: 20 kvcount.job_state=finished: 25 kvcount.os=archLinux: 45 kvcount.suite=cci-depends: 28 kvcount.suite=cci-makepkg: 5 kvcount.suite=iperf: 3 kvcount.suite=iperf-walk-os-test: 9 kvcount.tbox_group=vm-2p16g--wangyong: 1 kvcount.tbox_group=vm-2p16g--wcl: 1 kvcount.tbox_group=vm-2p16g.wangyong: 34 kvcount.tbox_group=vm-2p8g: 9 sum.stats.stderr./lkp/lkp/src/monitors/perf-stat:#:main: 12 sum.stats.stderr.Can_not_find_perf_command: 12 sum.stats.stderr./lkp/lkp/src/tests/wrapper:line#:which:command_not_found: 10 sum.stats.last_state.test.iperf.exit_code.127: 12 ... raw.stats.sched_debug.cfs_rq:/.load.stddev: "[524288.0, nil, nil, nil, nil, nil, nil, nil, nil, 524288.0, 524288.0, nil, nil, nil, nil, nil, 516608.0, nil, 1048576.0, nil, nil, nil, nil, nil, nil, nil, 524288.0, nil, nil, nil, nil, 2104832.0, 1572864.0, nil, nil, 2097152.0, nil, nil, nil, nil, nil, nil, nil, nil, nil]"

上面的值是字符串，不是数组。

Thanks, Fengguang

... avg.stats.sched_debug.cfs_rq:/.load.stddev: 1048576.0 avg.stats.softirqs.CPU1.NET_RX: 2.5833333333333335 avg.stats.slabinfo.kmalloc-512.active_objs: 1372.75 ...

Signed-off-by: Lu Kaiyi 2392863668@qq.com

lib/es_jobs.rb | 105 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 7 deletions(-)

diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..90e4302 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -2,7 +2,9 @@ # frozen_string_literal: true

LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' +KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo submit_id group_id]

+require 'yaml' require "#{LKP_SRC}/lib/stats" require_relative './es_query'

@@ -123,17 +125,106 @@ class ESJobs return 0 end

• def output

• def query_jobs_from_es(items)
• es = ESQuery.new(ES_HOST, ES_PORT)
• result = es.multi_field_query items
• jobs = result['hits']['hits']
• jobs.map! { |job| job['_source'] }
• return jobs
• end
• def get_all_metrics(jobs)
• metrics = []
• jobs.each do |job|

•  stats = job['stats']
  

•  next unless stats
  

•  metrics.concat(stats.keys)
  

• end
• metrics.uniq!
• end
• def initialize_result_hash(jobs, metrics) result = {

•  'stats.count' => @stats['stats.count']
  

•  'kvcount' => {},
  

•  'sum.stats' => {},
  

•  'raw.stats' => {},
  

•  'avg.stats' => {},
  

•  'max.stats' => {},
  

•  'min.stats' => {}
  

  }
• metrics.each { |metric| result['raw.stats'][metric] = [] }
• result
• end
• def set_default_value(result, stats, metrics)
• left_metrics = metrics - stats.keys
• left_metrics.each { |metric| result['raw.stats'][metric] << nil }
• stats.each do |key, value|

•  result['raw.stats'][key] << value
  

• end
• end
• def kvcount(result, job)
• KEYWORD.each do |keyword|

•  next unless job[keyword]
  

•  result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0
  

•  result['kvcount']["#{keyword}=#{job[keyword]}"] += 1
  

• end
• end
• def stats_count(result)
• result['raw.stats'].each do |key, value|

•  if function_stat?(key)
  

•    result['sum.stats'][key] = value.compact.size
  

•  else
  

•    result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f
  

•    result['max.stats'][key] = value.compact.max
  

•    result['min.stats'][key] = value.compact.min
  

•  end
  

•  result['raw.stats'][key] = value.to_s
  

• end
• end
• def query_jobs_state(jobs)
• metrics = get_all_metrics(jobs)
• result = initialize_result_hash(jobs, metrics)
• jobs.each do |job|

•  stats = job['stats']
  

•  next unless stats
  
  

• @stats.each do |key, value|

•  result[key] = value if @fields.include?(key)
  

•  set_default_value(result, stats, metrics)
  

•  kvcount(result, job)
  

  end

• @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty?
• @result.merge!(result)
• puts JSON.pretty_generate(@result)

• stats_count(result)
• result
• end
• def compact_hash(prefix, result)
• result.each do |key, value|

•  if prefix.empty?
  

•    prefix_key = "#{key}"
  

•  else
  

•    prefix_key = "#{prefix}.#{key}"
  

•  end
  

•  if value.is_a? Hash
  

•    compact_hash(prefix_key, value)
  

•  else
  

•    @results[prefix_key] = value
  

•  end
  

• end
• end
• def output
• jobs = query_jobs_from_es(@es_query)
• @result = query_jobs_state(jobs)
• @result['kvcount'] = @result['kvcount'].to_a.sort.to_h
• @results = {}
• compact_hash('', @result)
• puts @results.to_yaml end

end

2.23.0