[Compass-ci] Re: [PATCH v6 compass-ci] lib/es_jobs.rb: compact es-jobs output results

raw.stats.sched_debug.cfs_rq:/.load.stddev: [524288.0, nil, nil, nil, nil, nil, nil, nil, nil, 524288.0, 524288.0, nil, nil, nil, nil, nil, 516608.0, nil, 1048576.0, nil, nil, nil, nil, nil, nil, nil, 524288.0, nil, nil, nil, nil, 2104832.0, 1572864.0, nil, nil, 2097152.0, nil, nil, nil, nil, nil, nil, nil, nil, nil]

nil -> null

Thanks, RenWen

... avg.stats.sched_debug.cfs_rq:/.load.stddev: 1048576.0 avg.stats.softirqs.CPU1.NET_RX: 2.5833333333333335 avg.stats.slabinfo.kmalloc-512.active_objs: 1372.75 ...

Signed-off-by: Lu Kaiyi 2392863668@qq.com

lib/es_jobs.rb | 97 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 7 deletions(-)

diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..a36a241 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -2,7 +2,9 @@ # frozen_string_literal: true

LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' +KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo submit_id group_id]

+require 'yaml' require "#{LKP_SRC}/lib/stats" require_relative './es_query'

@@ -123,17 +125,98 @@ class ESJobs return 0 end

• def output

• def get_all_metrics(jobs)
• metrics = []
• jobs.each do |job|

•  stats = job['stats']
  

•  next unless stats
  

•  metrics.concat(stats.keys)
  

• end
• metrics.uniq!
• end
• def initialize_result_hash(metrics) result = {

•  'stats.count' => @stats['stats.count']
  

•  'kvcount' => {},
  

•  'sum.stats' => {},
  

•  'raw.stats' => {},
  

•  'avg.stats' => {},
  

•  'max.stats' => {},
  

•  'min.stats' => {}
  

  }
• metrics.each { |metric| result['raw.stats'][metric] = [] }
• result
• end
• def set_default_value(result, stats, metrics)
• left_metrics = metrics - stats.keys
• left_metrics.each { |metric| result['raw.stats'][metric] << nil }
• stats.each do |key, value|

•  result['raw.stats'][key] << value
  

• end
• end
• def kvcount(result, job)
• KEYWORD.each do |keyword|

•  next unless job[keyword]
  

•  result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0
  

•  result['kvcount']["#{keyword}=#{job[keyword]}"] += 1
  

• end
• end

• @stats.each do |key, value|

•  result[key] = value if @fields.include?(key)
  

• def stats_count(result)
• result['raw.stats'].each do |key, value|

•  if function_stat?(key)
  

•    result['sum.stats'][key] = value.compact.size
  

•  else
  

•    result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f
  

•    result['max.stats'][key] = value.compact.max
  

•    result['min.stats'][key] = value.compact.min
  

•  end
  

•  result['raw.stats'][key] = '---' + value.inspect + '---'
  

• end
• end
• def query_jobs_state(jobs)
• metrics = get_all_metrics(jobs)
• result = initialize_result_hash(metrics)
• jobs.each do |job|

•  stats = job['stats']
  

•  next unless stats
  

•  set_default_value(result, stats, metrics)
  

•  kvcount(result, job)
  

• end
• stats_count(result)
• result
• end
• def compact_hash(prefix, result)
• result.each do |key, value|

•  if prefix.empty?
  

•    prefix_key = "#{key}"
  

•  else
  

•    prefix_key = "#{prefix}.#{key}"
  

•  end
  

•  if value.is_a? Hash
  

•    compact_hash(prefix_key, value)
  

•  else
  

•    @results[prefix_key] = value
  

•  end
  

  end
• end

• @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty?
• @result.merge!(result)
• puts JSON.pretty_generate(@result)

• def output
• @result = query_jobs_state(@jobs.values)
• @result['kvcount'] = @result['kvcount'].to_a.sort.to_h
• @results = {}
• compact_hash('', @result)
• yaml = @results.to_yaml.delete_prefix!("---\n")
• puts yaml.gsub('"---', '').gsub('---"', '') end

end

2.23.0