[Compass-ci] Re: [PATCH v2 compass-ci] tool/es-jobs: a new tool for summarizing jobs information

On Wed, Jan 13, 2021 at 04:16:40PM +0800, Wu Zhende wrote:

Use es-jobs to summarizes information about a group of related jobs.

[Example] es-jobs submit_id=xxx -f 'job_state,dmesg.timestamp:last' -r '-1' -s 'on_fail' means:

1. query the job whose submit_id is xxx
2. I want to get job_state,dmesg.timestamp:last
3. don't refine jobs
4. -s 'on_fail' to get data in stats which contain 'on_fail'

[Output] { "job_state": [ "crystal.605716.failed" ], "dmesg.timestamp:last": [ 657.181408 ], "stats_filter_result": { "crystal.605716.openeuler_docker.\u001b[0mwordpress_build_on_fail": 1 }, "stats.count": { "stats.has_error": 1, "stats.has_error_jobs": [ "crystal.605716" ] } }

Signed-off-by: Wu Zhende wuzhende666@163.com

lib/es_jobs.rb | 139 +++++++++++++++++++++++++++++++++++++++++++++++++ sbin/es-jobs | 58 +++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 lib/es_jobs.rb create mode 100755 sbin/es-jobs

diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb new file mode 100644 index 0000000..a90b1ba --- /dev/null +++ b/lib/es_jobs.rb @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: GPL-2.0-only +# frozen_string_literal: true

miss shebang? #!/usr/bin/env ruby

Thanks, Liushaofei

+LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests'

+require "#{LKP_SRC}/lib/stats" +require_relative './es_query'

+# deal jobs search from es +class ESJobs

• def initialize(es_query, my_refine = [], fields = [], stats_filter = [])
• @es_query = es_query
• @es = ESQuery.new(ES_HOST, ES_PORT)
• @refine = my_refine
• @fields = fields
• @stats_filter = stats_filter
• @stats_filter_result = {}
• @refine_jobs = []
• @jobs = {}
• @stats_level = {

•  0 => 'stats.success',
  

•  1 => 'stats.unknown',
  

•  2 => 'stats.warning',
  

•  3 => 'stats.has_error'
  

• }
• set_defaults
• deal_jobs
• end
• def set_defaults
• query_result = @es.multi_field_query(@es_query)
• query_result['hits']['hits'].each do |job|

•  @jobs[job['_id']] = job['_source']
  

• end
• @stats = {

•  'stats.count' => Hash.new(0),
  

•  'stats.sum' => Hash.new(0),
  

•  'stats.avg' => Hash.new(0)
  

• }
• @result = {}
• @fields.each do |field|

•  @result[field] = []
  

• end
• end
• def add_result_fields(job, level)
• return unless @refine.include?(level) || @refine.include?(-1)
• @refine_jobs << job['id']
• @fields.each do |field|

•  value = job[field]
  

•  if value
  

•    value = job['id'] + '.' + value if field == 'job_state'
  

•    @result[field] << value
  

•  end
  

•  next unless job['stats']
  

•  @result[field] << job['stats'][field] if job['stats'][field]
  

• end
• end
• def deal_jobs
• stats_count = Hash.new(0)
• stats_jobs = {}
• @jobs.each do |job_id, job|

•  level = deal_stats(job)
  

•  add_result_fields(job, level)
  

•  stat_key = @stats_level[level]
  

•  stat_jobs_key = stat_key + '_jobs'
  

•  stats_count[stat_key] += 1
  

•  stats_jobs[stat_jobs_key] ||= []
  

•  stats_jobs[stat_jobs_key] << job_id
  

• end
• @stats['stats.count'].merge!(stats_count)
• @stats['stats.count'].merge!(stats_jobs)
• end
• def deal_stats(job, level = 0)
• return 1 unless job['stats']
• job['stats'].each do |key, value|

•  match_stats_filter(key, value, job['id'])
  

•  calculate_stat(key, value)
  

•  level = get_stat_level(key, level)
  

• end
• return level
• end
• def match_stats_filter(key, value, job_id)
• @stats_filter.each do |filter|

•  next unless key.include?(filter)
  

•  key = job_id + '.' + key
  

•  @stats_filter_result[key] = value
  

•  break
  

• end
• end
• def calculate_stat(key, value)
• if function_stat?(key)

•  return unless @fields.include?('stats.sum')
  

•  @stats['stats.sum'][key] += value
  

• else

•  return unless @fields.include?('stats.avg')
  

•  @stats['stats.avg'][key] = (@stats['stats.avg'][key] + value) / 2
  

• end
• end
• def get_stat_level(stat, level)
• return level if level >= 3
• return 3 if stat.match(/error|fail/i)
• return 2 if stat.match(/warn/i)
• return 0
• end
• def output
• result = {

•  'stats.count' => @stats['stats.count']
  

• }
• @stats.each do |key, value|

•  result[key] = value if @fields.include?(key)
  

• end
• @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty?
• @result.merge!(result)
• puts JSON.pretty_generate(@result)
• end

+end diff --git a/sbin/es-jobs b/sbin/es-jobs new file mode 100755 index 0000000..25cbe9c --- /dev/null +++ b/sbin/es-jobs @@ -0,0 +1,58 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true

+require 'optparse' +require 'json'

+require_relative '../lib/es_jobs'

+def parse_argv

• items = {}
• ARGV.each do |item|
• key, value = item.split('=')
• if key && value

•  value_list = value.split(',')
  

•  items[key] = value_list.length > 1 ? value_list : value
  

• end
• end
• items

+end +opt_refine = [-1] +opt_fields = [] +opt_stats_filter = [] +opt_parser = OptionParser.new do |opts|

• opts.banner = 'Usage: es-jobs [options] search_key1=val1[,val2..] ..'
• opts.separator 'search_key can be id, suite, os, etc.'
• opts.separator 'how to use -r'
• opts.separator 'like es-jobs submit_id=xxx -r "-1,0,1,2,3"'
• opts.separator '-1 means not refine'
• opts.separator '{
• -1 => null,
• 0 => stats.succes,
• 1 => stats.unknow,
• 2 => stats.warning,
• 3 => stats.has_error

+}'

• opts.on('-r fields', '--refine fields', 'refine jobs') do |fields|
• opt_refine = fields.split(',').map(&:to_i)
• end
• opts.on('-f fields', '--fields fields', 'fields you want to see') do |fields|
• opt_fields = fields.split(',')
• end
• opts.on('-s fields', '--stats-filter fields', 'return data contains fields') do |fields|
• opt_stats_filter = fields.split(',')
• end

+end

+opt_parser.parse!(ARGV) +items = parse_argv +raise 'Please enter a query' if items.empty?

+es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter)

+es_jobs.output

2.23.0