[Compass-ci] Re: [PATCH compass-ci] tool/es-jobs: add a new tool

On Tue, Jan 12, 2021 at 07:56:32PM +0800, Wu Zhende wrote:

Use es-jobs to summarizes information about a group of related jobs.

[Example] es-jobs submit_id=xxx -f 'id,job_state,dmesg.timestamp:last,stats.sum,stats.avg' -r '0' means: 1.query the job whose submit_id is xxx

Add space between 1. and query, the same as follows.

Thanks, Xijian

2.I want to get id,job_state,dmesg.timestamp:last,stats.sum,stats.avg 3.refine stats.success jobs [Output] { "id": [ "crystal.597090", "crystal.597093" ], "job_state": [ "finished", "finished" ], "dmesg.timestamp:last": [ 75.256233, 48.990905 ], "stats.sum": { "kmsg.timestamp:last": 124.24713799999999, "dmesg.timestamp:last": 124.24713799999999 }, "stats.avg": { "iperf.tcp.receiver.bps": 17008962077.755077, "iperf.tcp.sender.bps": 17036843967.505566, "iperf.udp.bps": 524286.6541701214 }, "stats.count": { "stats.success": 2, "stats.success_jobs": [ "crystal.597090", "crystal.597093" ] } }

Signed-off-by: Wu Zhende wuzhende666@163.com

lib/es_jobs.rb | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++ sbin/es-jobs | 53 +++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 lib/es_jobs.rb create mode 100755 sbin/es-jobs

diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb new file mode 100644 index 0000000..78db8f0 --- /dev/null +++ b/lib/es_jobs.rb @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: GPL-2.0-only +# frozen_string_literal: true

+LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests'

+require "#{LKP_SRC}/lib/stats" +require_relative './es_query'

+# deal jobs search from es +class ESJobs

• def initialize(es_query, my_refine = [-1], fields = [])
• @es = ESQuery.new(ES_HOST, ES_PORT)
• @refine = my_refine
• @fields = fields
• @refine_jobs = []
• @query_result = @es.multi_field_query(es_query)
• @jobs = {}
• @stats_count = Hash.new(0)
• @stats_sum = Hash.new(0)
• @stats_avg = Hash.new(0)
• @stats_level = {

•  0 => 'stats.success',
  

•  1 => 'stats.unknow',
  

•  2 => 'stats.warning',
  

•  3 => 'stats.has_error'
  

• }
• set_defaults
• deal_jobs
• end
• def set_defaults
• @query_result['hits']['hits'].each do |job|

•  @jobs[job['_id']] = job['_source']
  

• end
• @result = {}
• @fields.each do |field|

•  @result[field] = []
  

• end
• end
• def add_result_fields(job, level)
• return unless @refine.include?(level) || @refine.include?(-1)
• @refine_jobs << job['id']
• @fields.each do |field|

•  @result[field] << job[field] if job[field]
  

•  next unless job['stats']
  

•  @result[field] << job['stats'][field] if job['stats'][field]
  

• end
• end
• def deal_jobs
• @jobs.each do |job_id, job|

•  level = deal_stats(job)
  

•  add_result_fields(job, level)
  

•  @stats_count[@stats_level[level]] += 1
  

•  stat_key = @stats_level[level] + '_jobs'
  

•  @stats_count[stat_key] = [] unless @stats_count.key?(stat_key)
  

•  @stats_count[stat_key] << job_id
  

• end
• end
• def deal_stats(job, level = 0)
• return 1 unless job['stats']
• job['stats'].each do |key, value|

•  if function_stat?(key)
  

•    @stats_sum[key] += value
  

•  else
  

•    @stats_avg[key] = (@stats_avg[key] + value) / 2
  

•  end
  

•  level = get_stat_level(key, level)
  

• end
• return level
• end
• def get_stat_level(stat, level)
• return level if level >= 3
• return 3 if stat.match(/error|fail/i)
• return 2 if stat.match(/warn/i)
• return 0
• end
• def output
• result = {

•  'stats.count' => @stats_count
  

• }
• result['stats.sum'] = @stats_sum if @fields.include?('stats.sum')
• result['stats.avg'] = @stats_avg if @fields.include?('stats.avg')
• @result.merge!(result)
• puts JSON.pretty_generate(@result)
• end

+end diff --git a/sbin/es-jobs b/sbin/es-jobs new file mode 100755 index 0000000..f075ef2 --- /dev/null +++ b/sbin/es-jobs @@ -0,0 +1,53 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true

+require 'optparse' +require 'json'

+require_relative '../lib/es_jobs'

+def parse_argv

• items = {}
• ARGV.each do |item|
• key, value = item.split('=')
• if key && value

•  value_list = value.split(',')
  

•  items[key] = value_list.length > 1 ? value_list : value
  

• end
• end
• items

+end +opt_refine = [-1] +opt_fields = [] +opt_parser = OptionParser.new do |opts|

• opts.banner = 'Usage: es-jobs [options] search_key1=val1[,val2..] ..'
• opts.separator 'search_key can be id, suite, os, etc.'
• opts.separator 'How to use -r'
• opts.separator 'Like es-jobs submit_id=xxx -r "-1,0,1,2,3"'
• opts.separator '-1 means not refine'
• opts.separator '{
• -1 => null,
• 0 => stats.succes,
• 1 => stats.unknow,
• 2 => stats.warning,
• 3 => stats.has_error

+}'

• opts.on('-r fields', '--refine fields', 'refine jobs') do |fields|
• opt_refine = fields.split(',').map(&:to_i)
• end
• opts.on('-f fields', '--fields fields', 'fields you want to see') do |fields|
• opt_fields = fields.split(',')
• end

+end

+opt_parser.parse!(ARGV) +items = parse_argv +raise 'Please enter a query' if items.empty?

+es_jobs = ESJobs.new(items, opt_refine, opt_fields)

+es_jobs.output

2.23.0