Use es-jobs to summarizes information about a group of related jobs.
[Example]
es-jobs submit_id=xxx -f 'job_state,dmesg.timestamp:last' -r '-1' -s 'on_fail'
means:
1. query the job whose submit_id is xxx
2. I want to get job_state,dmesg.timestamp:last
3. don't refine jobs
4. -s 'on_fail' to get data in stats which contain 'on_fail'
[Output]
{
"job_state": [
"crystal.605716.failed"
],
"dmesg.timestamp:last": [
657.181408
],
"stats_filter_result": {
"crystal.605716.openeuler_docker.\u001b[0mwordpress_build_on_fail": 1
},
"stats.count": {
"stats.has_error": 1,
"stats.has_error_jobs": [
"crystal.605716"
]
}
}
Signed-off-by: Wu Zhende <wuzhende666(a)163.com>
---
lib/es_jobs.rb | 139 +++++++++++++++++++++++++++++++++++++++++++++++++
sbin/es-jobs | 58 +++++++++++++++++++++
2 files changed, 197 insertions(+)
create mode 100644 lib/es_jobs.rb
create mode 100755 sbin/es-jobs
diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb
new file mode 100644
index 0000000..a90b1ba
--- /dev/null
+++ b/lib/es_jobs.rb
@@ -0,0 +1,139 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# frozen_string_literal: true
+
+LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests'
+
+require "#{LKP_SRC}/lib/stats"
+require_relative './es_query'
+
+# deal jobs search from es
+class ESJobs
+ def initialize(es_query, my_refine = [], fields = [], stats_filter = [])
+ @es_query = es_query
+ @es = ESQuery.new(ES_HOST, ES_PORT)
+ @refine = my_refine
+ @fields = fields
+ @stats_filter = stats_filter
+ @stats_filter_result = {}
+ @refine_jobs = []
+ @jobs = {}
+ @stats_level = {
+ 0 => 'stats.success',
+ 1 => 'stats.unknown',
+ 2 => 'stats.warning',
+ 3 => 'stats.has_error'
+ }
+ set_defaults
+ deal_jobs
+ end
+
+ def set_defaults
+ query_result = @es.multi_field_query(@es_query)
+ query_result['hits']['hits'].each do |job|
+ @jobs[job['_id']] = job['_source']
+ end
+
+ @stats = {
+ 'stats.count' => Hash.new(0),
+ 'stats.sum' => Hash.new(0),
+ 'stats.avg' => Hash.new(0)
+ }
+ @result = {}
+ @fields.each do |field|
+ @result[field] = []
+ end
+ end
+
+ def add_result_fields(job, level)
+ return unless @refine.include?(level) || @refine.include?(-1)
+
+ @refine_jobs << job['id']
+ @fields.each do |field|
+ value = job[field]
+ if value
+ value = job['id'] + '.' + value if field == 'job_state'
+ @result[field] << value
+ end
+
+ next unless job['stats']
+
+ @result[field] << job['stats'][field] if job['stats'][field]
+ end
+ end
+
+ def deal_jobs
+ stats_count = Hash.new(0)
+ stats_jobs = {}
+
+ @jobs.each do |job_id, job|
+ level = deal_stats(job)
+ add_result_fields(job, level)
+
+ stat_key = @stats_level[level]
+ stat_jobs_key = stat_key + '_jobs'
+
+ stats_count[stat_key] += 1
+ stats_jobs[stat_jobs_key] ||= []
+ stats_jobs[stat_jobs_key] << job_id
+ end
+
+ @stats['stats.count'].merge!(stats_count)
+ @stats['stats.count'].merge!(stats_jobs)
+ end
+
+ def deal_stats(job, level = 0)
+ return 1 unless job['stats']
+
+ job['stats'].each do |key, value|
+ match_stats_filter(key, value, job['id'])
+ calculate_stat(key, value)
+ level = get_stat_level(key, level)
+ end
+ return level
+ end
+
+ def match_stats_filter(key, value, job_id)
+ @stats_filter.each do |filter|
+ next unless key.include?(filter)
+
+ key = job_id + '.' + key
+ @stats_filter_result[key] = value
+
+ break
+ end
+ end
+
+ def calculate_stat(key, value)
+ if function_stat?(key)
+ return unless @fields.include?('stats.sum')
+
+ @stats['stats.sum'][key] += value
+ else
+ return unless @fields.include?('stats.avg')
+
+ @stats['stats.avg'][key] = (@stats['stats.avg'][key] + value) / 2
+ end
+ end
+
+ def get_stat_level(stat, level)
+ return level if level >= 3
+ return 3 if stat.match(/error|fail/i)
+ return 2 if stat.match(/warn/i)
+
+ return 0
+ end
+
+ def output
+ result = {
+ 'stats.count' => @stats['stats.count']
+ }
+
+ @stats.each do |key, value|
+ result[key] = value if @fields.include?(key)
+ end
+
+ @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty?
+ @result.merge!(result)
+ puts JSON.pretty_generate(@result)
+ end
+end
diff --git a/sbin/es-jobs b/sbin/es-jobs
new file mode 100755
index 0000000..25cbe9c
--- /dev/null
+++ b/sbin/es-jobs
@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+# SPDX-License-Identifier: MulanPSL-2.0+
+# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
+# frozen_string_literal: true
+
+require 'optparse'
+require 'json'
+
+require_relative '../lib/es_jobs'
+
+def parse_argv
+ items = {}
+ ARGV.each do |item|
+ key, value = item.split('=')
+ if key && value
+ value_list = value.split(',')
+ items[key] = value_list.length > 1 ? value_list : value
+ end
+ end
+ items
+end
+opt_refine = [-1]
+opt_fields = []
+opt_stats_filter = []
+opt_parser = OptionParser.new do |opts|
+ opts.banner = 'Usage: es-jobs [options] search_key1=val1[,val2..] ..'
+
+ opts.separator 'search_key can be id, suite, os, etc.'
+ opts.separator 'how to use -r'
+ opts.separator 'like es-jobs submit_id=xxx -r "-1,0,1,2,3"'
+ opts.separator '-1 means not refine'
+ opts.separator '{
+ -1 => null,
+ 0 => stats.succes,
+ 1 => stats.unknow,
+ 2 => stats.warning,
+ 3 => stats.has_error
+}'
+
+ opts.on('-r fields', '--refine fields', 'refine jobs') do |fields|
+ opt_refine = fields.split(',').map(&:to_i)
+ end
+
+ opts.on('-f fields', '--fields fields', 'fields you want to see') do |fields|
+ opt_fields = fields.split(',')
+ end
+
+ opts.on('-s fields', '--stats-filter fields', 'return data contains fields') do |fields|
+ opt_stats_filter = fields.split(',')
+ end
+end
+
+opt_parser.parse!(ARGV)
+items = parse_argv
+raise 'Please enter a query' if items.empty?
+
+es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter)
+es_jobs.output
--
2.23.0