Use es-jobs to summarizes information about a group of related jobs.

[Example] es-jobs submit_id=xxx -f 'id,job_state,dmesg.timestamp:last,stats.sum,stats.avg' -r '0' means: 1.query the job whose submit_id is xxx 2.I want to get id,job_state,dmesg.timestamp:last,stats.sum,stats.avg 3.refine stats.success jobs [Output] { "id": [ "crystal.597090", "crystal.597093" ], "job_state": [ "finished", "finished" ], "dmesg.timestamp:last": [ 75.256233, 48.990905 ], "stats.sum": { "kmsg.timestamp:last": 124.24713799999999, "dmesg.timestamp:last": 124.24713799999999 }, "stats.avg": { "iperf.tcp.receiver.bps": 17008962077.755077, "iperf.tcp.sender.bps": 17036843967.505566, "iperf.udp.bps": 524286.6541701214 }, "stats.count": { "stats.success": 2, "stats.success_jobs": [ "crystal.597090", "crystal.597093" ] } }

Signed-off-by: Wu Zhende wuzhende666@163.com --- lib/es_jobs.rb | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++ sbin/es-jobs | 53 +++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 lib/es_jobs.rb create mode 100755 sbin/es-jobs

diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb new file mode 100644 index 0000000..78db8f0 --- /dev/null +++ b/lib/es_jobs.rb @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: GPL-2.0-only +# frozen_string_literal: true + +LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' + +require "#{LKP_SRC}/lib/stats" +require_relative './es_query' + +# deal jobs search from es +class ESJobs + def initialize(es_query, my_refine = [-1], fields = []) + @es = ESQuery.new(ES_HOST, ES_PORT) + @refine = my_refine + @fields = fields + @refine_jobs = [] + @query_result = @es.multi_field_query(es_query) + @jobs = {} + @stats_count = Hash.new(0) + @stats_sum = Hash.new(0) + @stats_avg = Hash.new(0) + @stats_level = { + 0 => 'stats.success', + 1 => 'stats.unknow', + 2 => 'stats.warning', + 3 => 'stats.has_error' + } + set_defaults + deal_jobs + end + + def set_defaults + @query_result['hits']['hits'].each do |job| + @jobs[job['_id']] = job['_source'] + end + + @result = {} + @fields.each do |field| + @result[field] = [] + end + end + + def add_result_fields(job, level) + return unless @refine.include?(level) || @refine.include?(-1) + + @refine_jobs << job['id'] + @fields.each do |field| + @result[field] << job[field] if job[field] + + next unless job['stats'] + + @result[field] << job['stats'][field] if job['stats'][field] + end + end + + def deal_jobs + @jobs.each do |job_id, job| + level = deal_stats(job) + add_result_fields(job, level) + + @stats_count[@stats_level[level]] += 1 + stat_key = @stats_level[level] + '_jobs' + @stats_count[stat_key] = [] unless @stats_count.key?(stat_key) + @stats_count[stat_key] << job_id + end + end + + def deal_stats(job, level = 0) + return 1 unless job['stats'] + + job['stats'].each do |key, value| + if function_stat?(key) + @stats_sum[key] += value + else + @stats_avg[key] = (@stats_avg[key] + value) / 2 + end + + level = get_stat_level(key, level) + end + return level + end + + def get_stat_level(stat, level) + return level if level >= 3 + return 3 if stat.match(/error|fail/i) + return 2 if stat.match(/warn/i) + + return 0 + end + + def output + result = { + 'stats.count' => @stats_count + } + result['stats.sum'] = @stats_sum if @fields.include?('stats.sum') + result['stats.avg'] = @stats_avg if @fields.include?('stats.avg') + @result.merge!(result) + puts JSON.pretty_generate(@result) + end +end diff --git a/sbin/es-jobs b/sbin/es-jobs new file mode 100755 index 0000000..f075ef2 --- /dev/null +++ b/sbin/es-jobs @@ -0,0 +1,53 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'optparse' +require 'json' + +require_relative '../lib/es_jobs' + +def parse_argv + items = {} + ARGV.each do |item| + key, value = item.split('=') + if key && value + value_list = value.split(',') + items[key] = value_list.length > 1 ? value_list : value + end + end + items +end +opt_refine = [-1] +opt_fields = [] +opt_parser = OptionParser.new do |opts| + opts.banner = 'Usage: es-jobs [options] search_key1=val1[,val2..] ..' + + opts.separator 'search_key can be id, suite, os, etc.' + opts.separator 'How to use -r' + opts.separator 'Like es-jobs submit_id=xxx -r "-1,0,1,2,3"' + opts.separator '-1 means not refine' + opts.separator '{ + -1 => null, + 0 => stats.succes, + 1 => stats.unknow, + 2 => stats.warning, + 3 => stats.has_error +}' + + opts.on('-r fields', '--refine fields', 'refine jobs') do |fields| + opt_refine = fields.split(',').map(&:to_i) + end + + opts.on('-f fields', '--fields fields', 'fields you want to see') do |fields| + opt_fields = fields.split(',') + end +end + +opt_parser.parse!(ARGV) +items = parse_argv +raise 'Please enter a query' if items.empty? + +es_jobs = ESJobs.new(items, opt_refine, opt_fields) +es_jobs.output