January 2021 - Compass-ci - mailweb.openeuler.org

[PATCH compass-ci] scheduler: change 'templates' to 'vt'
by Ren Wen 15 Jan '21

15 Jan '21

Use 'vt' (virt) instead of 'templates', the latter is not clear enough. Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/scheduler/find_job_boot.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 625c74c..152d549 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -181,7 +181,7 @@ class Sched "kernel_params" => _kernel_params, "result_root" => job.result_root, "LKP_SERVER" => job["LKP_SERVER"]?, - "templates" => job["templates"]?, + "vt" => job["vt"]?, "RESULT_WEBDAV_PORT" => job["RESULT_WEBDAV_PORT"]? || "3080", }.to_json end -- 2.23.0

1 0

[PATCH v4 compass-ci] lib/es_jobs.rb: compact es-jobs output results
by Lu Kaiyi 15 Jan '21

15 Jan '21

[why] Reduce the nesting of results and output yaml [example] es-jobs os=archLinux [output] --- kvcount.arch=aarch64: 43 kvcount.arch=x86_64: 2 kvcount.category=benchmark: 12 kvcount.category=functional: 33 kvcount.job_state=failed: 20 kvcount.job_state=finished: 25 kvcount.os=archLinux: 45 kvcount.suite=cci-depends: 28 kvcount.suite=cci-makepkg: 5 kvcount.suite=iperf: 3 kvcount.suite=iperf-walk-os-test: 9 kvcount.tbox_group=vm-2p16g--wangyong: 1 kvcount.tbox_group=vm-2p16g--wcl: 1 kvcount.tbox_group=vm-2p16g.wangyong: 34 kvcount.tbox_group=vm-2p8g: 9 sum.stats.stderr./lkp/lkp/src/monitors/perf-stat:#:main: 12 sum.stats.stderr.Can_not_find_perf_command: 12 sum.stats.stderr./lkp/lkp/src/tests/wrapper:line#:which:command_not_found: 10 sum.stats.last_state.test.iperf.exit_code.127: 12 ... raw.stats.sched_debug.cfs_rq:/.load.stddev: "[524288.0, nil, nil, nil, nil, nil, nil, nil, nil, 524288.0, 524288.0, nil, nil, nil, nil, nil, 516608.0, nil, 1048576.0, nil, nil, nil, nil, nil, nil, nil, 524288.0, nil, nil, nil, nil, 2104832.0, 1572864.0, nil, nil, 2097152.0, nil, nil, nil, nil, nil, nil, nil, nil, nil]" ... avg.stats.sched_debug.cfs_rq:/.load.stddev: 1048576.0 avg.stats.softirqs.CPU1.NET_RX: 2.5833333333333335 avg.stats.slabinfo.kmalloc-512.active_objs: 1372.75 ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_jobs.rb | 105 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 7 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..90e4302 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -2,7 +2,9 @@ # frozen_string_literal: true LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' +KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo submit_id group_id] +require 'yaml' require "#{LKP_SRC}/lib/stats" require_relative './es_query' @@ -123,17 +125,106 @@ class ESJobs return 0 end - def output + def query_jobs_from_es(items) + es = ESQuery.new(ES_HOST, ES_PORT) + result = es.multi_field_query items + jobs = result['hits']['hits'] + jobs.map! { |job| job['_source'] } + return jobs + end + + def get_all_metrics(jobs) + metrics = [] + jobs.each do |job| + stats = job['stats'] + next unless stats + + metrics.concat(stats.keys) + end + metrics.uniq! + end + + def initialize_result_hash(jobs, metrics) result = { - 'stats.count' => @stats['stats.count'] + 'kvcount' => {}, + 'sum.stats' => {}, + 'raw.stats' => {}, + 'avg.stats' => {}, + 'max.stats' => {}, + 'min.stats' => {} } + metrics.each { |metric| result['raw.stats'][metric] = [] } + result + end + + def set_default_value(result, stats, metrics) + left_metrics = metrics - stats.keys + left_metrics.each { |metric| result['raw.stats'][metric] << nil } + + stats.each do |key, value| + result['raw.stats'][key] << value + end + end + + def kvcount(result, job) + KEYWORD.each do |keyword| + next unless job[keyword] + + result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0 + result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 + end + end + + def stats_count(result) + result['raw.stats'].each do |key, value| + if function_stat?(key) + result['sum.stats'][key] = value.compact.size + else + result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f + result['max.stats'][key] = value.compact.max + result['min.stats'][key] = value.compact.min + end + result['raw.stats'][key] = value.to_s + end + end + + def query_jobs_state(jobs) + metrics = get_all_metrics(jobs) + result = initialize_result_hash(jobs, metrics) + jobs.each do |job| + stats = job['stats'] + next unless stats - @stats.each do |key, value| - result[key] = value if @fields.include?(key) + set_default_value(result, stats, metrics) + kvcount(result, job) end - @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? - @result.merge!(result) - puts JSON.pretty_generate(@result) + stats_count(result) + result + end + + def compact_hash(prefix, result) + result.each do |key, value| + if prefix.empty? + prefix_key = "#{key}" + else + prefix_key = "#{prefix}.#{key}" + end + + if value.is_a? Hash + compact_hash(prefix_key, value) + else + @results[prefix_key] = value + end + end + end + + def output + jobs = query_jobs_from_es(@es_query) + @result = query_jobs_state(jobs) + @result['kvcount'] = @result['kvcount'].to_a.sort.to_h + @results = {} + compact_hash('', @result) + puts @results.to_yaml end end -- 2.23.0

2 1

[PATCH compass-ci] scheduler: boot.libvirt: response 'SRV_HTTP_PORT'
by Ren Wen 15 Jan '21

15 Jan '21

For caller to download libvirt config from '/srv/cci/libvirt-xml'. Signed-off-by: Ren Wen <15991987063(a)163.com> --- container/scheduler/start | 2 ++ src/scheduler/constants.cr | 2 ++ src/scheduler/find_job_boot.cr | 1 + 3 files changed, 5 insertions(+) diff --git a/container/scheduler/start b/container/scheduler/start index 70a3496..d8f10dc 100755 --- a/container/scheduler/start +++ b/container/scheduler/start @@ -25,6 +25,7 @@ names = Set.new %w[ SSHR_PORT SSHR_PORT_BASE SSHR_PORT_LEN + SRV_HTTP_PORT lab ] @@ -35,6 +36,7 @@ defaults['SSHR_PORT_LEN'] ||= 2000 defaults['SCHED_PORT'] ||= '3000' defaults['SCHED_HOST'] ||= '172.17.0.1' defaults['MASTER_FLUENTD_PORT'] ||= '24224' +defaults['SRV_HTTP_PORT'] ||= '11300' env = docker_env(defaults) CCI_REPOS = ENV['CCI_REPOS'] || '/c' diff --git a/src/scheduler/constants.cr b/src/scheduler/constants.cr index 1146b21..377ea72 100644 --- a/src/scheduler/constants.cr +++ b/src/scheduler/constants.cr @@ -29,6 +29,8 @@ LAB_REPO = "lab-z9" SRV_OS = "/srv/os" SRV_INITRD = "/srv/initrd" +SRV_HTTP_PORT = (ENV.has_key?("SRV_HTTP_PORT") ? ENV["SRV_HTTP_PORT"] : "11300") + INITRD_HTTP_PREFIX = "http://#{INITRD_HTTP_HOST}:#{INITRD_HTTP_PORT}" OS_HTTP_PREFIX = "http://#{OS_HTTP_HOST}:#{OS_HTTP_PORT}" SCHED_HTTP_PREFIX = "http://#{SCHED_HOST}:#{SCHED_PORT}" diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 625c74c..3a8be59 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -183,6 +183,7 @@ class Sched "LKP_SERVER" => job["LKP_SERVER"]?, "templates" => job["templates"]?, "RESULT_WEBDAV_PORT" => job["RESULT_WEBDAV_PORT"]? || "3080", + "SRV_HTTP_PORT" => SRV_HTTP_PORT }.to_json end -- 2.23.0

1 0

[PATCH compass-ci 2/3] container - libvirt-helper: build & start script
by Ren Wen 15 Jan '21

15 Jan '21

Mount '/srv/cci/libvirt-xml' and upload config. Use port 11392 by defaut. Signed-off-by: Ren Wen <15991987063(a)163.com> --- container/libvirt-helper/build | 5 ++++ container/libvirt-helper/start | 46 ++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100755 container/libvirt-helper/build create mode 100755 container/libvirt-helper/start diff --git a/container/libvirt-helper/build b/container/libvirt-helper/build new file mode 100755 index 0000000..1d216ea --- /dev/null +++ b/container/libvirt-helper/build @@ -0,0 +1,5 @@ +#!/bin/sh +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +docker build -t libvirt-helper . diff --git a/container/libvirt-helper/start b/container/libvirt-helper/start new file mode 100755 index 0000000..6222eaa --- /dev/null +++ b/container/libvirt-helper/start @@ -0,0 +1,46 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig.rb' + +names = Set.new %w[ + LIBVIRT_HELPER_HOST + LIBVIRT_HELPER_PORT +] + +defaults = relevant_defaults(names) +env = docker_env(defaults) + +DEFAULT_LKP = '/c/lkp-tests' +DEFAULT_CCI = '/c/compass-ci' +DEFAULT_CONFIG_DIR = '/etc/compass-ci/defaults' +LIBVIRT_HELPER_PORT = defaults['LIBVIRT_HELPER_PORT'] || 11392 +LIBVIRT_PATH = '/srv/cci/libvirt-xml' +docker_rm 'libvirt-helper' + +cmd = %w[ + docker run + --name libvirt-helper + --restart=always + -d +] + env + %W[ + -e LIBVIRT_PATH=#{LIBVIRT_PATH} + -e LIBVIRT_HELPER_PORT=#{LIBVIRT_HELPER_PORT} + -e LKP_SRC=#{DEFAULT_LKP} + -e CCI_SRC=#{DEFAULT_CCI} + -v #{ENV['LKP_SRC']}:#{DEFAULT_LKP} + -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} + -v #{DEFAULT_CONFIG_DIR}:#{DEFAULT_CONFIG_DIR}:ro + -p #{LIBVIRT_HELPER_PORT}:#{LIBVIRT_HELPER_PORT} + -v /etc/localtime:/etc/localtime:ro + -v #{LIBVIRT_PATH}:#{LIBVIRT_PATH} + -w #{DEFAULT_CCI}/container/libvirt-helper/ + libvirt-helper +] + +cmd += ['sh', '-c', 'umask 002 && ruby ./libvirt_app.rb'] + +system(*cmd) -- 2.23.0

2 2

[PATCH v3 compass-ci] lib/es_jobs.rb: compact es-jobs output results
by Lu Kaiyi 15 Jan '21

15 Jan '21

[why] Reduce the nesting of results and output yaml [example] es-jobs submit_id=a7c2f144-aa64-4a23-a390-cfe5bca3b430 [output] --- kvcount.suite=iperf: 2 kvcount.os=openeuler: 2 kvcount.arch=aarch64: 2 kvcount.job_state=finished: 2 kvcount.tbox_group=vm-2p16g: 2 kvcount.submit_id=a7c2f144-aa64-4a23-a390-cfe5bca3b430: 2 sum.stats.kmsg.timestamp:last: 2 sum.stats.dmesg.timestamp:last: 2 raw.stats.kmsg.timestamp:last: "[75.256233, 48.990905]" raw.stats.iperf.tcp.receiver.bps: "[34017924155.510155, nil]" raw.stats.dmesg.timestamp:last: "[75.256233, 48.990905]" raw.stats.iperf.tcp.sender.bps: "[34073687935.01113, nil]" raw.stats.iperf.udp.bps: "[nil, 1048573.3083402428]" avg.stats.iperf.tcp.receiver.bps: 34017924155.510155 avg.stats.iperf.tcp.sender.bps: 34073687935.01113 avg.stats.iperf.udp.bps: 1048573.3083402428 max.stats.iperf.tcp.receiver.bps: 34017924155.510155 max.stats.iperf.tcp.sender.bps: 34073687935.01113 max.stats.iperf.udp.bps: 1048573.3083402428 min.stats.iperf.tcp.receiver.bps: 34017924155.510155 min.stats.iperf.tcp.sender.bps: 34073687935.01113 min.stats.iperf.udp.bps: 1048573.3083402428 Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_jobs.rb | 109 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 100 insertions(+), 9 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..31a57a9 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -2,7 +2,9 @@ # frozen_string_literal: true LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' +KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo submit_id group_id] +require 'yaml' require "#{LKP_SRC}/lib/stats" require_relative './es_query' @@ -123,17 +125,106 @@ class ESJobs return 0 end - def output - result = { - 'stats.count' => @stats['stats.count'] - } + def query_jobs_from_es(items) + es = ESQuery.new(ES_HOST, ES_PORT) + result = es.multi_field_query items + jobs = result['hits']['hits'] + jobs.map! { |job| job['_source'] } + return jobs + end + + def get_all_metrics(jobs) + metrics = [] + jobs.each do |job| + stats = job['stats'] + next unless stats + + metrics.concat(stats.keys) + end + metrics.uniq! + end + + def initialize_result_hash(jobs, metrics) + result = {} + result['kvcount'] = {} + result['sum.stats'] = {} + result['raw.stats'] = {} + result['avg.stats'] = {} + result['max.stats'] = {} + result['min.stats'] = {} + metrics.each { |metric| result['raw.stats'][metric] = [] } + result + end + + def set_default_value(result, stats, metrics) + job_metrics = stats.keys + left_metrics = metrics - job_metrics + left_metrics.each { |metric| result['raw.stats'][metric] << nil } + + stats.each do |key, value| + result['raw.stats'][key] << value + end + end + + def kvcount(result, job) + KEYWORD.each do |keyword| + next unless job[keyword] - @stats.each do |key, value| - result[key] = value if @fields.include?(key) + result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0 + result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 end + end - @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? - @result.merge!(result) - puts JSON.pretty_generate(@result) + def stats_count(result) + result['raw.stats'].each do |key, value| + if function_stat?(key) + result['sum.stats'][key] = value.compact.size + else + result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f + result['max.stats'][key] = value.compact.max + result['min.stats'][key] = value.compact.min + end + result['raw.stats'][key] = value.to_s + end + end + + def query_jobs_state(jobs) + metrics = get_all_metrics(jobs) + result = initialize_result_hash(jobs, metrics) + jobs.each do |job| + stats = job['stats'] + next unless stats + + set_default_value(result, stats, metrics) + kvcount(result, job) + end + + stats_count(result) + result + end + + def compact_hash(prefix, result) + result.each do |key, value| + if prefix.empty? + prefix_key = "#{key}" + else + prefix_key = "#{prefix}.#{key}" + end + + if value.is_a? Hash + compact_hash(prefix_key, value) + else + @results[prefix_key] = value + end + end + end + + def output + jobs = query_jobs_from_es(@es_query) + @result = query_jobs_state(jobs) + @results = {} + compact_hash('', @result) + puts @results.to_yaml + #puts JSON.pretty_generate(@results) end end -- 2.23.0

2 1

[PATCH compass-ci] container/delimiter: auto push image to local docker hub
by Cao Xueliang 15 Jan '21

15 Jan '21

Before: Manual push image to local docker hub. After: Auto push image to local docker hub, others can use the latest image after deploy. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/delimiter/build | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/container/delimiter/build b/container/delimiter/build index fc1af95..7d9b437 100755 --- a/container/delimiter/build +++ b/container/delimiter/build @@ -2,4 +2,20 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +. ../defconfig.sh + +load_cci_defaults + docker build -t delimiter . + +push_image() +{ + local local_docker_hub="$DOCKER_REGISTRY_HOST:$DOCKER_REGISTRY_PORT" + local src_tag=delimiter:latest + local dst_tag="$local_docker_hub/$src_tag" + + docker tag "$src_tag" "$dst_tag" + docker push "$dst_tag" +} + +push_image -- 2.23.0

1 1

[PATCH v1 compass-ci] container/delimiter: auto push image to local docker hub
by Cao Xueliang 15 Jan '21

15 Jan '21

Before: Manual push image to local docker hub. After: Auto push image to local docker hub, others can use the latest image after deploy. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/defconfig.sh | 10 ++++++++++ container/delimiter/build | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/container/defconfig.sh b/container/defconfig.sh index 97d924d..4a5f5fa 100755 --- a/container/defconfig.sh +++ b/container/defconfig.sh @@ -31,3 +31,13 @@ set_es_indices() { find $CCI_SRC/sbin/ -name "es-*-mapping.sh" -exec sh {} \; } + +push_image() +{ + local local_docker_hub="$DOCKER_REGISTRY_HOST:$DOCKER_REGISTRY_PORT" + local src_tag=$1 + local dst_tag="$local_docker_hub/$src_tag" + + docker tag "$src_tag" "$dst_tag" + docker push "$dst_tag" +} diff --git a/container/delimiter/build b/container/delimiter/build index fc1af95..f7f53e7 100755 --- a/container/delimiter/build +++ b/container/delimiter/build @@ -2,4 +2,10 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +. ../defconfig.sh + +load_cci_defaults + docker build -t delimiter . + +push_image delimiter:latest -- 2.23.0

1 0

[PATCH compass-ci 1/3] container - libvirt-helper: Dockerfile
by Ren Wen 15 Jan '21

15 Jan '21

Signed-off-by: Ren Wen <15991987063(a)163.com> --- container/libvirt-helper/Dockerfile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 container/libvirt-helper/Dockerfile diff --git a/container/libvirt-helper/Dockerfile b/container/libvirt-helper/Dockerfile new file mode 100644 index 0000000..638ebd4 --- /dev/null +++ b/container/libvirt-helper/Dockerfile @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM alpine:3.11 + +MAINTAINER 15991987063(a)163.com + +RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories && \ + apk update && \ + apk add --no-cache 'ruby-dev' 'ruby-etc' \ + 'g++' 'gcc' 'pcre' 'libevent' 'make' 'git' 'bash' 'grep' 'coreutils' 'curl' 'util-linux' + +RUN umask 002 && \ + echo ':sources: ["http://rubygems.org"]' >> ~/.gemrc && \ + gem install rest-client activesupport json yaml elasticsearch sinatra puma -- 2.23.0

1 0

[PATCH v2 compass-ci] lib/es_jobs.rb: compact es-jobs output results
by Lu Kaiyi 15 Jan '21

15 Jan '21

[why] Reduce the nesting of output results [example] es-jobs submit_id=a7c2f144-aa64-4a23-a390-cfe5bca3b430 [output] {"kvcount.suite=iperf"=>2, "kvcount.os=openeuler"=>2, "kvcount.arch=aarch64"=>2, "kvcount.job_state=finished"=>2, "kvcount.tbox_group=vm-2p16g"=>2, "kvcount.submit_id=a7c2f144-aa64-4a23-a390-cfe5bca3b430"=>2, "sum.stats.kmsg.timestamp:last"=>2, "sum.stats.dmesg.timestamp:last"=>2, "raw.stats.kmsg.timestamp:last"=>[75.256233, 48.990905], "raw.stats.iperf.tcp.receiver.bps"=>[34017924155.510155, nil], "raw.stats.dmesg.timestamp:last"=>[75.256233, 48.990905], "raw.stats.iperf.tcp.sender.bps"=>[34073687935.01113, nil], "raw.stats.iperf.udp.bps"=>[nil, 1048573.3083402428], "avg.stats.iperf.tcp.receiver.bps"=>34017924155.510155, "avg.stats.iperf.tcp.sender.bps"=>34073687935.01113, "avg.stats.iperf.udp.bps"=>1048573.3083402428, "max.stats.iperf.tcp.receiver.bps"=>34017924155.510155, "max.stats.iperf.tcp.sender.bps"=>34073687935.01113, "max.stats.iperf.udp.bps"=>1048573.3083402428, "min.stats.iperf.tcp.receiver.bps"=>34017924155.510155, "min.stats.iperf.tcp.sender.bps"=>34073687935.01113, "min.stats.iperf.udp.bps"=>1048573.3083402428} Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_jobs.rb | 115 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 102 insertions(+), 13 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..754578d 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' +KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo submit_id group_id] require "#{LKP_SRC}/lib/stats" require_relative './es_query' @@ -18,10 +19,10 @@ class ESJobs @refine_jobs = [] @jobs = {} @stats_level = { - 0 => 'stats.success', - 1 => 'stats.unknown', - 2 => 'stats.warning', - 3 => 'stats.has_error' + 0 => 'success', + 1 => 'unknown', + 2 => 'warning', + 3 => 'has_error' } set_defaults deal_jobs @@ -123,17 +124,105 @@ class ESJobs return 0 end - def output - result = { - 'stats.count' => @stats['stats.count'] - } + def query_jobs_from_es(items) + es = ESQuery.new(ES_HOST, ES_PORT) + result = es.multi_field_query items + jobs = result['hits']['hits'] + jobs.map! { |job| job['_source'] } + return jobs + end + + def get_all_metrics(jobs) + metrics = [] + jobs.each do |job| + stats = job['stats'] + next unless stats + + metrics.concat(stats.keys) + end + metrics.uniq! + end + + def initialize_result_hash(jobs, metrics) + result = {} + result['kvcount'] = {} + result['sum.stats'] = {} + result['raw.stats'] = {} + result['avg.stats'] = {} + result['max.stats'] = {} + result['min.stats'] = {} + metrics.each { |metric| result['raw.stats'][metric] = [] } + result + end + + def set_default_value(result, stats, metrics) + job_metrics = stats.keys + left_metrics = metrics - job_metrics + left_metrics.each { |metric| result['raw.stats'][metric] << nil } + + stats.each do |key, value| + result['raw.stats'][key] << value + end + end + + def kvcount(result, job) + KEYWORD.each do |keyword| + next unless job[keyword] - @stats.each do |key, value| - result[key] = value if @fields.include?(key) + result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0 + result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 end + end - @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? - @result.merge!(result) - puts JSON.pretty_generate(@result) + def stats_count(result) + result['raw.stats'].each do |key, value| + if function_stat?(key) + result['sum.stats'][key] = value.compact.size + else + result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f + result['max.stats'][key] = value.compact.max + result['min.stats'][key] = value.compact.min + end + end + end + + def query_jobs_state(jobs) + metrics = get_all_metrics(jobs) + result = initialize_result_hash(jobs, metrics) + jobs.each do |job| + stats = job['stats'] + next unless stats + + set_default_value(result, stats, metrics) + kvcount(result, job) + end + + stats_count(result) + result + end + + def compact_hash(prefix, result) + result.each do |key, value| + if prefix.empty? + prefix_key = "#{key}" + else + prefix_key = "#{prefix}.#{key}" + end + + if value.is_a? Hash + compact_hash(prefix_key, value) + else + @results[prefix_key] = value + end + end + end + + def output + jobs = query_jobs_from_es(@es_query) + @result = query_jobs_state(jobs) + @results = {} + compact_hash('', @result) + pp @results + #puts JSON.pretty_generate(@results) end end -- 2.23.0

2 1

[PATCH compass-ci] sbin/run_job: kill process after running qemu/docker
by Hu Xuejiao 15 Jan '21

15 Jan '21

[why] 1.host-info is adapted in physical machine, so it replaced to ebizzy.yaml 2.after running multi-docker and multi-qemu, these processes need to be deleted Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sbin/run_job | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sbin/run_job b/sbin/run_job index 325fa9f..24d35cc 100755 --- a/sbin/run_job +++ b/sbin/run_job @@ -24,7 +24,7 @@ submit_one_job() submit_job() { - yaml_args=("$cci_makepkg_file" "host-info.yaml") + yaml_args=("$cci_makepkg_file" "ebizzy.yaml") for yaml in "${yaml_args[@]}" do submit_one_job & @@ -75,3 +75,6 @@ run_docker() run_qemu run_docker wait + +kill $(cat pid) +kill $(cat $CCI_SRC/providers/dc.pid) -- 2.23.0

2 1