- Compass-ci - mailweb.openeuler.org

[PATCH v3 lkp-tests] (job2sh.rb) add on_fail to job.sh
by Hu Xuejiao 04 Nov '20

04 Nov '20

[why] If job.yaml is defined: on_fail: sleep(tests): 300(runtime) [how] By sh_on_fail, it will appear in job.sh: on_fail() { . $LKP_SRC/lib/http.sh . $LKP_SRC/lib/job.sh . $LKP_SRC/lib/env.sh export_top_env run_test $LKP_SRC/tests/wrapper sleep 500 } Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- lib/job2sh.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/job2sh.rb b/lib/job2sh.rb index 6e00b088..7d2c2d5c 100755 --- a/lib/job2sh.rb +++ b/lib/job2sh.rb @@ -367,9 +367,7 @@ class Job2sh < Job @programs = available_programs(:workload_elements) - if !job - job = (@jobx || @job).clone - end + job ||= (@jobx || @job).clone @cur_func = :on_fail -- 2.23.0

2 1

[PATCH v3 lkp-tests] pkg: add PKGBUILD for libmicro
by Bai Jing 04 Nov '20

04 Nov '20

Signed-off-by: Bai Jing <799286817(a)qq.com> --- pkg/libmicro/PKGBUILD | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 pkg/libmicro/PKGBUILD diff --git a/pkg/libmicro/PKGBUILD b/pkg/libmicro/PKGBUILD new file mode 100644 index 00000000..8efc8b19 --- /dev/null +++ b/pkg/libmicro/PKGBUILD @@ -0,0 +1,19 @@ +pkgname=libMicro +pkgver=0.4 +pkgrel=0 +pkgdesc="A portable set of microbenchmarks focused on the syscall and system library interfaces." +arch=('i386' 'x86_64') +url="https://codeload.github.com/redhat-performance/libMicro" +license=('CDDL') +source=("https://codeload.github.com/redhat-performance/libMicro/zip/0.4.0-rh") +md5sums=('SKIP') + +build() { + cd "$srcdir/$pkgname-$pkgver.$pkgrel-rh" + make +} + +package() { + mkdir -p "$pkgdir/lkp/benchmarks/libMicro" + cp -r "$srcdir/$pkgname-$pkgver.$pkgrel-rh/"* "$pkgdir/lkp/benchmarks/libMicro" +} -- 2.23.0

2 1

[PATCH v2 compass-ci 1/2] auto group jobs with user-defined template
by Lu Weitao 04 Nov '20

04 Nov '20

auto group jobs(query form ES) with user-defined template background: To support compare with user-defined template feature, the work-flow are: load_compare_template.yaml --> query_results(ES) ---> auto group jobs_list ---> create groups_matrices ---> compare_values by each metrics ---> format/show results current patch is the step: auto group jobs_list Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/params_group.rb | 120 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 1 deletion(-) diff --git a/lib/params_group.rb b/lib/params_group.rb index 0f07d26..97827cd 100644 --- a/lib/params_group.rb +++ b/lib/params_group.rb @@ -2,7 +2,7 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true -# Exammple: +# Example: # Input: jobs_list. The results of ES query. # # eg: [ jobs1, jobs2, ...... ] @@ -113,3 +113,121 @@ end def remove_singleton(groups) groups.delete_if { |_k, v| v.length < 2 } end + +# -------------------------------------------------------------------------------------------------- +# auto_group_by_template: auto group job_list by user's template +# Example: +# Input: +# 1. jobs_list. +# 2. params from user's template that include: +# groups_params(x_params): +# eg: ['block_size', 'package_size'] +# dimensions: +# eg: [ +# {'os' => 'openeuler', 'os_version' => '20.03'}, +# {'os' => 'centos', 'os_version' => '7.6'} +# ] +# metrics: +# eg: ['fio.read_iops', 'fio_write_iops'] +# Output: +# eg: +# { +# '4K|1G' => { +# 'openeuler 20.03' => [ +# {'stats' => {'fio.write_iops' => 312821.002387, 'fio.read_iops' => 212821.2387}}, +# {'stats' => {'fio.write_iops' => 289661.878453}}, +# ... +# ], +# 'centos 7.6' => [...] +# }, +# '16K|1G' => {...}, +# ... +# } + +def auto_group_by_template(jobs_list, group_params, dimensions, metrics) + job_list = extract_jobs_list(jobs_list) + get_group_by_template(job_list, group_params, dimensions, metrics) +end + +def get_group_by_template(job_list, group_params, dimensions, metrics) + groups = {} + job_list.each do |job| + new_job = get_new_job(job, metrics) + next if new_job.empty? + + group_key = get_user_group_key(job, group_params) + dimension = get_user_dimension(job, dimensions) + next unless group_key && dimension + + groups[group_key] ||= {} + groups[group_key][dimension] ||= [] + groups[group_key][dimension] << new_job + end + groups +end + +# @group_params Array(String) +# eg: +# ['block_size', 'package_size'] +# return eg: +# '4K|1G' +def get_user_group_key(job, group_params) + group_key_list = [] + group_params.each do |param| + value = find_param_in_job(job, param) + group_key_list << value if value + end + return nil if group_key_list.size < group_params.size || group_key_list.empty? + + group_key_list.join('|') +end + +def find_param_in_job(job, param) + return job[param] if job.key?(param) + + job.each_value do |v| + return v[param] if v.is_a?(Hash) && v.key?(param) + end +end + +# @dimension Array(Hash) +# eg: +# [ +# {os => openeuler, os_version => 20.03}, +# {os => centos, os_version => 7.6} +# ] +# return eg: +# 'openeuler 20.03' +def get_user_dimension(job, dimensions) + dimension_list = [] + dimensions.each do |dim| + dim.each do |key, value| + if job[key] == value + dimension_list << value + end + end + return nil if !dimension_list.empty? && dimension_list.size < dim.size + end + return nil if dimension_list.empty? + + dimension_list.join(' ') +end + +# @metrics Array(String) +# eg: +# ["fio.read_iops", "fio.write_iops"] +# return new_job +# eg: +# {'stats' => {'fio.write_iops' => 312821.002387, 'fio.read_iops' => 212821.2387}}, +def get_new_job(job, metrics) + return {} unless job['stats'] + + new_job = {} + metrics.each do |metric| + if job['stats'].key?(metric) + new_job['stats'] ||= {} + new_job['stats'][metric] = job['stats'][metric] + end + end + new_job +end -- 2.23.0

1 1

[PATCH v2 compass-ci 09/10] container/mail-robot: start
by Luan Shengde 04 Nov '20

04 Nov '20

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/start | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 container/mail-robot/start diff --git a/container/mail-robot/start b/container/mail-robot/start new file mode 100755 index 0000000..b65b002 --- /dev/null +++ b/container/mail-robot/start @@ -0,0 +1,37 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig' + +docker_rm 'mail-robot' + +names = Set.new %w[ + JUMPER_HOST + JUMPER_PORT + SEND_MAIL_PORT +] + +defaults = relevant_defaults(names) + +env = docker_env(defaults) + +cmd = %w[ + docker run + --restart=always + --name=mail-robot + -d +] + env + %W[ + -v /c/upstream-repos:/c/upstream-repos:ro + -v /srv/cci/Maildir/.compass-ci:/srv/cci/Maildir/.compass-ci:rw + -v #{ENV['CCI_SRC']}:/c/compass-ci:rw + -w /c/compass-ci/container/mail-robot + --log-driver json-file + mail-robot +] + +cmd += ['ruby', 'run.rb'] + +system(*cmd) -- 2.23.0

1 0

[PATCH v2 compass-ci 08/10] container/mail-robot: build
by Luan Shengde 04 Nov '20

04 Nov '20

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/build | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 container/mail-robot/build diff --git a/container/mail-robot/build b/container/mail-robot/build new file mode 100755 index 0000000..c564add --- /dev/null +++ b/container/mail-robot/build @@ -0,0 +1,5 @@ +#!/bin/sh +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +docker build -t mail-robot:latest . -- 2.23.0

1 0

[PATCH v2 compass-ci 04/10] container/mail-robot: assign-jumper-account.rb
by Luan Shengde 04 Nov '20

04 Nov '20

AssignJumperAccount apply account with apply_account_info apply_account_info - my_email - my_name - my_uuid - my_ssh_pubkey check account exists Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/assign-jumper-account.rb | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100755 container/mail-robot/assign-jumper-account.rb diff --git a/container/mail-robot/assign-jumper-account.rb b/container/mail-robot/assign-jumper-account.rb new file mode 100755 index 0000000..0728b89 --- /dev/null +++ b/container/mail-robot/assign-jumper-account.rb @@ -0,0 +1,41 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' + +# apply jumper account for user +# the enter point is apply_jumper_account +# the apply_account_info hash need include +# my_email, my_name, my_uuid, my_ssh_pubkey +class AssignJumperAccount + def initialize(apply_account_info) + @apply_account_info = apply_account_info + end + + def send_error_email(error_message) + @apply_account_info['error_message'] = error_message + send_err_mail = SendApplyAccountEmail.new(@apply_account_info) + send_err_mail.send_error_email + end + + def apply_jumper_account + account_info_str = %x(curl -XGET "#{ENV['JUMPER_HOST']}:#{ENV['JUMPER_PORT']}/assign_account" \ + -d '#{(a)apply_account_info.to_json}') + account_info = JSON.parse account_info_str + + check_account_info(account_info) + + return account_info + end + + def check_account_info(account_info) + return unless account_info['my_login_name'].nil? + + error_message = 'No more available jumper account.' + send_error_email(error_message) + + raise error_message + end +end -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] create matrices-groups according to group jobs
by Lu Weitao 04 Nov '20

04 Nov '20

create matrices-groups according to auto group job_list background: To support compare with user-defined template feature, the work-flow are: load_compare_template.yaml --> query_results(ES) ---> auto group jobs_list ---> create groups_matrices ---> compare_values by each metrics ---> format/show results current patch is the step: create groups_matrices Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/matrix2.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index 05f96dc..b38b2fb 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -120,3 +120,27 @@ def combine_group_query_data(query_data, dims) end groups end + +# input: +# 1. query results(job_list) from es_query that will be auto group by auto_group_by_template() +# 2. params from user's template include: +# groups_params(x_params): +# eg: ['block_size', 'package_size'] +# dimensions: +# eg: [ +# {'os' => 'openeuler', 'os_version' => '20.03'}, +# {'os' => 'centos', 'os_version' => '7.6'} +# ] +# metrics: +# eg: ['fio.read_iops', 'fio_write_iops'] +# return: group_matrix of Hash(String, Hash(String, matrix)) +def combine_group_jobs_list(query_data, groups_params, dimensions, metrics) + job_list = query_data['hits']['hits'] + groups = auto_group_by_template(job_list, groups_params, dimensions, metrics) + groups.each do |group_key, dims| + dims.each do |dim_key, jobs| + groups[group_key][dim_key] = create_matrix(jobs) + end + end + groups +end -- 2.23.0

1 0

[PATCH v4 compass-ci 2/2] create matrices-groups according to group jobs
by Lu Weitao 04 Nov '20

04 Nov '20

create matrices-groups according to auto group job_list background: To support compare with user-defined template feature, the work-flow are: load_compare_template.yaml --> query_results(ES) ---> auto group jobs_list ---> create groups_matrices ---> compare_values by each metrics ---> format/show results current patch is the step: create groups_matrices Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/matrix2.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index 05f96dc..b38b2fb 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -120,3 +120,27 @@ def combine_group_query_data(query_data, dims) end groups end + +# input: +# 1. query results(job_list) from es_query that will be auto group by auto_group_by_template() +# 2. params from user's template include: +# groups_params(x_params): +# eg: ['block_size', 'package_size'] +# dimensions: +# eg: [ +# {'os' => 'openeuler', 'os_version' => '20.03'}, +# {'os' => 'centos', 'os_version' => '7.6'} +# ] +# metrics: +# eg: ['fio.read_iops', 'fio_write_iops'] +# return: group_matrix of Hash(String, Hash(String, matrix)) +def combine_group_jobs_list(query_data, groups_params, dimensions, metrics) + job_list = query_data['hits']['hits'] + groups = auto_group_by_template(job_list, groups_params, dimensions, metrics) + groups.each do |group_key, dims| + dims.each do |dim_key, jobs| + groups[group_key][dim_key] = create_matrix(jobs) + end + end + groups +end -- 2.23.0

1 0

[PATCH v3 lkp-tests] execute on_fail when exit-code isn't 0
by Hu Xuejiao 04 Nov '20

04 Nov '20

[why] when a job-task fails(exit_code != 0), the on_fail function needs to be called to keep the program in the waiting state to user adjustment Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- bin/run-lkp | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/run-lkp b/bin/run-lkp index 26669331..c4d43db4 100755 --- a/bin/run-lkp +++ b/bin/run-lkp @@ -97,6 +97,7 @@ elif [ "$run_job_exit_code" != 0 ]; then else set_job_state 'failed' fi + $LKP_DEBUG_PREFIX $job_script on_fail elif [ -f '/var/log/wtmp' ] && has_cmd last && last -n1 | grep -q -E '(pts/|tty)[0-9].*(still logged in)'; then set_job_state 'disturbed' disturbed=1 -- 2.23.0

1 0

[PATCH v3 lkp-tests] (job2sh.rb) add on_fail to job.sh
by Hu Xuejiao 04 Nov '20

04 Nov '20

[why] If job.yaml is defined: on_fail: sleep(tests): 300(runtime) [how] By sh_on_fail, it will appear in job.sh: on_fail() { . $LKP_SRC/lib/http.sh . $LKP_SRC/lib/job.sh . $LKP_SRC/lib/env.sh export_top_env run_test $LKP_SRC/tests/wrapper sleep 500 } Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- lib/job2sh.rb | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/lib/job2sh.rb b/lib/job2sh.rb index 9c95ff4b..d128e8ef 100755 --- a/lib/job2sh.rb +++ b/lib/job2sh.rb @@ -142,7 +142,7 @@ class Job2sh < Job end def parse_one(ancestors, key, val, pass) - if ['pp', 'define_files', 'monitors'].include?(key) + if ['pp', 'define_files', 'monitors', 'on_fail'].include?(key) return false end tabs = indent(ancestors) @@ -266,11 +266,13 @@ class Job2sh < Job sh_run_job(job) local_script_lines += @script_lines - sh_extract_stats(job) local_script_lines += @script_lines sh_define_files() + local_script_lines += @script_lines + + sh_on_fail(job) out_line '"$@"' @@ -357,6 +359,28 @@ class Job2sh < Job out_line "\tchmod +x $LKP_SRC/#{file}" end end - out_line '}' + out_line "}\n\n" + end + + def sh_on_fail(job = nil) + @script_lines = [] + + @programs = available_programs(:workload_elements) + + job ||= (@jobx || @job).clone + + @cur_func = :on_fail + + out_line 'on_fail()' + out_line '{' + out_line + out_line "\t. $LKP_SRC/lib/http.sh" + out_line "\t. $LKP_SRC/lib/job.sh" + out_line "\t. $LKP_SRC/lib/env.sh" + out_line + out_line "\texport_top_env" + out_line + parse_hash [], job['on_fail'] + out_line "}" end end -- 2.23.0

1 0