- Compass-ci - mailweb.openeuler.org

[PATCH v3 lkp-tests] tests/build-pkg: fix upload in container
by Wang Yong 08 Dec '20

08 Dec '20

docker can not use mount operation, so now use curl to upload build-pkg cgz file Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/build-pkg | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/build-pkg b/tests/build-pkg index b26798509..622bef1a3 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -18,6 +18,8 @@ ## See: https://wiki.archlinux.org/index.php/Makepkg . $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/env.sh +. $LKP_SRC/distro/common check_vars() { @@ -29,6 +31,13 @@ check_vars() [ -n "$os_mount" ] || die "os_mount is empty" } +mount_pkg_mnt() +{ + [ -n "$LKP_SERVER" ] && { + mount -t cifs -o guest,vers=1.0,noacl,nouser_xattr,port=446 //$LKP_SERVER$PKG_MNT $PKG_MNT || die "Failed to run mount" + } +} + mount_dest() { # the same image is mounted to cifs and nfs, the generated cgz files @@ -39,9 +48,7 @@ mount_dest() PKG_MNT=/initrd/build-pkg mkdir -p "$PKG_MNT" - [ -n "$LKP_SERVER" ] && { - mount -t cifs -o guest,vers=1.0,noacl,nouser_xattr,port=446 //$LKP_SERVER$PKG_MNT $PKG_MNT || die "Failed to run mount" - } + is_docker || mount_pkg_mnt } get_pkgfile() @@ -142,7 +149,11 @@ build_source_pkg() PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ $LKP_SRC/sbin/makepkg -A --check --skippgpcheck --config $LKP_SRC/etc/makepkg.conf 2>&1 - [ "$?" == 0 ] && create_softlink + [ "$?" == 0 ] || exit 1 + create_softlink + + is_docker || exit 0 + upload_to_target_dir ${PKG_MNT}/${pack_to}/${pkgname} } check_vars -- 2.23.0

1 0

[PATCH v2 compass-ci] answerback-email: add check account exists
by Luan Shengde 08 Dec '20

08 Dec '20

add check account exists for assigning/updating account avoid re-assign accounts for single email address avoid update account for non-account email [why] the tool may assign a new account for the email even if the email already has one. it will cause a big waste of account resource for wrong operation or test. [how] add check account exists for the email before assigning a new account. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/assign-account/answerback-email.rb | 36 ++++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index 53c321d..c08f10e 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -61,21 +61,19 @@ conf_info = { 'is_update_account' => false } -def init_info(mail_content, email_info, my_info) +def init_info(mail_content, email_info, my_info, my_info_es) my_info['my_email'] = mail_content.from[0] + + read_my_login_name(my_info['my_email'], my_info_es) email_info['my_name'] = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') - return if mail_content.attachments.empty? - email_info['new_email_pubkey'] = mail_content.attachments[0].body.decoded.strip + email_info['new_email_pubkey'] = mail_content.attachments[0].body.decoded.strip || nil end def read_my_login_name(my_email, my_info_es) - my_account_info_str = %x(curl -XGET localhost:9200/accounts/_doc/#{my_email}) - my_account_info = YAML.safe_load my_account_info_str - message = "No such email found from the ES: #{my_email}" - raise message unless my_account_info['found'] + my_account_info = ESQuery.new(index: 'accounts').query_by_id(my_email) - my_info_es.update my_account_info['_source'] + my_info_es.update my_account_info || {} end options = OptionParser.new do |opts| @@ -102,6 +100,7 @@ options = OptionParser.new do |opts| end my_info['my_email'] = email_address + read_my_login_name(email_address, my_info_es) end opts.on('-n name', '--name name', 'appoint name') do |name| @@ -134,7 +133,7 @@ options = OptionParser.new do |opts| return false end mail_content = Mail.read(email_file) - init_info(mail_content, email_info, my_info) + init_info(mail_content, email_info, my_info, my_info_es) end opts.on('-g', '--gen-sshkey', 'generate jumper rsa public/private key and return pubkey') do @@ -142,7 +141,6 @@ options = OptionParser.new do |opts| end opts.on('-u', '--update', 'updata configurations') do - read_my_login_name(my_info['my_email'], my_info_es) conf_info['is_update_account'] = true end @@ -240,9 +238,27 @@ def check_my_name_exist(my_info) return false end +def check_email_assigned_account(conf_info, my_info_es) + if conf_info['is_update_account'] + return true unless my_info_es.empty? + + message = "The email has not assigned an account yet.\n" + message += "Delete the '-u' option to assign a new account." + else + return true if my_info_es.empty? + + message = "The email has already assigned an account.\n" + message += "You can use option '-u' to update the account." + end + puts message + + return false +end + def send_account(my_info, conf_info, email_info, my_info_es, stdin_info) return unless check_server return unless check_my_email(my_info) + return unless check_email_assigned_account(conf_info, my_info_es) my_info['my_uuid'] = %x(uuidgen).chomp unless conf_info['is_update_account'] build_my_info_from_input(my_info, email_info, my_info_es, stdin_info) -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] lib/compare_data_format.rb: fix NoMethodError
by Lu Kaiyi 08 Dec '20

08 Dec '20

[error] /home/lukaiyi/compass-ci/lib/compare_data_format.rb:57:in `block (2 levels) in assign_echart_source': undefined method `[]' for nil:NilClass (NoMethodError) from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:55:in `each' from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:55:in `block in assign_echart_source' from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:53:in `each' from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:53:in `assign_echart_source' from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:46:in `block in assign_echart_data_set' from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:39:in `each' from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:39:in `assign_echart_data_set' from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:30:in `block (2 levels) in convert_to_echart_dataset' from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:29:in `each' from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:29:in `block in convert_to_echart_dataset' from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:28:in `each_value' from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:28:in `convert_to_echart_dataset' from /home/lukaiyi/compass-ci/lib/compare_data_format.rb:21:in `format_for_echart' from /home/lukaiyi/compass-ci/lib/compare_matrixes.rb:512:in `show_compare_result' from /home/lukaiyi/compass-ci/lib/compare.rb:103:in `compare_by_template' from /home/lukaiyi/compass-ci/sbin/compare:70:in `<main>' Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/compare_data_format.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compare_data_format.rb b/lib/compare_data_format.rb index 1959045..bf4013b 100644 --- a/lib/compare_data_format.rb +++ b/lib/compare_data_format.rb @@ -53,7 +53,7 @@ class FormatEchartData dimensions.each do |dimension| dimension_values = [dimension] @x_params.each do |x_param| - if @metrics_compare_results[x_param][metric] + if @metrics_compare_results[x_param][metric] && @metrics_compare_results[x_param][metric][value_type] dimension_values << @metrics_compare_results[x_param][metric][value_type][dimension] else source[0].delete(x_param) -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] sbin/compare_template: friendly show compare template result in command line
by Lu Kaiyi 08 Dec '20

08 Dec '20

[why] friendly show compare template result in command line. [how] rehandle the json format result of "compare -t <compare_template.yaml>" with third-party module in python3. [usage] compare_template <compare_template.yaml> [example] compare_template compare_template.yaml in z9 [input] a compare_template.yaml like below: compare_metrics: - fio.write_iops - fio.read_iops filter: suite: - fio-basic os_arch: - aarch64 - x86 compare_dimensions: - os: debian os_version: sid - os: openeuler os_version: 20.03 x_params: - bs - test_size title: Hackbench Performance Testing unit: KB/s [output] Hackbench Performance Testing +------------------------------------+-----------+----------+----------+----------+----------+---------+---------+---------+----------+ | fio.read_iops | 4k|1G | 4k|80G | 16k|1G | 32k|1G | 64k|1G | 128k|1G | 256k|1G | 512k|1G | 1024k|1G | +------------------------------------+-----------+----------+----------+----------+----------+---------+---------+---------+----------+ | openeuler 20.03 average | 144076.29 | 11601.10 | 37865.30 | 21145.10 | 14010.34 | 6701.24 | 3205.08 | 1367.48 | 673.33 | | openeuler 20.03 standard_deviation | 195.00 | 0.00 | 214.00 | 205.00 | 188.00 | 183.00 | 180.00 | 191.00 | 191.00 | +------------------------------------+-----------+----------+----------+----------+----------+---------+---------+---------+----------+ Hackbench Performance Testing +------------------------------------+-----------+----------+----------+----------+---------+---------+---------+----------+ | fio.write_iops | 4k|1G | 16k|1G | 32k|1G | 64k|1G | 128k|1G | 256k|1G | 512k|1G | 1024k|1G | +------------------------------------+-----------+----------+----------+----------+---------+---------+---------+----------+ | openeuler 20.03 average | 122003.54 | 33528.53 | 31469.06 | 13870.14 | 8249.71 | 4329.45 | 1976.54 | 1141.00 | | openeuler 20.03 standard_deviation | 174.00 | 188.00 | 171.00 | 197.00 | 181.00 | 175.00 | 170.00 | 176.00 | +------------------------------------+-----------+----------+----------+----------+---------+---------+---------+----------+ Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- sbin/compare_template | 53 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100755 sbin/compare_template diff --git a/sbin/compare_template b/sbin/compare_template new file mode 100755 index 0000000..6b702c0 --- /dev/null +++ b/sbin/compare_template @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +import os +import sys +import json +import prettytable as pt + +# receive compare_template.yaml and auto pretty show compare results +class TableShow: + def __init__(self, result_dict): + self.title = result_dict['title'] + self.tables = result_dict['tables'] + self.tb = None + + def show_table(self): + for (table_title, table) in self.tables.items(): + self.tb = pt.PrettyTable() + self.set_field_names(table, table_title) + self.set_align(table_title) + self.add_row(table) + self.print_table() + + def set_field_names(self, table, table_title): + field_names = [table_title] + field_names.extend(table['average']['source'][0]) + self.tb.field_names = field_names + + def set_align(self, table_title): + for field_name in self.tb.field_names: + self.tb.align[field_name] = 'r' + self.tb.align[table_title] = 'l' + + def add_row(self, table): + row_names = ['average', 'standard_deviation'] + for row_name in row_names: + row = table[row_name]['source'][1] + row_title = ' '.join([row[0], row_name]) + format_data_row = ["%.2f" % data for data in row[1:]] + self.tb.add_row([row_title, *format_data_row]) + + def print_table(self): + print(self.title) + print(self.tb) + print() + + +if __name__ == '__main__': + template_yaml = sys.argv[1] + result_dict = json.loads(os.popen(f"compare -t {template_yaml}").read()) + table_show = TableShow(result_dict) + table_show.show_table() -- 2.23.0

1 0

[PATCH lkp-tests] stats/iozone: fix rubocop warning
by Zhang Yu 08 Dec '20

08 Dec '20

iozone:17:34: C: Style/SlicingWithRange: Prefer ary[n..] over ary[n..-1]. data.insert i + 1, data[i][-8..-1] ^^^^^ iozone:21:8: C: Style/SlicingWithRange: Prefer ary[n..] over ary[n..-1]. data[2..-1].each_with_index do |v, i| ^^^^^ Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- stats/iozone | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stats/iozone b/stats/iozone index 32dce7ba..62184cb2 100755 --- a/stats/iozone +++ b/stats/iozone @@ -14,11 +14,11 @@ while (line = STDIN.gets) data = line.split data.each_index do |i| if data[i].size > 8 - data.insert i + 1, data[i][-8..-1] + data.insert i + 1, data[i][-8..] data[i] = data[i][0..-9] end end - data[2..-1].each_with_index do |v, i| + data[2..].each_with_index do |v, i| puts data[0] + 'KB_' + data[1] + 'reclen.' + keys[i] + ': ' + v per_io_type[keys[i]] ||= 0 per_io_type[keys[i]] += v.to_i -- 2.23.0

1 0

[PATCH lkp-tests] tests: replace source field with upstream_repo
by Liu Shaofei 07 Dec '20

07 Dec '20

The source fields in PKGBUILD file has multiple values. Therefore, only the git repository address needs to be replaced with. Here is as follow: source=("$_pkgname::git+https://github.com/cesanta/mongoose.git" "$_pkgname.conf" "$_pkgname.service") => source=("$_pkgname::git://172.168.131.113/m/mongoose/mongoose.git#commit=HEAD" "$_pkgname.conf" "$_pkgname.service") Signed-off-by: Liu Shaofei <liushaofei5(a)huawei.com> --- tests/build-pkg | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/tests/build-pkg b/tests/build-pkg index b2679850..6cf931bc 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -95,32 +95,20 @@ create_softlink() replace_source() { - sed -i "s|^source=[^)]*|$1|g" "$2" -} - -# replace *.tar.* address from source field in PKGBUILD -replace_tar_source() -{ - echo "source=(" > PKGBUILD.src + echo "source=(" > $1 for url in ${source[@]} do - echo "$url" | egrep '.+\.tar' && { - local_url="http://${LKP_SERVER}:8800/initrd/build-tar/${pkgname}/${url##*/}" - - # check whether local server exists *.tar.* file - wget -q --spider "${local_url}" - [ "$?" == 0 ] || continue - - url="\"${local_url}\"" - } - echo "$url" >> PKGBUILD.src + echo "$url" | egrep 'git\+|\.git$' && { + url="\"${repo_dir}git://${LKP_SERVER}/${upstream_repo}#commit=${upstream_commit}\"" + } + echo "$url" >> $1 done - echo ")" >> PKGBUILD.src + echo ")" >> $1 } build_source_pkg() { - local repo_dir="" + repo_dir="" source PKGBUILD [ -n "$upstream_repo" ] && [ -n "$upstream_url" ] && { @@ -130,13 +118,11 @@ build_source_pkg() repo_dir="$(basename ${upstream_url})" repo_dir="${repo_dir%.git*}::" fi - - upstream_source="source=(\"${repo_dir}git://${LKP_SERVER}/${upstream_repo}#commit=${upstream_commit}\"" - replace_source "${upstream_source}" PKGBUILD - [ -n "$PKGBUILD_TAG" ] && replace_source "${upstream_source}" "PKGBUILD-$PKGBUILD_TAG" } - replace_tar_source + replace_source "PKGBUILD.src" + + [ -n "$PKGBUILD_TAG" ] && replace_source "PKGBUILD-$PKGBUILD_TAG" cgz_name="$PKG_MNT/${pack_to}/${pkgname}/${upstream_commit}.cgz" PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ -- 2.23.0

1 0

[PATCH compass-ci 2/3] scheduler/scheduler.cr: replace puts with JSONLogger
by Wu Zhende 07 Dec '20

07 Dec '20

Use JSONLogger can add additional information Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/scheduler/scheduler.cr | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr index 9f5e670..17a6f15 100644 --- a/src/scheduler/scheduler.cr +++ b/src/scheduler/scheduler.cr @@ -4,6 +4,7 @@ require "kemal" require "../lib/sched" +require "../lib/json_logger" # ------------------------------------------------------------------------------------------- # end_user: @@ -31,10 +32,12 @@ module Scheduler VERSION = "0.2.0" add_context_storage_type(Sched) + add_context_storage_type(JSONLogger) add_context_storage_type(Time::Span) before_all do |env| env.set "start_time", Time.monotonic + env.set "log", JSONLogger.new(env: env) env.set "sched", Sched.new(env) env.response.headers["Connection"] = "close" end @@ -43,9 +46,13 @@ module Scheduler env.get("sched").as(Sched) end + def self.log(env) + env.get("log").as(JSONLogger) + end + # for debug (maybe kemal debug|logger does better) def self.debug_message(env, response) - puts %({"from": "#{env.request.remote_address}", "response": #{response.to_json}}) + log(env).info(%({"from": "#{env.request.remote_address}", "response": #{response.to_json}})) end # echo alive @@ -63,7 +70,7 @@ module Scheduler response = get_sched_from_env(env).find_job_boot job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? - puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id + log(env).info(%({"job_id": "#{job_id}", "job_state": "boot"})) if job_id response end @@ -73,7 +80,7 @@ module Scheduler response = get_sched_from_env(env).find_next_job_boot job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? - puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id + log(env).info(%({"job_id": "#{job_id}", "job_state": "boot"})) if job_id response end @@ -85,7 +92,7 @@ module Scheduler job_messages = get_sched_from_env(env).submit_job job_messages.each do |job_message| - puts job_message.to_json + log(env).info(job_message.to_json) end job_messages.to_json @@ -97,7 +104,7 @@ module Scheduler job_package = env.params.url["job_package"] file_path = ::File.join [Kemal.config.public_folder, job_id, job_package] - puts %({"job_id": "#{job_id}", "job_state": "download"}) + log(env).info(%({"job_id": "#{job_id}", "job_state": "download"})) send_file env, file_path end @@ -157,7 +164,7 @@ module Scheduler # ?job_file=/lkp/scheduled/job.yaml&job_state=post_run&job_id=10 # ?job_file=/lkp/scheduled/job.yaml&loadavg=0.28 0.82 0.49 1/105 3389&start_time=1587725398&end_time=1587725698&job_id=10 get "/~lkp/cgi-bin/lkp-jobfile-append-var" do |env| - puts get_sched_from_env(env).update_job_parameter + log(env).info(get_sched_from_env(env).update_job_parameter) "Done" end @@ -194,14 +201,14 @@ module Scheduler # get job_id from request job_id = env.params.query["job_id"]? if job_id - puts get_sched_from_env(env).close_job(job_id) + log(env).info(get_sched_from_env(env).close_job(job_id)) end "Done" end get "/~lkp/cgi-bin/lkp-wtmp" do |env| - puts get_sched_from_env(env).update_tbox_wtmp + log(env).info(get_sched_from_env(env).update_tbox_wtmp) "Done" end @@ -215,7 +222,7 @@ module Scheduler get_sched_from_env(env).report_ssh_port(testbox, ssh_port) end - puts %({"job_id": "#{job_id}", "state": "set ssh port", "ssh_port": "#{ssh_port}", "tbox_name": "#{testbox}"}) + log(env).info(%({"job_id": "#{job_id}", "state": "set ssh port", "ssh_port": "#{ssh_port}", "tbox_name": "#{testbox}"})) "Done" end -- 2.23.0

2 1

[PATCH compass-ci] doc/job: add result_root.md
by Zhang Yu 07 Dec '20

07 Dec '20

Add a document for introducing the result_root Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- doc/job/result_root.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/job/result_root.md diff --git a/doc/job/result_root.md b/doc/job/result_root.md new file mode 100644 index 0000000..ec9ae8d --- /dev/null +++ b/doc/job/result_root.md @@ -0,0 +1,8 @@ +# result_root + +Meaning: +- The result_root field specifies the path for storing the result +- if this field is not specified, the default value is: /result/#{suite}/#{submit_date}/ + #{tbox_group}/#{rootfs}", "#{pp_params}", "#{id}" +- we can specify the value of result_root in the job.yaml file, here is an example: + result_root: /srv/result/iperf -- 2.23.0

1 0

openEuler Summit 报名渠道开启，我们为社区内的开发者准备了很多专属环节
by public openEuler 07 Dec '20

07 Dec '20

各位社区内的开发者： openEuler Summit 所有议程 <https://openeuler.org/zh/interaction/summit-list/> 本周已经全部上线。本次峰会设置了 Linux 操作系统、云与云原生、虚拟化、基础软件、开源基础设施、安全与可信六大专场。一些通过 Call for Speaker 提交的议题已经入选。欢迎大家快来围观有没有你感兴趣的议题。 [image: image.png] 针对社区的开发者，我们准备了很多专属环节： 1. 大会为社区开发者准备了专属报名通道，在现场有专属的签到通道和定制的伴手礼。 2. 大会为Maintainer 准备了一份十分具有荣誉感和纪念意义的徽章。大会现场还准备了惊喜环节：大会主题演讲时会通过报名时你填写的 Gitee ID 和Gitee 后台的数据演示一个非常炫酷的Live Demo。我们诚挚地邀请社区开发者到现场参会。社区开发者专属报名通道：https://etherpad.openeuler.org/p/openEuler-Summit-2020 P.S：通过该链接报名的人员可以参加 Day1和 Day2 两天会议。 openEuler 也将迎来自己的一周岁生日，社区准备了一个“我为 openEuler 代言”的百人视频合集，诚挚地邀请所有热爱openEuler 的人来拍摄一段视频，来纪念openEuler这朝气蓬勃的第一年。 “我为 openEuler 代言”的视频拍摄需求：自拍形式，对着镜头回答以下问题： 1. 我想要的操作系统是****的， 2. 我在**时候参与/认识/加入/知道的openEuler（社区） 3. 我在openEuler贡献了**** 4. 我是***，我为openEuler代言使用场景：您拍摄的视频素材将用于 openEuler Summit 2020 前期的会议宣传和开场视频中。拍摄指导： 1. 背景任您喜欢 2. 人物面对镜头，位于画面中央，取半身构图，横向拍摄16:9 的画面比例 3. 最好选择白炽灯，白色光源 4. 尽量由另一个人辅助，用手机后置摄像头拍摄，不要用前置，拍摄前擦拭摄像头 5. 读口播时，声音越大越好 6. 尽量保证每一句话都是完整通顺的，中间重来的部分可以忽略 7. 视频输出，用电脑将手机视频导出，打压缩包传输即可（不要用微信上传，会压缩画质） 8. 拍摄完毕后请将视频上传到百度网盘，将分享链接期限设置为永久，然后发给 public(a)openeuler.io 9. 视频收集截止日期：2020年12月11日23:59分

1 0

[PATCH compass-ci] src/lib: add delete function in job.cr
by Zhang Yu 07 Dec '20

07 Dec '20

Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- src/lib/job.cr | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index a4345e0..c877a43 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -136,6 +136,19 @@ class Job update(json.as_h) end + def delete(key : String) + initialized_keys = get_initialized_keys + if initialized_keys.includes?(key) + raise "Should not delete #{key}" + else + if @hash.has_key?(key) + @hash.delete(key) + else + raise "This field does not exist: #{key}" + end + end + end + private def set_defaults append_init_field() set_user_lkp_src() @@ -324,7 +337,7 @@ class Job @hash.delete("my_uuid") end - private def initialized? + private def get_initialized_keys() initialized_keys = [] of String REQUIRED_KEYS.each do |key| @@ -350,7 +363,10 @@ class Job "kernel_uri", "kernel_params", "linux_vmlinuz_path"] - + end + + private def initialized? + initialized_keys = get_initialized_keys initialized_keys.each do |key| return false unless @hash.has_key?(key) end -- 2.23.0

1 0

[PATCH compass-ci] container: fix /usr/local/bin/impi.sh not found error
by Bai Jing 07 Dec '20

07 Dec '20

[error] /usr/local/bin/impi.sh not found Signed-off-by: Bai Jing <799286817(a)qq.com> --- container/conserver/generate_conserver.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/conserver/generate_conserver.rb b/container/conserver/generate_conserver.rb index 3d136f0..4e923c2 100755 --- a/container/conserver/generate_conserver.rb +++ b/container/conserver/generate_conserver.rb @@ -22,7 +22,7 @@ def generate_conserver(lab) console = <<~HEREDOC console #{host} { - exec /usr/local/bin/ipmi.sh #{ipmi_ip}; + exec /usr/local/bin/ipmi-sol #{ipmi_ip}; } HEREDOC File.open('conserver.cf', 'a') { |f| f << console } -- 2.23.0

1 0

[PATCH v2 compass-ci] src/scheduler: delete job2sh field in job.yaml
by Zhang Yu 07 Dec '20

07 Dec '20

[before] ... monitors: kmsg: heartbeat: meminfo: job2sh: run_job: - run_job() ... id: crystal.164398 os: openeuler ... [after] ... monitors: kmsg: heartbeat: meminfo: id: crystal.164398 os: openeuler ... Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- src/scheduler/create_job_cpio.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scheduler/create_job_cpio.cr b/src/scheduler/create_job_cpio.cr index 5c1be7e..cfa40ab 100644 --- a/src/scheduler/create_job_cpio.cr +++ b/src/scheduler/create_job_cpio.cr @@ -80,6 +80,7 @@ class Sched # put job2sh in an array if job_content.has_key?("job2sh") tmp_job_sh_content = job_content["job2sh"] + job_content.delete("job2sh") job_sh_array = [] of JSON::Any tmp_job_sh_content.as_h.each do |_key, val| -- 2.23.0

1 0

[PATCH compass-ci] doc/manual: add English version
by Li Ping 07 Dec '20

07 Dec '20

Signed-off-by: Li Ping <1477412247(a)qq.com> --- .../how-do-i-apply-for-a-test-machine.md | 245 ++++++++++++++++++ doc/manual/how-do-i-compare-test-results.md | 48 ++++ doc/manual/how-do-i-compile-pkgbuild.md | 85 ++++++ ...i-platform-to-test-open-source-projects.md | 71 +++++ doc/manual/how-do-i-view-task-results.md | 23 ++ doc/manual/submit-command-description.md | 208 +++++++++++++++ 6 files changed, 680 insertions(+) create mode 100644 doc/manual/how-do-i-apply-for-a-test-machine.md create mode 100644 doc/manual/how-do-i-compare-test-results.md create mode 100644 doc/manual/how-do-i-compile-pkgbuild.md create mode 100644 doc/manual/how-do-i-use-the-compass-ci-platform-to-test-open-source-projects.md create mode 100644 doc/manual/how-do-i-view-task-results.md create mode 100644 doc/manual/submit-command-description.md diff --git a/doc/manual/how-do-i-apply-for-a-test-machine.md b/doc/manual/how-do-i-apply-for-a-test-machine.md new file mode 100644 index 0000000..33ef47f --- /dev/null +++ b/doc/manual/how-do-i-apply-for-a-test-machine.md @@ -0,0 +1,245 @@ +# Prerequisites + +Ensure that you have performed the following operations according to the [apply-for-an-account.md](https://gitee.com/wu_fengguang/compass-ci/blob/mas…: + +- Send an email to apply for an account. +- Receive an email from compass-ci(a)qq.com. +- Configure the local environment. + +# Applying for a Test Machine (VM) + +1. Generate a local RSA private-public key pair. + + ```shell + hi684@account-vm ~% ssh-keygen -t rsa + Generating public/private rsa key pair. + Enter file in which to save the key (/home/hi684/.ssh/id_rsa): + Created directory '/home/hi684/.ssh'. + Enter passphrase (empty for no passphrase): + Enter same passphrase again: + Your identification has been saved in /home/hi684/.ssh/id_rsa. + Your public key has been saved in /home/hi684/.ssh/id_rsa.pub. + The key fingerprint is: + SHA256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx hi684@account-vm + The key's randomart image is: + +---[RSA 2048]----+ + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + +----[SHA256]-----+ + hi684@account-vm ~% ls -hla .ssh + total 16K + drwx------. 2 hi684 hi684 4.0K Nov 26 16:37 . + drwx------. 7 hi684 hi684 4.0K Nov 26 16:37 .. + -rw-------. 1 hi684 hi684 1.8K Nov 26 16:37 id_rsa + -rw-r--r--. 1 hi684 hi684 398 Nov 26 16:37 id_rsa.pub + ``` + +2. Select the YAML file as required. + + A **lkp-tests** folder is stored in each user directory `/home/${USER}`. + + ```shell + hi684@account-vm ~% cd lkp-tests/jobs + hi684@account-vm ~/lkp-tests/jobs% ls -hl borrow-* + -rw-r--r--. 1 root root 53 Nov 2 14:54 borrow-10d.yaml + -rw-r--r--. 1 root root 64 Nov 2 14:54 borrow-1d.yaml + -rw-r--r--. 1 root root 235 Nov 19 15:27 borrow-1h.yaml + ``` + +3. Submit the YAML file and connect to the test machine (VM). + + ```shell + hi684@account-vm ~/lkp-tests/jobs% submit -c -m testbox=vm-2p8g borrow-1h.yaml + submit borrow-1h.yaml, got job_id=z9.170593 + query=>{"job_id":["z9.170593"]} + connect to ws://172.168.131.2:11310/filter + {"job_id":"z9.170593","message":"","job_state":"submit","result_root":"/srv/result/borrow/2020-11-26/vm-2p8g/openeuler-20.03-aarch64/3600/z9.170593"} + {"job_id": "z9.170593", "result_root": "/srv/result/borrow/2020-11-26/vm-2p8g/openeuler-20.03-aarch64/3600/z9.170593", "job_state": "set result root"} + {"job_id": "z9.170593", "job_state": "boot"} + {"job_id": "z9.170593", "job_state": "download"} + {"time":"2020-11-26 14:45:06","mac":"0a-1f-0d-3c-91-5c","ip":"172.18.156.13","job_id":"z9.170593","state":"running","testbox":"vm-2p8g.taishan200-2280-2s64p-256g--a38-12"} + {"job_state":"running","job_id":"z9.170593"} + {"job_id": "z9.170593", "state": "set ssh port", "ssh_port": "51840", "tbox_name": "vm-2p8g.taishan200-2280-2s64p-256g--a38-12"} + Host 172.168.131.2 not found in /home/hi684/.ssh/known_hosts + Warning: Permanently added '[172.168.131.2]:51840' (ECDSA) to the list of known hosts. + Last login: Wed Sep 23 11:10:58 2020 + + + Welcome to 4.19.90-2003.4.0.0036.oe1.aarch64 + + System information as of time: Thu Nov 26 06:44:18 CST 2020 + + System load: 0.83 + Processes: 107 + Memory used: 6.1% + Swap used: 0.0% + Usage On: 89% + IP address: 172.18.156.13 + Users online: 1 + + + + root@vm-2p8g ~# + ``` + + For more information about how to use the **submit** command, testbox options, and how to borrow the specified operating system, see the FAQ at the end of this document. + +4. Return the test machine (VM) after use. + + ```shell + root@vm-2p8g ~# reboot + Connection to 172.168.131.2 closed by remote host. + Connection to 172.168.131.2 closed. + hi684@account-vm ~/lkp-tests/jobs% + ``` + +# Applying for a Test Machine (Physical Machine) + +1. Generate a local RSA private-public key pair. + + ```shell + hi684@account-vm ~% ssh-keygen -t rsa + Generating public/private rsa key pair. + Enter file in which to save the key (/home/hi684/.ssh/id_rsa): + Created directory '/home/hi684/.ssh'. + Enter passphrase (empty for no passphrase): + Enter same passphrase again: + Your identification has been saved in /home/hi684/.ssh/id_rsa. + Your public key has been saved in /home/hi684/.ssh/id_rsa.pub. + The key fingerprint is: + SHA256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx hi684@account-vm + The key's randomart image is: + +---[RSA 2048]----+ + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + |xxxxxxxxxxxxxxxxx| + +----[SHA256]-----+ + hi684@account-vm ~% ls -hla .ssh + total 16K + drwx------. 2 hi684 hi684 4.0K Nov 26 16:37 . + drwx------. 7 hi684 hi684 4.0K Nov 26 16:37 .. + -rw-------. 1 hi684 hi684 1.8K Nov 26 16:37 id_rsa + -rw-r--r--. 1 hi684 hi684 398 Nov 26 16:37 id_rsa.pub + ``` + +2. Select the YAML file as required. + + A **lkp-tests** folder is stored in each user directory `/home/${USER}`. + + ```shell + hi684@account-vm ~% cd lkp-tests/jobs + hi684@account-vm ~/lkp-tests/jobs% ls -hl borrow-* + -rw-r--r--. 1 root root 53 Nov 2 14:54 borrow-10d.yaml + -rw-r--r--. 1 root root 64 Nov 2 14:54 borrow-1d.yaml + -rw-r--r--. 1 root root 235 Nov 19 15:27 borrow-1h.yaml + ``` + +3. Submit the YAML file and connect to the test machine (physical machine). + + ```shell + hi684@account-vm ~/lkp-tests/jobs% submit -c -m testbox=taishan200-2280-2s64p-256g borrow-1h.yaml + submit borrow-1h.yaml, got job_id=z9.170594 + query=>{"job_id":["z9.170594"]} + connect to ws://172.168.131.2:11310/filter + {"job_id":"z9.170594","message":"","job_state":"submit","result_root":"/srv/result/borrow/2020-11-26/taishan200-2280-2s64p-256g/openeuler-20.03-aarch64/3600/z9.170594"} + {"job_id": "z9.170594", "result_root": "/srv/result/borrow/2020-11-26/taishan200-2280-2s64p-256g/openeuler-20.03-aarch64/3600/z9.170594", "job_state": "set result root"} + {"job_id": "z9.170594", "job_state": "boot"} + {"job_id": "z9.170594", "job_state": "download"} + {"time":"2020-11-26 14:51:56","mac":"84-46-fe-26-d3-47","ip":"172.168.178.48","job_id":"z9.170594","state":"running","testbox":"taishan200-2280-2s64p-256g--a5"} + {"job_state":"running","job_id":"z9.170594"} + {"job_id": "z9.170594", "state": "set ssh port", "ssh_port": "50420", "tbox_name": "taishan200-2280-2s64p-256g--a5"} + Host 172.168.131.2 not found in /home/hi684/.ssh/known_hosts + Warning: Permanently added '[172.168.131.2]:50420' (ECDSA) to the list of known hosts. + Last login: Wed Sep 23 11:10:58 2020 + + + Welcome to 4.19.90-2003.4.0.0036.oe1.aarch64 + + System information as of time: Thu Nov 26 14:51:59 CST 2020 + + System load: 1.31 + Processes: 1020 + Memory used: 5.1% + Swap used: 0.0% + Usage On: 3% + IP address: 172.168.178.48 + Users online: 1 + + + + root@taishan200-2280-2s64p-256g--a5 ~# + ``` + + For more information about how to use the **submit** command, testbox options, and how to borrow the specified operating system, see the FAQ at the end of this document. + +4. Return the test machine (physical machine) after use. + + ```shell + root@taishan200-2280-2s64p-256g--a5 ~# reboot + Connection to 172.168.131.2 closed by remote host. + Connection to 172.168.131.2 closed. + hi684@account-vm ~/lkp-tests/jobs% + ``` + +# FAQ + +* How Do I Change the Duration of Keeping the Test Machine when Applying for It? + + ```shell + hi684@account-vm ~/lkp-tests/jobs% cat borrow-1h.yaml + suite: borrow + testcase: borrow + + pub_key: <%= + begin + File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp + rescue + nil + end + %> + sshd: + # sleep at the bottom + sleep: 1h + hi684@account-vm ~/lkp-tests/jobs% grep sleep: borrow-1h.yaml + sleep: 1h + # Use the VIM editor to change the value of the sleep field. + hi684@account-vm ~/lkp-tests/jobs% vim borrow-1h.yaml + # After changing the value, submit the request again. + hi684@account-vm ~/lkp-tests/jobs% submit -c -m testbox=vm-2p8g borrow-1h.yaml + ``` + +* Guide to the **submit** Command + + Reference: [submit Command Description.md](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/m… + +* What Are the testbox Options? + + For details about the testbox options, visit https://gitee.com/wu_fengguang/lab-z9/tree/master/hosts. + + > ![](./../public_sys-resources/icon-note.gif) **Note** + > + > VM testbox: vm-xxx + > + > PM testbox: taishan200-2280-xxx + + > ![](./../public_sys-resources/icon-notice.gif) **Notice** + > +> - If the testbox of a physical machine ends with `--axx`, a physical machine is specified. If a task is already in the task queue of the physical machine, the borrow task you submitted will not be processed until the previous task in the queue is completed. + > - If the testbox of a physical machine does not end with `-axx`, no physical machine is specified. In this case, the borrow task you submitted will be immediately allocated to idle physical machines in the cluster for execution. + +* How Do I Borrow the Specified Operating System? + + For details about the supported `os`, `os_arch`, and `os_version`, see [os-os\_verison-os\_arch.md](https://gitee.com/wu_fengguang/compass-ci/blob/…. \ No newline at end of file diff --git a/doc/manual/how-do-i-compare-test-results.md b/doc/manual/how-do-i-compare-test-results.md new file mode 100644 index 0000000..8a7e5f7 --- /dev/null +++ b/doc/manual/how-do-i-compare-test-results.md @@ -0,0 +1,48 @@ +# How Do I Compare Test Results? +The Compare feature is used to analyze the results of different jobs, show the performance waves and changes under different influencing factors, for users to analyze performance influencing factor. + +## url: https://compass-ci.openeuler.org/compare + +## options directions +### filters: + - suite: iperf, netperf, mysql, ... + - OS: openeuler 20.03, centos 7.6, ... + - os_arch: aarch64, x86 + - tbox_group: vm-2p8g, taishan200-2880-2s48p-256g, ... + we can combine the above options arbitrarily to limit compare scope. + choose at least one option as filter. + +### dimension + Dimension can select: os, os_version, os_arch, suite, tbox_group. + Within filter, we will compare all different job result by dimension + and keep other test conditions are same. + +## example: + filter: suite = iperf + dimension: os_version + + result: + os=openeuler/os_arch=aarch64/pp.iperf.protocol=tcp/pp.iperf.runtime=20/tbox_group=vm-2p8g # other test conditions keep same + + + 20.09 20.03 metric + -------------------- ------------------------------ ------------------------------ + fails:runs change fails:runs + | | | + 3:3 -100.0% 0:3 last_state.exit_fail + 3:3 -100.0% 0:3 last_state.is_incomplete_run + 3:3 -100.0% 0:3 last_state.test.iperf.exit_code.127 + 3:3 -100.0% 0:3 stderr.perf_command_failed + + + + 20.09 20.03 metric + -------------------- ------------------------------ ------------------------------ + %stddev change %stddev + \ | \ + 4.461021e+10 ± 6% -17.4% 3.686392e+10 ± 12% iperf.tcp.receiver.bps + 4.461112e+10 ± 6% -17.4% 3.686935e+10 ± 12% iperf.tcp.sender.bps + 94.82 -44.0% 53.10 boot-time.boot + 123.11 -58.4% 51.19 boot-time.idle + 0.00 0 4.87 boot-time.kernel_boot + 4165.50 ± 12% -99.9% 5.00 interrupts.38:GICv3.36.Level.virtio0 \ No newline at end of file diff --git a/doc/manual/how-do-i-compile-pkgbuild.md b/doc/manual/how-do-i-compile-pkgbuild.md new file mode 100644 index 0000000..097e0b8 --- /dev/null +++ b/doc/manual/how-do-i-compile-pkgbuild.md @@ -0,0 +1,85 @@ +# PKGBUILD + +A PKGBUILD is a shell script. makepkg uses the instructions contained in PKGBUILD to generate a software package that incorporates binary files and installation instructions. + +# What Does PKGBUILD Include? + +PKGBUILD includes variables and functions. + +## Defining a Variable + +- pkgname: Mandatory. It indicates the name of a software package. +- pkgver: Mandatory. It indicates the version of a software package. +- pkgrel: Mandatory. It indicates the release number of a software package. +- arch: Mandatory. It indicates the architecture sequence of a software package. +- depends: Optional. It indicates the name of the dependency package required for running a software test. +- makedepends: Optional. It indicates the list of files required for building a software package. +- source: Optional. It indicates the MD5 hash value of each source file to verify the integrity of the source file during the build process. + +## Defining a Function + +- package function + + Mandatory. It is used for installing files to the directory that will be the root directory of the build package. + +- prepare function + + Optional. It is used for executing the operation of building source code. + +- build function + + Optional. It is used for compiling and/or building source code. + +- check function + + Optional. It is used for running the test suite of the program package. + +> ![](./../public_sys-resources/icon-notice.gif) **Notice** +> +> **srcdir** is the directory for extracting or copying source files. All packaging functions run in the **srcdir** directory. **pkgdir** is the root directory for building software packages and is used only in the package function. + +# How Do I Compile PKGBUILD? + +1. Run the following command to create a PKGBUILD. The file name must be **PKGBUILD**. + + ```shell + touch PKGBUILD + ``` + +2. Use the vim editor to open the **PKGBUILD** file and compile it. The following is an example of the **PKGBUILD** file: + + ```shell + pkgname=zstd + pkgver=1.4.4 + pkgrel=2 + arch=('i686' 'x86_64' 'aarch64') + url='https://github.com/facebook/zstd' + license=('custom:BSD3' 'GPL2') + depends=('xz' 'zlib' 'lz4') + makedepends=('git') + source=('git://github.com/facebook/zstd.git#branch=dev') + md5sums=('SKIP') + + pkgver() { + cd "$srcdir/$pkgname" + git describe --long --tags | sed 's/$[^-]*-g$/r\1/;s/-/./g;s/^v//g' + } + + build() { + cd "$srcdir/$pkgname" + make + make -C contrib/pzstd + } + + package() { + cd "$srcdir/$pkgname" + make PREFIX="/usr" DESTDIR="$pkgdir/" install + install -D -m755 contrib/pzstd/pzstd "$pkgdir/usr/bin/pzstd" + install -D -m644 LICENSE "${pkgdir}/usr/share/licenses/${pkgname}/LICENSE" + } + ``` + +# Reference + +- [PKGBUILD(5) Manual Page](https://www.archlinux.org/pacman/PKGBUILD.5.html) +- [pkgbuild demo](https://git.archlinux.org/pacman.git/plain/proto/PKGBUILD.proto) \ No newline at end of file diff --git a/doc/manual/how-do-i-use-the-compass-ci-platform-to-test-open-source-projects.md b/doc/manual/how-do-i-use-the-compass-ci-platform-to-test-open-source-projects.md new file mode 100644 index 0000000..bb0af45 --- /dev/null +++ b/doc/manual/how-do-i-use-the-compass-ci-platform-to-test-open-source-projects.md @@ -0,0 +1,71 @@ +# How DO I Use the Compass-CI Platform to Test Open Source Projects? + +This document describes how to use the Compass-CI platform to test open source projects. + +### Adding the URL of the Repository to Be Tested to the upstream-repos Repository + +Perform the following steps to add the information of the code repository to be tested to the **upstream-repos** repository (https://gitee.com/wu_fengguang/upstream-repos) in YAML format: + +1. Fork the code repository to be tested and clone it to the local host. This document uses the **backlight** repository (https://github.com/baskerville/backlight) as an example. + +![](./../pictures/fork_blacklight.png) + +2. Run the following command to create a file path named with the repository name and its initial letter: + + ``` + mkdir -p b/backlight + ``` + +3. Run the following command to create the **backlight** file in the directory: + + ``` + cd b/backlight + touch backlight + ``` + +4. Run the following command to write the URL of the **backlight** repository to the **backlight** file: + + ``` + vim backlight + ``` + + The format is as follows: + + ``` + --- + url: + - https://github.com/baskerville/backlight + ``` + + > ![](./../public_sys-resources/icon-notice.gif) **Note** + > + > You can refer to the existing file format in the **upstream-repos** repository. Ensure that the formats are consistent. + +5. Run the **Pull Request** command to submit the new **backlight** file to the **backlight** repository. + +### Submitting the Test Task to the Compass-CI Platform + +1. Prepare a test case. + + You can compile and add a test case to the **lkp-tests** repository, or directly use the existing test cases in the **jobs** directory of the **lkp-tests** repository (https://gitee.com/wu_fengguang/lkp-tests) + + * Use the test cases that have been adapted in the repository. + + Use the test cases in the **lkp-tests** repository that meet the requirements. The **iperf.yaml** file is used as an example. The **iperf.yaml** file is a test case that has been adapted. It is stored in the **jobs** directory of the **lkp-tests** repository, and contains some basic test parameters. + + * Compile a test case and add it to the repository. + + For details, see [How To Add Test Cases](https://gitee.com/wu_fengguang/lkp-tests/blob/master/doc/add-testcas…. + +2. Configure the **auto\_submit.yaml** file and submit the test task. + + You only need to add the following configuration information to the **sbin/auto\_submit.yaml** file in the **compass-ci** repository: + + ``` + b/backlight/backlight: + - testbox=vm-2p8g os=openEuler os_version=20.03 os_mount=initramfs os_arch=aarch64 iperf.yaml + ``` + + Submit the modified **auto\_submit.yaml** file to the **compass-ci** repository using Pull Request. Then you can use the Compass-CI platform to test your project. + + For details about how to set parameters in the **auto\_submit.yaml** file, see https://gitee.com/wu_fengguang/compass-ci/tree/master/doc/job. \ No newline at end of file diff --git a/doc/manual/how-do-i-view-task-results.md b/doc/manual/how-do-i-view-task-results.md new file mode 100644 index 0000000..bd266df --- /dev/null +++ b/doc/manual/how-do-i-view-task-results.md @@ -0,0 +1,23 @@ +# How Do I View Task Results? + +After a test case is executed, you can log in to the https://compass-ci.openeuler.org/jobs to view the execution result. + +You can find the task on the website using the suite defined during task submission or the ID returned after the task is submitted, and click the **job\_state** column corresponding to the task to view the task result. + +![](./../pictures/jobs.png) + +## Result Files + +**job.yaml File** + +Some fields in the **job.yaml** file are submitted by users, and other fields in the file are automatically added by the platform based on the submitted jobs. This file contains all the parameters required for the test task. + +**output File** + +The **output** file records the execution process of a test case. The **check\_exit\_code** status code is usually displayed at the end of the file. If the status code is not 0, the test case is incorrect. + +**stats.json** + +After a test case is executed, a file with the same name as that of the test case is generated. The file records the test commands and standardized output results. Compass-CI parses these files and generates a file with the file name extension **.json**. + +The **stats.json** file incorporates all JSON files. The key results of all test commands are included in this file for subsequent comparison and analysis. \ No newline at end of file diff --git a/doc/manual/submit-command-description.md b/doc/manual/submit-command-description.md new file mode 100644 index 0000000..b0faf99 --- /dev/null +++ b/doc/manual/submit-command-description.md @@ -0,0 +1,208 @@ +# submit Command Description + +### Prerequisites + +The Compass-CI client has been installed. For details, see [Installing the Local Compass-CI Client](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/本地…. + +### Purpose + +You can run the **submit** command to submit a test task. This command provides multiple options to help you submit tasks more flexibly. You can enter the **submit** command in the command line to view the help information and use the command flexibly as required. + +### Basic Usage + +The test task is submitted in a YAML file. You need to prepare the YAML file of the test task. This document uses **iperf.yaml** as an example. Run the following command to submit a test task: + +``` +submit iperf.yaml +``` + +The following message is displayed: + +```shell +hi8109@account-vm ~% submit iperf.yaml +submit iperf.yaml, got job_id=z9.173924 +``` + +The **testbox** field has been added to the **iperf.yaml** file shown in this document. If the YAML file does not contain this field, an error is reported: + +```shell +hi8109@account-vm ~% submit iperf.yaml +submit iperf.yaml failed, got job_id=0, error: Missing required job key: 'testbox' +``` + +You can add the **testbox** field to the YAML file or run the following command: + +``` +submit iperf.yaml testbox=vm-2p8g +``` + +The value of the **testbox** field specifies the required test machine. You can run the `ls` command to view the available test machines in the `lkp-tests/hosts` path, as shown in the following figure: + +```shell +hi8109@account-vm ~/lkp-tests/hosts% ll +total 120K +-rw-r--r--. 1 root root 76 2020-11-02 14:54 vm-snb +-rw-r--r--. 1 root root 64 2020-11-02 14:54 vm-pxe-hi1620-2p8g +-rw-r--r--. 1 root root 64 2020-11-02 14:54 vm-pxe-hi1620-2p4g +-rw-r--r--. 1 root root 64 2020-11-02 14:54 vm-pxe-hi1620-2p1g +-rw-r--r--. 1 root root 64 2020-11-02 14:54 vm-pxe-hi1620-1p1g +-rw-r--r--. 1 root root 75 2020-11-02 14:54 vm-hi1620-2p8g +-rw-r--r--. 1 root root 75 2020-11-02 14:54 vm-hi1620-2p4g +-rw-r--r--. 1 root root 75 2020-11-02 14:54 vm-hi1620-2p1g +-rw-r--r--. 1 root root 75 2020-11-02 14:54 vm-hi1620-1p1g +-rw-r--r--. 1 root root 61 2020-11-02 14:54 vm-2p8g-pxe +-rw-r--r--. 1 root root 61 2020-11-02 14:54 vm-2p8g +-rw-r--r--. 1 root root 61 2020-11-02 14:54 vm-2p4g-pxe +-rw-r--r--. 1 root root 61 2020-11-02 14:54 vm-2p4g +-rw-r--r--. 1 root root 61 2020-11-02 14:54 vm-2p1g-pxe +-rw-r--r--. 1 root root 61 2020-11-02 14:54 vm-2p1g +-rw-r--r--. 1 root root 61 2020-11-02 14:54 vm-1p1g-pxe +-rw-r--r--. 1 root root 61 2020-11-02 14:54 vm-1p1g +-rw-r--r--. 1 root root 14 2020-11-02 14:54 taishan200-2280-2s64p-256g +-rw-r--r--. 1 root root 497 2020-11-02 14:54 lkp-skl-d01 +-rw-r--r--. 1 root root 574 2020-11-02 14:54 lkp-ivb-d04 +-rw-r--r--. 1 root root 606 2020-11-02 14:54 lkp-ivb-d02 +-rw-r--r--. 1 root root 384 2020-11-02 14:54 lkp-ivb-d01 +-rw-r--r--. 1 root root 388 2020-11-02 14:54 lkp-hsw-d01 +-rw-r--r--. 1 root root 385 2020-11-02 14:54 lkp-bdw-de1 +-rw-r--r--. 1 root root 11 2020-11-02 14:54 dc-8g +-rw-r--r--. 1 root root 11 2020-11-02 14:54 dc-4g +-rw-r--r--. 1 root root 11 2020-11-02 14:54 dc-2g +-rw-r--r--. 1 root root 11 2020-11-02 14:54 dc-1g +-rw-r--r--. 1 root root 13 2020-11-02 14:54 2288hv5-2s64p +-rw-r--r--. 1 root root 74 2020-11-02 14:54 vm-snb-i386 +``` + +> ![](./../public_sys-resources/icon-note.gif) **Note** +> +> Use the equal sign (=) to update the fields in the YAML file. The priority of the ***=*** varies with its position in the command line. +> +> * In the **submit iperf.yaml testbox=vm-2p8g** command, the **=** is defined after the YAML file. As a result, the priority of the **=** is higher than that of the YAML file. **testbox=vm-2p8g** overrides the **testbox** field defined in the YAML file. +> * In the **submit testbox=vm-2p8g iperf.yaml** command, the **=** is defined before the YAML file. As a result, the priority of **=** is lower than that of the YAML file. **testbox=vm-2p8g** does not override the **testbox** field defined in the YAML file. A value is assigned only when the YAML file does not contain the **testbox** field. + +### Advanced Usage + +The following figure shows the options of the **submit** command: + +```shell +hi8109@account-vm ~% submit +Usage: submit [options] job1.yaml job2.yaml ... + submit test jobs to the scheduler + +options: + -s, --set 'KEY: VALUE' add YAML hash to job + -o, --output DIR save job yaml to DIR/ + -a, --auto-define-files auto add define_files + -c, --connect auto connect to the host + -m, --monitor monitor job status: use -m 'KEY: VALUE' to add rule + --my-queue add to my queue +``` + +* **Usage of -s** + + You can use the **-s'KEY:VALUE'** parameter to update the key-value pair to the submitted task. An example is shown in the following figure: + + ``` + submit -s 'testbox: vm-2p8g' iperf.yaml + ``` + + * If the **iperf.yaml** file does not contain **testbox: vm-2p8g**, the field will be added to the submitted task. + * If the **iperf.yaml** file contains the **testbox** field but the value is not **vm-2p8g**, the value of **testbox** in the submitted task will be updated as **vm-2p8g**. + +* **Usage of -o** + + You can run the **-o DIR** command to save the generated YAML file to the specified directory **DIR**. An example is shown in the following figure: + + ``` + submit -o ~/iperf.yaml + ``` + + After the command is executed, the YAML file that has been processed by the **submit** command is generated in the specified directory. + +* **Usage of -a** + + If the **lkp-tests** on the client is changed in the test case, you need to use the **-a** option for adaptation. Synchronize the modification made in the **lkp-tests** on the client to the server, and generate a customized test script on the test machine. An example is shown in the following figure: + + ``` + submit -a iperf.yaml + ``` + +* **Usage of -m** + + You can use the **-m** parameter to enable the task monitoring function and print the status information during the task execution on the console. In this way, you can monitor the execution process of the test task in real time. An example is shown in the following figure: + + ``` + submit -m iperf.yaml + ``` + + The following information is displayed on the console: + + ```shell + hi8109@account-vm ~% submit -m iperf.yaml + submit iperf.yaml, got job_id=z9.173923 + query=>{"job_id":["z9.173923"]} + connect to ws://172.168.131.2:11310/filter + {"job_id":"z9.173923","message":"","job_state":"submit","result_root":"/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch6 + {"job_id": "z9.173923", "result_root": "/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/tcp-30/z9.173923", "job_state + {"job_id": "z9.173923", "job_state": "boot"} + {"job_id": "z9.173923", "job_state": "download"} + {"time":"2020-11-30 20:28:16","mac":"0a-f5-9f-83-62-ea","ip":"172.18.192.21","job_id":"z9.173923","state":"running","testbox":"vm- + {"job_state":"running","job_id":"z9.173923"} + {"job_state":"post_run","job_id":"z9.173923"} + {"start_time":"2020-11-30 12:25:15","end_time":"2020-11-30 12:25:45","loadavg":"1.12 0.38 0.14 1/105 1956","job_id":"z9.173923"} + {"job_state":"finished","job_id":"z9.173923"} + {"job_id": "z9.173923", "job_state": "complete"} + {"time":"2020-11-30 20:28:54","mac":"0a-f5-9f-83-62-ea","ip":"172.18.192.21","job_id":"z9.173923","state":"rebooting","testbox":"v + {"job_id": "z9.173923", "job_state": "extract_finished"} + connection closed: normal + ``` + +* **Usage of -c** + + The **-c** parameter must be used together with the **-m** parameter to implement the automatic login function in the task of applying for a device. + + An example is shown in the following figure: + + ``` + submit -m -c borrow-1h.yaml + ``` + + After submitting a task of applying for a device, you will receive the returned login information, such as `ssh ip -p port`. After adding the **-c** parameter, you can log in to the executor without manually entering the SSH login command. + + The following information is displayed on the console: + + ```shell + hi8109@account-vm ~% submit -m -c borrow-1h.yaml + submit borrow-1h.yaml, got job_id=z9.173925 + query=>{"job_id":["z9.173925"]} + connect to ws://172.168.131.2:11310/filter + {"job_id":"z9.173925","message":"","job_state":"submit","result_root":"/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch + {"job_id": "z9.173925", "result_root": "/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/3600/z9.173925", "job_state" + {"job_id": "z9.173925", "job_state": "boot"} + {"job_id": "z9.173925", "job_state": "download"} + {"time":"2020-11-30 20:35:04","mac":"0a-24-5d-c8-aa-d0","ip":"172.18.101.4","job_id":"z9.173925","state":"running","testbox":"vm-2 + {"job_state":"running","job_id":"z9.173925"} + {"job_id": "z9.173925", "state": "set ssh port", "ssh_port": "50200", "tbox_name": "vm-2p8g.taishan200-2280-2s48p-256g--a52-7"} + Host 172.168.131.2 not found in /home/hi8109/.ssh/known_hosts + Warning: Permanently added '[172.168.131.2]:50200' (ECDSA) to the list of known hosts. + Last login: Wed Sep 23 11:10:58 2020 + + + Welcome to 4.19.90-2003.4.0.0036.oe1.aarch64 + + System information as of time: Mon Nov 30 12:32:04 CST 2020 + + System load: 0.50 + Processes: 105 + Memory used: 6.1% + Swap used: 0.0% + Usage On: 89% + IP address: 172.17.0.1 + Users online: 1 + + + + root@vm-2p8g ~# + ``` + + You log in to the executor successfully. \ No newline at end of file -- 2.23.0

1 0

[PATCH compass-ci] container: change the style to get the hosts informations
by Bai Jing 07 Dec '20

07 Dec '20

[Why] We can prepare host information in advance. Signed-off-by: Bai Jing <799286817(a)qq.com> --- container/conserver/generate_conserver.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/container/conserver/generate_conserver.rb b/container/conserver/generate_conserver.rb index 285c393..3d136f0 100755 --- a/container/conserver/generate_conserver.rb +++ b/container/conserver/generate_conserver.rb @@ -5,15 +5,15 @@ require 'fileutils' require 'yaml' -lab = ENV['lab'] -system 'git clone file:///$CCI_REPOS/lab-$lab.git' +lab = ENV['LAB'] || 'z9' +CCI_REPOS = ENV['CCI_REPOS'] || '/c' def generate_conserver(lab) return unless File.exist?('conserver-head.cf') FileUtils.cp('conserver-head.cf', 'conserver.cf') - host_dir = "lab-#{lab}/hosts/" + host_dir = "#{CCI_REPOS}/lab-#{lab}/hosts/" return unless Dir.exist?(host_dir) Dir.each_child(host_dir) do |host| -- 2.23.0

1 0

[PATCH compass-ci] container: change conserver.cf to conserver-head.cf
by Bai Jing 07 Dec '20

07 Dec '20

[Why] the generate_conserver.rb need cp conserver-head.cf code is: FileUtils.cp('conserver-head.cf','conserver.cf') Signed-off-by: Bai Jing <799286817(a)qq.com> --- container/conserver/{conserver.cf => conserver-head.cf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename container/conserver/{conserver.cf => conserver-head.cf} (100%) diff --git a/container/conserver/conserver.cf b/container/conserver/conserver-head.cf similarity index 100% rename from container/conserver/conserver.cf rename to container/conserver/conserver-head.cf -- 2.23.0

1 0

[PATCH v2 lkp-tests] lib/job.rb: delete matrix args
by Wei Jihui 07 Dec '20

07 Dec '20

if user add matrix args by overrides args, like: submit xxx.yaml 'matrix_key=matrix_value' ... it should delete matrix args from overrides, or it will keep matrix args in job.yaml all the time. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/job.rb b/lib/job.rb index 2869dcb3..3ecc9927 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -302,6 +302,7 @@ class Job hash_job[key_array[i]] = v_array[i] end hash_job.delete(key) + @overrides.delete(key) jobs_array << hash_job end } -- 2.23.0

1 0

[PATCH v2 lkp-tests] lib: delete the space on the left and right sides of the vertical bar
by Bai Jing 07 Dec '20

07 Dec '20

[why] The multi-args is like this "test|nr_threads", if user add it like this "test |nr_threads" or the value like "- shell1 | 1 ". So we need to delete the space Signed-off-by: Bai Jing <799286817(a)qq.com> --- lib/job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/job.rb b/lib/job.rb index 2869dcb3..bcf4aa8a 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -290,11 +290,11 @@ class Job hash.each { |key, value| next unless key =~ /^\w.*\|.*\w$/ - key_array = key.split('|') + key_array = key.split('|').map(&:strip) [value].flatten.each do |v| next unless v =~ /^\w.*\|.*\w$/ - v_array = v.split('|') + v_array = v.split('|').map(&:strip) next unless key_array.size == v_array.size hash_job = hash.clone -- 2.23.0

2 1

[PATCH v5 lkp-tests] spec/submit_spec.rb: update submit_job
by Hu Xuejiao 07 Dec '20

07 Dec '20

[why] Originally submit_job is too complicated, it needs to be simplified. Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- .../job_on_fail-3000.yaml => job-3000.yaml} | 0 .../{job_on_fail.yaml => job.yaml} | 1 + ...68000000-1-65535-set_get-never-never.yaml} | 0 .../redis.yaml => link_jobs1/job.yaml} | 0 .../job-1000-2-100000ss-add.yaml} | 0 .../job-10000-2-100000ss-add.yaml} | 0 .../job-20-2-6000ss-cache.yaml} | 0 .../ftq.yaml => link_jobs2/job.yaml} | 0 ...e.yaml => job-1-300-whetstone-double.yaml} | 0 ...-300-shell1.yaml => job-3-300-shell1.yaml} | 0 .../matrix/{unixbench-spec.yaml => job.yaml} | 0 ..._files-defaults.yaml => job-defaults.yaml} | 0 spec/submit/merge_yaml/job.yaml | 1 + .../merge_yaml/merge_included_files.yaml | 1 - ...-add.yaml => job-1000-2-100000ss-add.yaml} | 0 ...add.yaml => job-10000-2-100000ss-add.yaml} | 0 ...-cache.yaml => job-20-2-6000ss-cache.yaml} | 0 .../separate_yaml/{ftq.yaml => job.yaml} | 0 spec/submit_spec.rb | 28 ++++--------------- 19 files changed, 7 insertions(+), 24 deletions(-) rename spec/submit/job_on_fail/{job_on_fail/job_on_fail-3000.yaml => job-3000.yaml} (100%) rename spec/submit/job_on_fail/{job_on_fail.yaml => job.yaml} (97%) rename spec/submit/{link_jobs/redis/redis-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml => link_jobs1/job-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml} (100%) rename spec/submit/{link_jobs/redis.yaml => link_jobs1/job.yaml} (100%) rename spec/submit/{link_jobs/ftq/ftq-1000-2-100000ss-add.yaml => link_jobs2/job-1000-2-100000ss-add.yaml} (100%) rename spec/submit/{link_jobs/ftq/ftq-10000-2-100000ss-add.yaml => link_jobs2/job-10000-2-100000ss-add.yaml} (100%) rename spec/submit/{link_jobs/ftq/ftq-20-2-6000ss-cache.yaml => link_jobs2/job-20-2-6000ss-cache.yaml} (100%) rename spec/submit/{link_jobs/ftq.yaml => link_jobs2/job.yaml} (100%) rename spec/submit/matrix/{unixbench-spec/unixbench-spec-1-300-whetstone-double.yaml => job-1-300-whetstone-double.yaml} (100%) rename spec/submit/matrix/{unixbench-spec/unixbench-spec-3-300-shell1.yaml => job-3-300-shell1.yaml} (100%) rename spec/submit/matrix/{unixbench-spec.yaml => job.yaml} (100%) rename spec/submit/merge_yaml/{merge_included_files/merge_included_files-defaults.yaml => job-defaults.yaml} (100%) create mode 100644 spec/submit/merge_yaml/job.yaml delete mode 100644 spec/submit/merge_yaml/merge_included_files.yaml rename spec/submit/separate_yaml/{ftq/ftq-1000-2-100000ss-add.yaml => job-1000-2-100000ss-add.yaml} (100%) rename spec/submit/separate_yaml/{ftq/ftq-10000-2-100000ss-add.yaml => job-10000-2-100000ss-add.yaml} (100%) rename spec/submit/separate_yaml/{ftq/ftq-20-2-6000ss-cache.yaml => job-20-2-6000ss-cache.yaml} (100%) rename spec/submit/separate_yaml/{ftq.yaml => job.yaml} (100%) diff --git a/spec/submit/job_on_fail/job_on_fail/job_on_fail-3000.yaml b/spec/submit/job_on_fail/job-3000.yaml similarity index 100% rename from spec/submit/job_on_fail/job_on_fail/job_on_fail-3000.yaml rename to spec/submit/job_on_fail/job-3000.yaml diff --git a/spec/submit/job_on_fail/job_on_fail.yaml b/spec/submit/job_on_fail/job.yaml similarity index 97% rename from spec/submit/job_on_fail/job_on_fail.yaml rename to spec/submit/job_on_fail/job.yaml index bf68ff40..5aaedf30 100644 --- a/spec/submit/job_on_fail/job_on_fail.yaml +++ b/spec/submit/job_on_fail/job.yaml @@ -2,4 +2,5 @@ suite: atomic on_fail: sleep: 3000 + atomic: diff --git a/spec/submit/link_jobs/redis/redis-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml b/spec/submit/link_jobs1/job-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml similarity index 100% rename from spec/submit/link_jobs/redis/redis-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml rename to spec/submit/link_jobs1/job-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml diff --git a/spec/submit/link_jobs/redis.yaml b/spec/submit/link_jobs1/job.yaml similarity index 100% rename from spec/submit/link_jobs/redis.yaml rename to spec/submit/link_jobs1/job.yaml diff --git a/spec/submit/link_jobs/ftq/ftq-1000-2-100000ss-add.yaml b/spec/submit/link_jobs2/job-1000-2-100000ss-add.yaml similarity index 100% rename from spec/submit/link_jobs/ftq/ftq-1000-2-100000ss-add.yaml rename to spec/submit/link_jobs2/job-1000-2-100000ss-add.yaml diff --git a/spec/submit/link_jobs/ftq/ftq-10000-2-100000ss-add.yaml b/spec/submit/link_jobs2/job-10000-2-100000ss-add.yaml similarity index 100% rename from spec/submit/link_jobs/ftq/ftq-10000-2-100000ss-add.yaml rename to spec/submit/link_jobs2/job-10000-2-100000ss-add.yaml diff --git a/spec/submit/link_jobs/ftq/ftq-20-2-6000ss-cache.yaml b/spec/submit/link_jobs2/job-20-2-6000ss-cache.yaml similarity index 100% rename from spec/submit/link_jobs/ftq/ftq-20-2-6000ss-cache.yaml rename to spec/submit/link_jobs2/job-20-2-6000ss-cache.yaml diff --git a/spec/submit/link_jobs/ftq.yaml b/spec/submit/link_jobs2/job.yaml similarity index 100% rename from spec/submit/link_jobs/ftq.yaml rename to spec/submit/link_jobs2/job.yaml diff --git a/spec/submit/matrix/unixbench-spec/unixbench-spec-1-300-whetstone-double.yaml b/spec/submit/matrix/job-1-300-whetstone-double.yaml similarity index 100% rename from spec/submit/matrix/unixbench-spec/unixbench-spec-1-300-whetstone-double.yaml rename to spec/submit/matrix/job-1-300-whetstone-double.yaml diff --git a/spec/submit/matrix/unixbench-spec/unixbench-spec-3-300-shell1.yaml b/spec/submit/matrix/job-3-300-shell1.yaml similarity index 100% rename from spec/submit/matrix/unixbench-spec/unixbench-spec-3-300-shell1.yaml rename to spec/submit/matrix/job-3-300-shell1.yaml diff --git a/spec/submit/matrix/unixbench-spec.yaml b/spec/submit/matrix/job.yaml similarity index 100% rename from spec/submit/matrix/unixbench-spec.yaml rename to spec/submit/matrix/job.yaml diff --git a/spec/submit/merge_yaml/merge_included_files/merge_included_files-defaults.yaml b/spec/submit/merge_yaml/job-defaults.yaml similarity index 100% rename from spec/submit/merge_yaml/merge_included_files/merge_included_files-defaults.yaml rename to spec/submit/merge_yaml/job-defaults.yaml diff --git a/spec/submit/merge_yaml/job.yaml b/spec/submit/merge_yaml/job.yaml new file mode 100644 index 00000000..6d64c381 --- /dev/null +++ b/spec/submit/merge_yaml/job.yaml @@ -0,0 +1 @@ +<< : template_yaml diff --git a/spec/submit/merge_yaml/merge_included_files.yaml b/spec/submit/merge_yaml/merge_included_files.yaml deleted file mode 100644 index 8d459a32..00000000 --- a/spec/submit/merge_yaml/merge_included_files.yaml +++ /dev/null @@ -1 +0,0 @@ -<<: template_yaml diff --git a/spec/submit/separate_yaml/ftq/ftq-1000-2-100000ss-add.yaml b/spec/submit/separate_yaml/job-1000-2-100000ss-add.yaml similarity index 100% rename from spec/submit/separate_yaml/ftq/ftq-1000-2-100000ss-add.yaml rename to spec/submit/separate_yaml/job-1000-2-100000ss-add.yaml diff --git a/spec/submit/separate_yaml/ftq/ftq-10000-2-100000ss-add.yaml b/spec/submit/separate_yaml/job-10000-2-100000ss-add.yaml similarity index 100% rename from spec/submit/separate_yaml/ftq/ftq-10000-2-100000ss-add.yaml rename to spec/submit/separate_yaml/job-10000-2-100000ss-add.yaml diff --git a/spec/submit/separate_yaml/ftq/ftq-20-2-6000ss-cache.yaml b/spec/submit/separate_yaml/job-20-2-6000ss-cache.yaml similarity index 100% rename from spec/submit/separate_yaml/ftq/ftq-20-2-6000ss-cache.yaml rename to spec/submit/separate_yaml/job-20-2-6000ss-cache.yaml diff --git a/spec/submit/separate_yaml/ftq.yaml b/spec/submit/separate_yaml/job.yaml similarity index 100% rename from spec/submit/separate_yaml/ftq.yaml rename to spec/submit/separate_yaml/job.yaml diff --git a/spec/submit_spec.rb b/spec/submit_spec.rb index 4096cb73..8310d629 100644 --- a/spec/submit_spec.rb +++ b/spec/submit_spec.rb @@ -16,20 +16,18 @@ end def traverse_file(output_dir) Dir.glob("#{output_dir}/*.yaml").each do |yaml_file| - stable_yaml_file(yaml_file) + stable_yaml_file(yaml_file) unless File.basename(yaml_file) == 'job.yaml' end end -def submit_job(submit_item) - submit_yaml_path = File.join("#{LKP_SRC}/spec/submit", submit_item) - Dir.glob("#{submit_yaml_path}/*.yaml").each do |yaml_file| - output_dir = File.join(submit_yaml_path, File.basename(yaml_file, '.yaml')) +def submit_job() + Dir.glob("#{LKP_SRC}/spec/submit/*").each do |output_dir| submit_cmd = [ "#{LKP_SRC}/sbin/submit", '-o', output_dir, '-s', 'lab: spec_lab', '-s', 'testbox: vm-2p8g--spec_submit', - yaml_file + "#{output_dir}/job.yaml" ] system(*submit_cmd) traverse_file(output_dir) @@ -38,22 +36,6 @@ end describe 'submit job spec' do it 'link jobs spec' do - submit_job('link_jobs') - end - - it 'link matrix' do - submit_job('matrix') - end - - it 'separate yaml spec' do - submit_job('separate_yaml') - end - - it 'job on fail' do - submit_job('job_on_fail') - end - - it 'merge yaml' do - submit_job('merge_yaml') + submit_job() end end -- 2.23.0

1 0

[PATCH compass-ci] result-webdav/nginx.conf: expand client_max_body_size
by Lin Jiaxin 07 Dec '20

07 Dec '20

error: curl:(22) The requested URL returned error: 413 Request Entity Too Large Our biggest cgz file is 656M now, so i set client_max_body_size to 700m. /srv/initrd/pkg% find ./ -size +100M |xargs ls -alhS -rw-r--r--. 1 lkp lkp 656M Nov 4 17:52 ./nfs/openeuler/aarch64/20.03/ycsb/0.15.0-0.cgz /srv/initrd/build-pkg% find ./ -size +100M |xargs ls -alhS -rw-rw-rw-. 1 lkp lkp 383M Aug 29 04:36 ./quake2world-data-git/HEAD.cgz Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- container/result-webdav/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/result-webdav/nginx.conf b/container/result-webdav/nginx.conf index 98382bd..1ccaa92 100644 --- a/container/result-webdav/nginx.conf +++ b/container/result-webdav/nginx.conf @@ -12,7 +12,7 @@ http { listen 3080; server_name "result-webdav"; server_tokens off; - client_max_body_size 100m; + client_max_body_size 700m; access_log /tmp/access.log; error_log /tmp/error.log; -- 2.23.0

2 1

[PATCH v2 compass-ci] lib/compare_matrixes.rb: fix spell error
by Lu Kaiyi 07 Dec '20

07 Dec '20

[errors] /home/lukaiyi/compass-ci/lib/compare_matrixes.rb:482:in `block in assign_metric_change': undefined method `get_dimensions_combination' for main:Object (NoMethodError) Did you mean? get_dimentions_combination from /home/lukaiyi/compass-ci/lib/compare_matrixes.rb:476:in `each' from /home/lukaiyi/compass-ci/lib/compare_matrixes.rb:476:in `assign_metric_change' from /home/lukaiyi/compass-ci/lib/compare_matrixes.rb:461:in `get_metric_values' from /home/lukaiyi/compass-ci/lib/compare_matrixes.rb:448:in `block in compare_metrics_values' from /home/lukaiyi/compass-ci/lib/compare_matrixes.rb:447:in `each' from /home/lukaiyi/compass-ci/lib/compare_matrixes.rb:447:in `compare_metrics_values' from /home/lukaiyi/compass-ci/lib/compare.rb:102:in `compare_by_template' from /home/lukaiyi/compass-ci/sbin/compare:70:in `<main>' Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/compare_matrixes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compare_matrixes.rb b/lib/compare_matrixes.rb index 98d396f..5293989 100644 --- a/lib/compare_matrixes.rb +++ b/lib/compare_matrixes.rb @@ -495,7 +495,7 @@ end # ['openeuler 20.03', 'centos 7.6'], # ['debian sid', 'centos 7.6'] # ] -def get_dimentions_combination(dimension_list) +def get_dimensions_combination(dimension_list) dims = [] dimension_list_size = dimension_list.size (1..dimension_list_size - 1).each do |i| -- 2.23.0

1 0

[PATCH v3 compass-ci] user-client/maintain: use multi args in submit command
by Liu Yinsi 07 Dec '20

07 Dec '20

multi args changed from os os_arch os_version to os|os_arch|os_version, and if use os|os_arch|os_version, job yaml must as the first parameter. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- .../maintain/walk-os-test/walk-os-iperf-test | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/user-client/maintain/walk-os-test/walk-os-iperf-test b/user-client/maintain/walk-os-test/walk-os-iperf-test index ed97999..9f75516 100755 --- a/user-client/maintain/walk-os-test/walk-os-iperf-test +++ b/user-client/maintain/walk-os-test/walk-os-iperf-test @@ -101,8 +101,8 @@ def run_qemu end end -def submit_job(os_str, os_mount) - `submit --my-queue 'os os_arch os_version=#{os_str}' 'testbox=#{TESTBOX}' 'os_mount=#{os_mount}' #{TEST_YAML}` +def submit_job(os_args, os_mount) + `submit #{TEST_YAML} --my-queue 'os|os_arch|os_version=#{os_args}' testbox=#{TESTBOX} os_mount=#{os_mount}` end def traversal_os_mount @@ -113,12 +113,13 @@ end def traversal_test_os(os_mount) TEST_OS.each do |os_str| - test_rootfs(os_str, os_mount) + os_args = os_str.gsub(' ', '|') + test_rootfs(os_args, os_mount) end end -def test_rootfs(os_str, os_mount) - message = submit_job(os_str, os_mount).chomp +def test_rootfs(os_args, os_mount) + message = submit_job(os_args, os_mount).chomp job_id = message.split('=')[1] run_qemu @@ -128,7 +129,7 @@ def test_rootfs(os_str, os_mount) find_by_id(job_id) end - report_all = job_id.split(', ') + os_str.split + os_mount.split + job_result + report_all = job_id.split(', ') + os_args.split + os_mount.split + job_result write_report(report_all) end -- 2.23.0

1 0

[PATCH compass-ci] sparrow/0-package: add registry-mirrors
by Liu Yinsi 05 Dec '20

05 Dec '20

speed up docker pull speed and time. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/0-package/common | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sparrow/0-package/common b/sparrow/0-package/common index b381d73..57ab701 100755 --- a/sparrow/0-package/common +++ b/sparrow/0-package/common @@ -39,6 +39,10 @@ cat > /etc/docker/daemon.json <<EOF "fluentd-async-connect": "true", "tag": "{{.Name}}" }, + "registry-mirrors": [ + "http://hub-mirror.c.163.com", + "http://docker.mirrors.ustc.edu.cn" + ], "insecure-registries": ["$DOCKER_REGISTRY_HOST:$DOCKER_REGISTRY_PORT"] } EOF -- 2.23.0

1 0

[PATCH compass-ci] scheduler: cluster: add return values
by Ren Wen 05 Dec '20

05 Dec '20

Return all ips of cluster nodes, not direct ips only. Only for one role now. Example: direct_server_ip=<direct ip> server=<public ip> Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/scheduler/request_cluster_state.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/scheduler/request_cluster_state.cr b/src/scheduler/request_cluster_state.cr index 79a13cc..1782797 100644 --- a/src/scheduler/request_cluster_state.cr +++ b/src/scheduler/request_cluster_state.cr @@ -58,7 +58,9 @@ class Sched cluster_state.each_value do |host_state| roles = host_state["roles"] direct_ips = host_state["direct_ips"] - roles_ip << "#{roles}=#{direct_ips}" + node_ip = host_state["ip"] + roles_ip << "direct_#{roles}_ips=#{direct_ips}" + roles_ip << "#{roles}=#{node_ip}" end return roles_ip.join('\n') -- 2.23.0

1 0

[PATCH compass-ci] sbin/compare_template: friendly show compare template result in command line
by Lu Kaiyi 05 Dec '20

05 Dec '20

[why] friendly show compare template result in command line. [how] rehandle the json format result of "compare -t <compare_template.yaml>" with third-party module in python3. [usage] compare_template <compare_template.yaml> [example] compare_template compare_template.yaml in z9 [input] a compare_template.yaml like below: compare_metrics: - fio.write_iops - fio.read_iops filter: suite: - fio-basic os_arch: - aarch64 - x86 compare_dimensions: - os: debian os_version: sid - os: openeuler os_version: 20.03 x_params: - bs - test_size title: Hackbench Performance Testing unit: KB/s [output] Hackbench Performance Testing +------------------------------------+-----------+----------+----------+----------+----------+---------+---------+---------+----------+ | fio.read_iops | 4k|1G | 4k|80G | 16k|1G | 32k|1G | 64k|1G | 128k|1G | 256k|1G | 512k|1G | 1024k|1G | +------------------------------------+-----------+----------+----------+----------+----------+---------+---------+---------+----------+ | openeuler 20.03 average | 144076.29 | 11601.10 | 37865.30 | 21145.10 | 14010.34 | 6701.24 | 3205.08 | 1367.48 | 673.33 | | openeuler 20.03 standard_deviation | 195.00 | 0.00 | 214.00 | 205.00 | 188.00 | 183.00 | 180.00 | 191.00 | 191.00 | +------------------------------------+-----------+----------+----------+----------+----------+---------+---------+---------+----------+ Hackbench Performance Testing +------------------------------------+-----------+----------+----------+----------+---------+---------+---------+----------+ | fio.write_iops | 4k|1G | 16k|1G | 32k|1G | 64k|1G | 128k|1G | 256k|1G | 512k|1G | 1024k|1G | +------------------------------------+-----------+----------+----------+----------+---------+---------+---------+----------+ | openeuler 20.03 average | 122003.54 | 33528.53 | 31469.06 | 13870.14 | 8249.71 | 4329.45 | 1976.54 | 1141.00 | | openeuler 20.03 standard_deviation | 174.00 | 188.00 | 171.00 | 197.00 | 181.00 | 175.00 | 170.00 | 176.00 | +------------------------------------+-----------+----------+----------+----------+---------+---------+---------+----------+ Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- sbin/compare_template | 53 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100755 sbin/compare_template diff --git a/sbin/compare_template b/sbin/compare_template new file mode 100755 index 0000000..6b702c0 --- /dev/null +++ b/sbin/compare_template @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +import os +import sys +import json +import prettytable as pt + +# receive compare_template.yaml and auto pretty show compare results +class TableShow: + def __init__(self, result_dict): + self.title = result_dict['title'] + self.tables = result_dict['tables'] + self.tb = None + + def show_table(self): + for (table_title, table) in self.tables.items(): + self.tb = pt.PrettyTable() + self.set_field_names(table, table_title) + self.set_align(table_title) + self.add_row(table) + self.print_table() + + def set_field_names(self, table, table_title): + field_names = [table_title] + field_names.extend(table['average']['source'][0]) + self.tb.field_names = field_names + + def set_align(self, table_title): + for field_name in self.tb.field_names: + self.tb.align[field_name] = 'r' + self.tb.align[table_title] = 'l' + + def add_row(self, table): + row_names = ['average', 'standard_deviation'] + for row_name in row_names: + row = table[row_name]['source'][1] + row_title = ' '.join([row[0], row_name]) + format_data_row = ["%.2f" % data for data in row[1:]] + self.tb.add_row([row_title, *format_data_row]) + + def print_table(self): + print(self.title) + print(self.tb) + print() + + +if __name__ == '__main__': + template_yaml = sys.argv[1] + result_dict = json.loads(os.popen(f"compare -t {template_yaml}").read()) + table_show = TableShow(result_dict) + table_show.show_table() -- 2.23.0

1 0

[PATCH lkp-tests] stats/sysbench-threads-git: refactor regex match
by Lu Kaiyi 04 Dec '20

04 Dec '20

Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- stats/sysbench-threads-git | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stats/sysbench-threads-git b/stats/sysbench-threads-git index afe5d664..440eb798 100755 --- a/stats/sysbench-threads-git +++ b/stats/sysbench-threads-git @@ -8,7 +8,7 @@ # min: 0.04ms $stdin.each_line do |line| - next unless line =~ /^\s+total time:\s+([0-9.]+)/ + next unless line =~ /^\s+total\stime:\s+(\S+)/ - puts "total time: #{$1.to_f}s" + puts "total time: #{$1}" end -- 2.23.0

1 0