November 2020 - Compass-ci - mailweb.openeuler.org

[PATCH v2 lkp-tests 6/6] iso2rootfs: allow the use of job.yaml and test script from the internet
by Yu Chuan 19 Nov '20

19 Nov '20

[Why] Not all the yaml and script of test job will be committed to the lkp-tests repo. [How] Support three optional fields for every test in job.yaml: - testX_git_url: the url of git repo which contains yaml and script; - testX_git_yaml: the relative path of the yaml file in the git repo; - testX_git_script: the relative path of the test script in the git repo; Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 5 +++++ tests/iso2rootfs | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index f88ff02ad592..29bf84013f8e 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -26,3 +26,8 @@ iso2rootfs: test1_yaml: test1_os_mount: test1_testbox: + ## follow three fields is required when your job.yaml and script for + ## this test are from the internet. + test1_git_url: + test1_git_yaml: + test1_git_script: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index 48c1fc772890..e30cde9af1e6 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -451,10 +451,36 @@ generate_submit_append_str() echo "os=${iso_os} os_arch=${iso_arch} os_version=${iso_version} os_mount=$1 testbox=$2" } +prepare_submit() +{ + local submit_dir="/tmp/iso2rootfs/submit_tmp" + mkdir -p "${submit_dir}" + + local submit_git_dir="${submit_dir}/${test_git_url##*/}" + submit_git_dir="${submit_git_dir%.git}" + + [ -d "${submit_git_dir}" ] || { + cd "${submit_dir}" + git clone "${test_git_url}" + } + + local test_git_yaml="${submit_git_dir}/$(eval echo "\$${test_num}_git_yaml")" + local test_git_script="${submit_git_dir}/$(eval echo "\$${test_num}_git_script")" + + [ -f "${test_git_yaml}" ] || die "cannot find file: ${test_num}_git_yaml." + cp "${test_git_yaml}" "${LKP_SRC}/jobs/" + + [ -f "${test_git_script}" ] || die "cannot find file: ${test_num}_git_script." + cp "${test_git_script}" "${LKP_SRC}/tests/" +} + submit_one_yaml() { log_info "starting submit ${test_yaml} ..." + local test_git_url="$(eval echo "\$${test_num}_git_url")" + [ -z "${test_git_url}" ] || prepare_submit + cd "${LKP_SRC}/jobs" [ -f "${test_yaml}" ] || die "cannot find yaml in LKP_SRC/jobs: ${test_yaml}." -- 2.23.0

1 0

[PATCH v2 lkp-tests 5/6] iso2rootfs: allow submit multiple test job.yaml for result rootfs
by Yu Chuan 19 Nov '20

19 Nov '20

[Why] Result rootfs of dailybuild requires multiple test jobs. [How] 1. jobs/iso2rootfs.yaml. Before: test_yaml, test_os_mount, test_testbox After: test1_yaml, test1_os_mount, test1_testbox test2_yaml, test2_os_mount, test2_testbox ... 2. tests/iso2rootfs. - Use while-loop to traverse if $testX_yaml exists. - [ -n "${test1_yaml}" ] || return 0. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 8 ++++--- tests/iso2rootfs | 52 +++++++++++++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 222ac309d33e..f88ff02ad592 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -21,6 +21,8 @@ iso2rootfs: dailybuild_iso_url_file: # submit test yaml related fields - test_yaml: - test_os_mount: - test_testbox: + ## 1. you can add as many jobs as you like. + ## 2. the following three fields is requied for every test job. + test1_yaml: + test1_os_mount: + test1_testbox: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index 4f6e3e578ce4..48c1fc772890 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -56,7 +56,7 @@ check_yaml_vars() for yaml_t in "${yaml_vars[@]}" do - [ -n "$(eval echo "\$${yaml_t}")" ] || die "cannot fine value of var: ${yaml_t}." + [ -n "$(eval echo "\$${yaml_t}")" ] || die "cannot find value of var: ${yaml_t}." done } @@ -451,17 +451,33 @@ generate_submit_append_str() echo "os=${iso_os} os_arch=${iso_arch} os_version=${iso_version} os_mount=$1 testbox=$2" } -submit_test_job_yaml() +submit_one_yaml() { - [ -n ${test_yaml} ] || return 0 - log_info "starting submit ${test_yaml} ..." - [ -d "${LKP_SRC}" ] || die "cannot find value of var: LKP_SRC." + cd "${LKP_SRC}/jobs" + [ -f "${test_yaml}" ] || die "cannot find yaml in LKP_SRC/jobs: ${test_yaml}." + + local test_os_mount="$(eval echo "\$${test_num}_os_mount")" + local test_testbox="$(eval echo "\$${test_num}_testbox")" + + [ -n "${test_os_mount}" ] || die "cannot find value of var: ${test_num}_os_mount." + [ -n "${test_testbox}" ] || die "cannot find value of var: ${test_num}_testbox." - [ -n "${test_os_mount}" ] || die "cannot fine value of var: test_os_mount." - [ -n "${test_testbox}" ] || die "cannot fine value of var: test_testbox." + "${LKP_SRC}/sbin/submit" \ + $(generate_submit_append_str "${test_os_mount}" "${test_testbox}") \ + "${test_yaml}" + log_info "submit ${test_yaml} finished" +} + +submit_yamls() +{ + [ -n "$test1_yaml" ] || return 0 + + log_info "starting submit test yamls ..." + + [ -d "${LKP_SRC}" ] || die "cannot find value of var: LKP_SRC." cd "${LKP_SRC}" && { chmod +x ./sbin/install-dependencies.sh ./sbin/install-dependencies.sh @@ -469,17 +485,19 @@ submit_test_job_yaml() generate_global_yaml } - cd "${LKP_SRC}/jobs" && { - [ -f "${test_yaml}" ] || - die "cannot find test yaml in LKP_SRC/jobs. - test yaml: ${test_yaml}." + local test_yaml_index=1 + local test_num="test${test_yaml_index}" + local test_yaml="$(eval echo "\$${test_num}_yaml")" - "${LKP_SRC}/sbin/submit" \ - $(generate_submit_append_str "${test_os_mount}" "${test_testbox}") \ - "${test_yaml}" - } + while [ -n "${test_yaml}" ] + do + submit_one_yaml + test_yaml_index=$((${test_yaml_index} +1)) + test_num="test${test_yaml_index}" + test_yaml="$(eval echo "\$${test_num}_yaml")" + done - log_info "submit ${test_yaml} finished" + log_info "submit test yamls finished" } ############ post works ############ @@ -519,7 +537,7 @@ main() config_rootfs test_rootfs - submit_test_job_yaml + submit_yamls post_works } -- 2.23.0

1 0

[PATCH v2 lkp-tests 3/6] iso2rootfs: add a switch to configure rootfs
by Yu Chuan 19 Nov '20

19 Nov '20

[Why] In some cases, we do not need to configure the result rootfs, so add a switch to configure rootfs. [How] Add 'config_rootfs' field in job.yaml. and support the following two fields now: - dns: will config /etc/resolv.conf. - no_selinux: will disable selinux in /etc/selinux/config. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 6 ++++++ tests/iso2rootfs | 2 ++ 2 files changed, 8 insertions(+) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 8796874ce79d..55bf71517174 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -11,6 +11,12 @@ iso2rootfs: initrd_http_host: initrd_http_port: + # config rootfs related fields + ## supportted fields: + ## - dns: will config /etc/resolv.conf. + ## - no_selinux: will disable selinux in /etc/selinux/config. + config_rootfs: + # dailybuild related fields dailybuild_iso_url_file: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index 5f85cc9db01e..c2e823c93ff9 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -345,6 +345,7 @@ run_qcow2rootfs() ############ config rootfs ############ config_dns_resolver() { + echo "${config_rootfs}" | grep -qw 'dns' || return 0 cat <<-EOF > "${ROOTFS_DES_DIR}/etc/resolv.conf" nameserver 114.114.114.114 nameserver 8.8.8.8 @@ -353,6 +354,7 @@ config_dns_resolver() disable_selinux() { + echo "${config_rootfs}" | grep -qw 'no_selinux' || return 0 sed -i 's/^SELINUX=.*/SELINUX=disabled/g' "${ROOTFS_DES_DIR}/etc/selinux/config" } -- 2.23.0

1 0

[PATCH v6 compass-ci] sparrow: fix 3-code/dev-env no such file error
by Liu Yinsi 19 Nov '20

19 Nov '20

[why] when execute 3-code/dev-env in openeuler and centos machine, error message as follows: ./dev-env: line 45: ./os/openEuler: No such file or directory the purpose of ". $(dirname ${BASH_SOURCE[0]})/os/${ID}" is to fix crystal-shards build error when adapt in debian. but it's unnecessary for other linux os. Please see: [PATCH compass-ci 3/4] sparrow: add domain name resolution for Docker images [how] add judge to avoid redundant error message. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 8928816..218b6c1 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -44,4 +44,6 @@ EOF source /etc/os-release -. $(dirname ${BASH_SOURCE[0]})/os/${ID} +path=$(dirname ${BASH_SOURCE[0]})/os/${ID} +[ -x "$path" ] || exit 0 +. "$path" -- 2.23.0

1 0

[PATCH v2 compass-ci] lib/es-query: fix NoMethodError
by Lu Weitao 19 Nov '20

19 Nov '20

[Why] Traceback (most recent call last): 4: from /c/compass-ci/sbin/compare:70:in `<main>' 3: from /c/compass-ci/lib/compare.rb:97:in `compare_by_template' 2: from /c/compass-ci/lib/compare.rb:128:in `create_groups_matrices' 1: from /c/compass-ci/lib/es_query.rb:21:in `multi_field_query' /c/compass-ci/lib/es_query.rb:54:in `build_mutli_field_subquery_body': undefined method `each' for nil:NilClass (NoMethodError) "compare_temlpate.yaml" may pass empty filter as input parameter [How] exit when the filter is empty Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/es_query.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/es_query.rb b/lib/es_query.rb index 037213d..c027aa9 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -18,6 +18,10 @@ class ESQuery # Example @items: { key1 => value1, key2 => [value2, value3, ..], ...} # means to query: key1 == value1 && (key2 in [value2, value3, ..]) def multi_field_query(items, size: 10_000) + unless items + warn 'empty filter!' + exit + end query_fields = build_mutli_field_subquery_body items query = { query: { -- 2.23.0

2 2

[PATCH v2 compass-ci 2/2] job.cr: give a default kernel_version and
by Xu Xijian 19 Nov '20

19 Nov '20

The versions of modules and headers should be same with vmlinuz, so give kernel_version a default value and delete soft link modules.cgz and headers.cgz in $nfsroot/boot, they are redundant. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/job.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 4bef329..fc27c5c 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -46,6 +46,7 @@ class Job os_version: "20.03", lkp_initrd_user: "latest", docker_image: "centos:7", + kernel_version: "4.19.90-2003" } def initialize(job_content : JSON::Any, id) @@ -312,7 +313,7 @@ class Job private def set_kernel_version boot_dir = "#{SRV_OS}/#{os_dir}/boot" - suffix = "-#{kernel_version}" if self["kernel_version"]? + suffix = "-#{kernel_version}" self["linux_vmlinuz_path"] = File.real_path("#{boot_dir}/vmlinuz#{suffix}") if "#{os_mount}" == "initramfs" self["linux_modules_initrd"] = File.real_path("#{boot_dir}/modules#{suffix}.cgz") -- 2.23.0

2 2

[PATCH lkp-tests 3/3] add the case of yaml_merge_included_files.yaml
by Hu Xuejiao 19 Nov '20

19 Nov '20

[description] it is situation of yaml_merge_included_files Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- spec/submit/merge_yaml/template.yaml | 3 +++ spec/submit/merge_yaml/yaml_merge_included_files.yaml | 1 + 2 files changed, 4 insertions(+) create mode 100644 spec/submit/merge_yaml/template.yaml create mode 100644 spec/submit/merge_yaml/yaml_merge_included_files.yaml diff --git a/spec/submit/merge_yaml/template.yaml b/spec/submit/merge_yaml/template.yaml new file mode 100644 index 00000000..0415606a --- /dev/null +++ b/spec/submit/merge_yaml/template.yaml @@ -0,0 +1,3 @@ +suite: test +category: functional +test: diff --git a/spec/submit/merge_yaml/yaml_merge_included_files.yaml b/spec/submit/merge_yaml/yaml_merge_included_files.yaml new file mode 100644 index 00000000..c291508d --- /dev/null +++ b/spec/submit/merge_yaml/yaml_merge_included_files.yaml @@ -0,0 +1 @@ +<<: template.yaml -- 2.23.0

3 2

[PATCH lkp-tests 1/3] create merge_yaml and submit merge_yaml/*.yaml
by Hu Xuejiao 19 Nov '20

19 Nov '20

[why] We will add more spec cases, so it should give each type of cases for different input and output directories [how] Create merge_yaml and submit merge_yaml/*.yaml Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- spec/submit_spec.rb | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/spec/submit_spec.rb b/spec/submit_spec.rb index 983d050e..b2ad7168 100644 --- a/spec/submit_spec.rb +++ b/spec/submit_spec.rb @@ -34,6 +34,21 @@ describe 'submit job spec' do ] system(*submit_cmd) traverse_file(output_dir) + end + end + it 'merge yaml spec' do + Dir.glob("#{LKP_SRC}/spec/submit/merge_yaml/*.yaml").each do |yaml_file| + job_name = File.basename(yaml_file, '.yaml') + output_dir = "#{LKP_SRC}/spec/submit/merge_yaml/#{job_name}/output" + submit_cmd = [ + "#{LKP_SRC}/sbin/submit", + '-o', output_dir, + '-s', 'lab: spec_lab', + '-s', 'testbox: vm-2p8g--spec_submit', + yaml_file + ] + system(*submit_cmd) + traverse_file(output_dir) + end end - end end -- 2.23.0

2 1

[PATCH lkp-tests] jobs: fix the spell error in the original lmbench3.ayml
by Bai Jing 19 Nov '20

19 Nov '20

Signed-off-by: Bai Jing <799286817(a)qq.com> --- jobs/lmbench3.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/lmbench3.yaml b/jobs/lmbench3.yaml index de1c9aa8..f84c6910 100644 --- a/jobs/lmbench3.yaml +++ b/jobs/lmbench3.yaml @@ -4,7 +4,7 @@ category: benchmark test_memory_size: 50% -# Test mdoe all,hardware,os were commented temporally,since it will take very long time +# Test mode all, hardware, os were commented temporally, since it will take very long time # more than 2 hours on test machine lkp-bdw-ep4. # lmbench3: # - mode: all -- 2.23.0

2 2

[PATCH v2 lkp-tests] jobs: add new job yaml for lmbench3 performance test
by Bai Jing 19 Nov '20

19 Nov '20

Signed-off-by: Bai Jing <799286817(a)qq.com> --- jobs/lmbench3-performance.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 jobs/lmbench3-performance.yaml diff --git a/jobs/lmbench3-performance.yaml b/jobs/lmbench3-performance.yaml new file mode 100644 index 00000000..c733d7d2 --- /dev/null +++ b/jobs/lmbench3-performance.yaml @@ -0,0 +1,17 @@ +suite: lmbench3 +category: benchmark + +test_memory_size: 4096 + +# Test mdoe all,hardware,os were commented temporally,since it will take very long time +# more than 2 hours on test machine lkp-bdw-ep4. +# lmbench3: +# - mode: all +# - mode: hardware +# - mode: os + +nr_threads: +- 1 + +lmbench3: + mode: all -- 2.23.0

4 6