- Compass-ci - mailweb.openeuler.org

[PATCH v4 lkp-tests 2/2] stats/crystal: mysql-server to mysql-server.cr
by Lu Kaiyi 26 Nov '20

26 Nov '20

Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- stats/crystal/mysql-server.cr | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 stats/crystal/mysql-server.cr diff --git a/stats/crystal/mysql-server.cr b/stats/crystal/mysql-server.cr new file mode 100644 index 00000000..99b24b67 --- /dev/null +++ b/stats/crystal/mysql-server.cr @@ -0,0 +1,20 @@ +#!/usr/bin/env crystal + +results = [] of String + +while (line = STDIN.gets) + case line + # The output is as below: + # 200706 14:35:48 [ 99%] main.subquery_sj_innodb_all w8 [ pass ] 140 + # 200706 14:35:52 [ 99%] main.ssl_dynamic_persisted w1 [ fail ] 5953 + # 200706 14:35:59 [ 99%] main.mysql_upgrade_grant w7 [ skipped ] 80805 + when /\[\s*\d+%\]/ + results << line.split(']')[1].split[0] + ".pass: 1" if line.includes?("[ pass ]") + results << line.split(']')[1].split[0] + ".fail: 1" if line.includes?("[ fail ]") + results << line.split(']')[1].split[0] + ".skip: 1" if line.includes?("[ skipped ]") + end +end + +results.each do |item| + puts item +end -- 2.23.0

1 0

[PATCH v4 lkp-tests 1/2] stats/mysql-server: remove redundant whitespace
by Lu Kaiyi 26 Nov '20

26 Nov '20

Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- stats/mysql-server | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/stats/mysql-server b/stats/mysql-server index 037d1f0f..ca31f9ea 100755 --- a/stats/mysql-server +++ b/stats/mysql-server @@ -1,5 +1,4 @@ #!/usr/bin/env ruby -# frozen_string_literal: true LKP_SRC = ENV['LKP_SRC'] || File.dirname(File.dirname(File.reapath($PROGRAM_NAME))) @@ -16,9 +15,9 @@ while (line = STDIN.gets) # 200706 14:35:52 [ 99%] main.ssl_dynamic_persisted w1 [ fail ] 5953 # 200706 14:35:59 [ 99%] main.mysql_upgrade_grant w7 [ skipped ] 80805 when /\[\s*\d+%\]/ - results << line.split(']')[1].split[0] + '.pass: 1' if line.include?('[ pass ]') - results << line.split(']')[1].split[0] + '.fail: 1' if line.include?('[ fail ]') - results << line.split(']')[1].split[0] + '.skip: 1' if line.include?('[ skipped ]') + results << line.split(']')[1].split[0] + '.pass: 1' if line.include?('[ pass ]') + results << line.split(']')[1].split[0] + '.fail: 1' if line.include?('[ fail ]') + results << line.split(']')[1].split[0] + '.skip: 1' if line.include?('[ skipped ]') end end -- 2.23.0

1 0

[PATCH v2 compass-ci 1/5] openeuler: the link for one command to build openeuler initramfs image
by Wang Chenglong 26 Nov '20

26 Nov '20

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- rootfs/initramfs/openeuler/aarch64/20.03-LTS/build | 1 + rootfs/initramfs/openeuler/aarch64/20.09/build | 1 + 2 files changed, 2 insertions(+) create mode 120000 rootfs/initramfs/openeuler/aarch64/20.03-LTS/build create mode 120000 rootfs/initramfs/openeuler/aarch64/20.09/build diff --git a/rootfs/initramfs/openeuler/aarch64/20.03-LTS/build b/rootfs/initramfs/openeuler/aarch64/20.03-LTS/build new file mode 120000 index 0000000..f04acac --- /dev/null +++ b/rootfs/initramfs/openeuler/aarch64/20.03-LTS/build @@ -0,0 +1 @@ +../../../../../container/osimage/openeuler/run \ No newline at end of file diff --git a/rootfs/initramfs/openeuler/aarch64/20.09/build b/rootfs/initramfs/openeuler/aarch64/20.09/build new file mode 120000 index 0000000..f04acac --- /dev/null +++ b/rootfs/initramfs/openeuler/aarch64/20.09/build @@ -0,0 +1 @@ +../../../../../container/osimage/openeuler/run \ No newline at end of file -- 2.23.0

2 2

[PATCH v2 compass-ci] web_backend.rb: fix : compare_groups_matrices needs arg suites_list
by Li Yuanchao 26 Nov '20

26 Nov '20

compare_groups_matrices needs three args, but now only get two, lost suites_list. divide method to pass rubocop limit Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 1f7224e..b5b06b4 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -147,6 +147,20 @@ def get_es_must(params) must end +def _get_group_matrices(query, dimension) + result = es_query(query) + matrices, suites_list = combine_group_query_data(result, dimension) + while matrices.empty? + from += size + break if from > total + + query[:from] = from + result = es_query(query) + matrices, suites_list = combine_group_query_data(result, dimension) + end + [matrices, suites_list] +end + def get_groups_matrices(conditions, dimension, must, size, from) must += build_mutli_field_subquery_body(conditions) count_query = { query: { bool: { must: must } } } @@ -166,27 +180,18 @@ def get_groups_matrices(conditions, dimension, must, size, from) }] } - result = es_query(query) - matrices = combine_group_query_data(result, dimension) - while matrices.empty? - from += size - break if from > total - - query[:from] = from - result = es_query(query) - matrices = combine_group_query_data(result, dimension) - end - matrices + matrices, suites_list = _get_groups_matrices(query, dimension) + [matrices, suites_list] end def get_compare_body(params) dimension, conditions = get_dimension_conditions(params) must = get_es_must(params) - groups_matrices = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) + groups_matrices, suites_list = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) if !groups_matrices || groups_matrices.empty? body = 'No Data.' else - body = compare_group_matrices(groups_matrices, { no_print: true }) + body = compare_group_matrices(groups_matrices, suites_list, { no_print: true }) body = 'No Difference.' if !body || body.empty? end return body -- 2.23.0

2 1

[PATCH v5 lkp-tests] sbin/submit: submit merge_yaml/*.yaml
by Hu Xuejiao 26 Nov '20

26 Nov '20

We will use 'rspec spec/submit_spec.rb' to merge job.yaml, it support merging from external file by "<<: job.yaml" Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- spec/submit/merge_yaml/template.yaml | 3 +++ .../template/template-defaults.yaml | 19 +++++++++++++++++++ .../merge_yaml/yaml_merge_included_files.yaml | 1 + .../yaml_merge_included_files-defaults.yaml | 19 +++++++++++++++++++ spec/submit_spec.rb | 4 ++++ 5 files changed, 46 insertions(+) create mode 100644 spec/submit/merge_yaml/template.yaml create mode 100644 spec/submit/merge_yaml/template/template-defaults.yaml create mode 100644 spec/submit/merge_yaml/yaml_merge_included_files.yaml create mode 100644 spec/submit/merge_yaml/yaml_merge_included_files/yaml_merge_included_files-defaults.yaml diff --git a/spec/submit/merge_yaml/template.yaml b/spec/submit/merge_yaml/template.yaml new file mode 100644 index 00000000..331cb470 --- /dev/null +++ b/spec/submit/merge_yaml/template.yaml @@ -0,0 +1,3 @@ +suite: atomic + +atomic: diff --git a/spec/submit/merge_yaml/template/template-defaults.yaml b/spec/submit/merge_yaml/template/template-defaults.yaml new file mode 100644 index 00000000..42b53c13 --- /dev/null +++ b/spec/submit/merge_yaml/template/template-defaults.yaml @@ -0,0 +1,19 @@ +--- +suite: atomic +atomic: +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + atomic: + cpufreq_governor: +monitors: diff --git a/spec/submit/merge_yaml/yaml_merge_included_files.yaml b/spec/submit/merge_yaml/yaml_merge_included_files.yaml new file mode 100644 index 00000000..c291508d --- /dev/null +++ b/spec/submit/merge_yaml/yaml_merge_included_files.yaml @@ -0,0 +1 @@ +<<: template.yaml diff --git a/spec/submit/merge_yaml/yaml_merge_included_files/yaml_merge_included_files-defaults.yaml b/spec/submit/merge_yaml/yaml_merge_included_files/yaml_merge_included_files-defaults.yaml new file mode 100644 index 00000000..42b53c13 --- /dev/null +++ b/spec/submit/merge_yaml/yaml_merge_included_files/yaml_merge_included_files-defaults.yaml @@ -0,0 +1,19 @@ +--- +suite: atomic +atomic: +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + atomic: + cpufreq_governor: +monitors: diff --git a/spec/submit_spec.rb b/spec/submit_spec.rb index 169d7812..d71ed891 100644 --- a/spec/submit_spec.rb +++ b/spec/submit_spec.rb @@ -48,4 +48,8 @@ describe 'submit job spec' do it 'separate yaml spec' do submit_job('separate_yaml') end + + it 'merge yaml spec' do + submit_job('merge_yaml') + end end -- 2.23.0

1 0

[PATCH compass-ci] job.cr: delete redundant "#{}" for string
by Xu Xijian 26 Nov '20

26 Nov '20

JobHelper.service_path() will return string value, so "#{}" is unnecessary for it. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/job.cr | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 378fd00..aa7ffc3 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -366,14 +366,14 @@ class Job private def set_kernel_uri self["kernel_uri"] = "kernel #{OS_HTTP_PREFIX}" + - "#{JobHelper.service_path("#{linux_vmlinuz_path}")}" + JobHelper.service_path("#{linux_vmlinuz_path}") end private def common_initrds temp_initrds = [] of String temp_initrds << "#{INITRD_HTTP_PREFIX}" + - "#{JobHelper.service_path("#{SRV_INITRD}/lkp/#{lkp_initrd_user}/lkp-#{os_arch}.cgz")}" + JobHelper.service_path("#{SRV_INITRD}/lkp/#{lkp_initrd_user}/lkp-#{os_arch}.cgz") temp_initrds << "#{SCHED_HTTP_PREFIX}/job_initrd_tmpfs/#{id}/job.cgz" return temp_initrds @@ -384,13 +384,13 @@ class Job osimage_dir = "#{SRV_INITRD}/osimage/#{os_dir}" temp_initrds << "#{INITRD_HTTP_PREFIX}" + - "#{JobHelper.service_path("#{osimage_dir}/current")}" + JobHelper.service_path("#{osimage_dir}/current") temp_initrds << "#{INITRD_HTTP_PREFIX}" + - "#{JobHelper.service_path("#{osimage_dir}/run-ipconfig.cgz")}" + JobHelper.service_path("#{osimage_dir}/run-ipconfig.cgz") temp_initrds << "#{OS_HTTP_PREFIX}" + - "#{JobHelper.service_path(self["linux_modules_initrd"])}" + JobHelper.service_path(self["linux_modules_initrd"]) temp_initrds << "#{OS_HTTP_PREFIX}" + - "#{JobHelper.service_path(self["linux_headers_initrd"])}" + JobHelper.service_path(self["linux_headers_initrd"]) temp_initrds.concat(initrd_deps.split(/ /)) unless initrd_deps.empty? temp_initrds.concat(initrd_pkg.split(/ /)) unless initrd_pkg.empty? @@ -402,7 +402,7 @@ class Job temp_initrds = [] of String temp_initrds << "#{OS_HTTP_PREFIX}" + - "#{JobHelper.service_path("#{SRV_OS}/#{os_dir}/initrd.lkp")}" + JobHelper.service_path("#{SRV_OS}/#{os_dir}/initrd.lkp") return temp_initrds end -- 2.23.0

2 1

[PATCH v2 compass-ci 5/5] openeuler: build openeuler initramfs image with one command
by Wang Chenglong 26 Nov '20

26 Nov '20

[Why] Use the tool to create a new openeuler-$version initramfs image with one command, and we also can customize the pre-installed software by configuring ./packages-to-install. [How] We create a link button for the user to use. $CCI_SRC/rootfs/initramfs/openeuler/aarch64/${os_version}/build --- container/osimage/openeuler/run | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 container/osimage/openeuler/run diff --git a/container/osimage/openeuler/run b/container/osimage/openeuler/run new file mode 100755 index 0000000..dde9985 --- /dev/null +++ b/container/osimage/openeuler/run @@ -0,0 +1,28 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. ${CCI_SRC}/container/osimage/openeuler/lib +. ${CCI_SRC}/container/defconfig.sh + +export_root_passwd +DIR=$(dirname $(realpath $0)) + +cmd=( + docker run + --name init_docker + -v $DIR/create-image:/root/bin/create-image + -v $DIR/packages-to-install:/tmp/packages-to-install + -v $DIR/files-to-exclude:/tmp/files-to-exclude + -v $DIR/${image}.repo:/etc/yum.repos.d/openEuler.repo + -e ROOT_PASSWD=$ROOT_PASSWD + -e IMAGE_NAME=$IMAGE_NAME + $image + /root/bin/create-image +) + +load_docker_image +"${cmd[@]}" +cp_image_to_host +docker_rm init_docker &> /dev/null +echo "build finished" -- 2.23.0

1 0

[PATCH v2 compass-ci 4/5] openeuler: generate the cgz file in container
by Wang Chenglong 26 Nov '20

26 Nov '20

Install packages, set rootfs, pack images package and other actions in docker. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/osimage/openeuler/create-image | 37 ++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 container/osimage/openeuler/create-image diff --git a/container/osimage/openeuler/create-image b/container/osimage/openeuler/create-image new file mode 100755 index 0000000..7402b2b --- /dev/null +++ b/container/osimage/openeuler/create-image @@ -0,0 +1,37 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +install_packages() +{ + yum -y install --skip-broken $(</tmp/packages-to-install) + yum clean all +} + +setup_rootfs() +{ + ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + rm -f /etc/machine-id /var/lib/dbus/machine-id + systemd-machine-id-setup +} + +setup_login() { + [ -n "$ROOT_PASSWD" ] || return + echo "Changing root password" + passwd_md5=$(openssl passwd -1 "$ROOT_PASSWD") + sed -i -r "s/^root:[^:]*:(.*)/root:${passwd_md5//\//\\/}:\1/" "$ROOTFS_DIR/etc/shadow" + + sed -i 's/[# ]PermitRootLogin.*/PermitRootLogin yes/' "$ROOTFS_DIR/etc/ssh/sshd_config" +} + +pack_cgz() +{ + echo "Packing package. Please wait." + find / ! -path "/tmp/${IMAGE_NAME}" | grep -vf /tmp/files-to-exclude | cpio -o -Hnewc | gzip -9 > /tmp/${IMAGE_NAME} + chmod 644 /tmp/${IMAGE_NAME} +} + +install_packages +setup_rootfs +setup_login +pack_cgz -- 2.23.0

1 0

[PATCH v2 compass-ci 3/5] openeuler: the function executed in host
by Wang Chenglong 26 Nov '20

26 Nov '20

Generate the root password, download and load the Docker image, and copy the image package to the host. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/osimage/openeuler/lib | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 container/osimage/openeuler/lib diff --git a/container/osimage/openeuler/lib b/container/osimage/openeuler/lib new file mode 100755 index 0000000..507b5b5 --- /dev/null +++ b/container/osimage/openeuler/lib @@ -0,0 +1,45 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +DIR="$(pwd)" +os_name="$(echo $DIR | awk -F "/" '{print $(NF - 2)}')" +os_version="$(echo $DIR | awk -F "/" '{print $NF}')" +docker_url="https://repo.openeuler.org/openEuler-${os_version}/docker_img/aarch64/openE…" +docker_tar_file="openEuler-${os_version}-docker.aarch64.tar.xz" +image=$(echo "${os_name}-${os_version,,}") +IMAGE_NAME="${os_name}-${os_version}-$(date +"%Y%m%d").cgz" + +export_root_passwd() +{ + local root_pwd_file="$HOME/.config/compass-ci/rootfs-passwd" + [ -f "$root_pwd_file" ] || { + echo "[INFO] Please set the password file." + echo "$HOME/.config/compass-ci/rootfs-passwd" + exit 1 + } + + export ROOT_PASSWD=$(cat "$root_pwd_file") +} + +load_docker_image() +{ + if [[ "$os_version" == '20.09' || "$os_version" == '20.03-LTS' ]]; then + [ -f "$HOME/$docker_tar_file" ] || wget -O "$HOME/$docker_tar_file" "$docker_url" + fi + + docker load -i "$HOME/$docker_tar_file" + + if [ $? = 0 ]; then + echo "finish downloading image" + else + echo "[ERROR] Pull docker image failed." + exit 1 + fi +} + +cp_image_to_host() +{ + docker cp -a init_docker:/tmp/${IMAGE_NAME} $HOME/ + echo "result: $(ls $HOME/${IMAGE_NAME})" +} -- 2.23.0

1 0

[PATCH lkp-tests] spec/submit_spec.rb: submit debug_yaml/job_fail.yaml
by Hu Xuejiao 26 Nov '20

26 Nov '20

[why] We will use 'rspec spec/submit_spec.rb' to debug job.yaml, and after the task is failed, it is be waiting state. Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- spec/submit/debug_yaml/job_fail.yaml | 6 +++++ .../debug_yaml/job_fail/job_fail-3000.yaml | 22 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 spec/submit/debug_yaml/job_fail.yaml create mode 100644 spec/submit/debug_yaml/job_fail/job_fail-3000.yaml diff --git a/spec/submit/debug_yaml/job_fail.yaml b/spec/submit/debug_yaml/job_fail.yaml new file mode 100644 index 00000000..86967162 --- /dev/null +++ b/spec/submit/debug_yaml/job_fail.yaml @@ -0,0 +1,6 @@ +suite: atomic + +on_fail: + sleep: 3000 + +atomic: diff --git a/spec/submit/debug_yaml/job_fail/job_fail-3000.yaml b/spec/submit/debug_yaml/job_fail/job_fail-3000.yaml new file mode 100644 index 00000000..3d7ad9b5 --- /dev/null +++ b/spec/submit/debug_yaml/job_fail/job_fail-3000.yaml @@ -0,0 +1,22 @@ +--- +suite: atomic +on_fail: + sleep: 3000 +atomic: +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + sleep: 3000 + atomic: + cpufreq_governor: +monitors: -- 2.23.0

2 1

[PATCH compass-ci 2/5] openeuler: the function executed in host
by Wang Chenglong 26 Nov '20

26 Nov '20

Generate the root password, download and load the Docker image, and copy the image package to the host. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/osimage/openeuler/lib | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100755 container/osimage/openeuler/lib diff --git a/container/osimage/openeuler/lib b/container/osimage/openeuler/lib new file mode 100755 index 0000000..cf291ea --- /dev/null +++ b/container/osimage/openeuler/lib @@ -0,0 +1,45 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +DIR="$(pwd)" +os_name="$(echo $DIR |awk -F "/" '{print $(NF - 2)}')" +os_version="$(echo $DIR |awk -F "/" '{print $NF}')" +docker_url="https://repo.openeuler.org/openEuler-${os_version}/docker_img/aarch64/openE…" +docker_tar="openEuler-${os_version}-docker.aarch64.tar.xz" +image=$(echo "${os_name}-${os_version}" | tr '[A-Z]' '[a-z]') +IMAGE_NAME="${os_name}-${os_version}-$(date +"%Y%m%d").cgz" + +export_root_passwd() { + local root_pwd_file="$HOME/.config/compass-ci/rootfs-passwd" + [ -f "$root_pwd_file" ] || { + echo "[INFO] Please set the password file." + echo "$HOME/.config/compass-ci/rootfs-passwd" + exit 1 + } + + export ROOT_PASSWD=$(cat "$root_pwd_file") +} + +pull_docker_image() +{ + if [ "$os_version" == '20.09' ]; then + [ -f "$HOME/$docker_tar" ] || wget -O "$HOME/$docker_tar" "$docker_url" + elif [ "$os_version" == '20.03-LTS' ]; then + [ -f "$HOME/$docker_tar" ] || wget -O "$HOME/$docker_tar" "$docker_url" + fi + + docker load -i "$HOME/$docker_tar" + if [ $? = 0 ]; then + echo "finish downloading image" + else + echo "[ERROR] Pull docker image failed." + exit 1 + fi +} + +docker_cp_image() +{ + docker cp -a init_docker:/tmp/${IMAGE_NAME} $HOME/ + echo "result: $(ls $HOME/${IMAGE_NAME})" +} -- 2.23.0

4 6

[PATCH compass-ci 2/3] web_backend.rb: fix : compare_groups_matrices needs arg suites_list
by Li Yuanchao 26 Nov '20

26 Nov '20

compare_groups_matrices needs three args, but now only get two, lost suites_list. divide method to pass rubocop limit Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 1f7224e..90c8a58 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -147,6 +147,20 @@ def get_es_must(params) must end +def _get_group_matrices(query, dimension) + result = es_query(query) + matrices, suites_list = combine_group_query_data(result, dimension) + while matrices.empty? + from += size + break if from > total + + query[:from] = from + result = es_query(query) + matrices, suites_list = combine_group_query_data(result, dimension) + end + [matrices, suites_list] +end + def get_groups_matrices(conditions, dimension, must, size, from) must += build_mutli_field_subquery_body(conditions) count_query = { query: { bool: { must: must } } } @@ -166,27 +180,17 @@ def get_groups_matrices(conditions, dimension, must, size, from) }] } - result = es_query(query) - matrices = combine_group_query_data(result, dimension) - while matrices.empty? - from += size - break if from > total - - query[:from] = from - result = es_query(query) - matrices = combine_group_query_data(result, dimension) - end - matrices + matrices, suites_list = _get_groups_matrices(query, dimension) end def get_compare_body(params) dimension, conditions = get_dimension_conditions(params) must = get_es_must(params) - groups_matrices = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) + groups_matrices, suites_list = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) if !groups_matrices || groups_matrices.empty? body = 'No Data.' else - body = compare_group_matrices(groups_matrices, { no_print: true }) + body = compare_group_matrices(groups_matrices, suites_list, { no_print: true }) body = 'No Difference.' if !body || body.empty? end return body -- 2.23.0

1 1

[PATCH compass-ci 1/3] container/git-mirror: fix that can't get group id of committer in z9
by Li Yuanchao 26 Nov '20

26 Nov '20

In z9, 'committer' is only a group, not a user, so command 'id -g committer' can't get id of committer group Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/git-mirror/start | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container/git-mirror/start b/container/git-mirror/start index e1d7d8f..a618ec3 100755 --- a/container/git-mirror/start +++ b/container/git-mirror/start @@ -10,13 +10,13 @@ DOCKER_REPO = '/c/upstream-repos' docker_rm "git-mirror" lkp = %x(id -u lkp) -committer = %x(id -g committer) +group_id = %x(cat /etc/group | grep '^committer' | awk -F ':' '{print $3}') cmd = %W[ docker run --restart=always --name git-mirror - -u #{lkp.to_i}:#{committer.to_i} + -u #{lkp.to_i}:#{group_id.to_i} -d -e CCI_SRC=#{DOCKER_CCI} -e REPO_SRC=#{DOCKER_REPO} -- 2.23.0

1 1

[PATCH compass-ci 3/3] web-backend: fix : wrong method name
by Li Yuanchao 26 Nov '20

26 Nov '20

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 90c8a58..88ca02a 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -147,7 +147,7 @@ def get_es_must(params) must end -def _get_group_matrices(query, dimension) +def _get_groups_matrices(query, dimension) result = es_query(query) matrices, suites_list = combine_group_query_data(result, dimension) while matrices.empty? -- 2.23.0

1 0

[PATCH v2 lkp-tests 3/3] iso2rootfs: add comment to iso2rootfs.yaml
by Yu Chuan 26 Nov '20

26 Nov '20

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index a0ffd280676d..2c4d3a93ff62 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -1,9 +1,18 @@ suite: iso2rootfs category: benchmark iso2rootfs: + # iso related fields to be used to generate rootfs iso_os: iso_arch: iso_version: + + # place the result rootfs related fields + ## 1. result rootfs will be placed in the following localtion on the + ## remote file server: + ## - {remote_file_server}/{rootfs_path}/{iso_os}/{iso_arch}/ + ## 2. remote file server protocols current supported: + ## - nfs + ## - cifs rootfs_protocol: rootfs_server: rootfs_path: -- 2.23.0

1 0

[PATCH v2 lkp-tests 2/3] iso2rootfs: delete the unnecessary fields
by Yu Chuan 26 Nov '20

26 Nov '20

[Why] 'INITRD_HTTP_HOST' and 'INITRD_HTTP_PORT' should be added by the user on demand at runtime, no need to merge into the git repository. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 2 -- tests/iso2rootfs | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 4f3e9c83e6cd..a0ffd280676d 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -7,8 +7,6 @@ iso2rootfs: rootfs_protocol: rootfs_server: rootfs_path: - initrd_http_host: - initrd_http_port: # config rootfs related fields ## supported fields: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index 761fbb8b70cf..9adbc930b48f 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -49,8 +49,6 @@ check_yaml_vars() "rootfs_protocol" "rootfs_server" "rootfs_path" - "initrd_http_host" - "initrd_http_port" ) for yaml_t in "${yaml_vars[@]}" @@ -190,7 +188,7 @@ check_os() get_cgz() { - local cgz_url="http://${initrd_http_host}:${initrd_http_port}/initrd/pkg/nfs/${os}/${os_arch}/${os_version}/iso2rootfs.cgz" + local cgz_url="http://${INITRD_HTTP_HOST}:${INITRD_HTTP_PORT}/initrd/pkg/nfs/${os}/${os_arch}/${os_version}/iso2rootfs.cgz" local cgz_name cgz_name=$(basename "${cgz_url}") local cgz_path_t=${ENV_DIR}/${cgz_name} -- 2.23.0

1 0

[PATCH v2 lkp-tests 1/3] iso2rootfs: delete the meaningless field
by Yu Chuan 26 Nov '20

26 Nov '20

[Why] The 'iso2rootfs_cgz' field will always be 'iso2rootfs.cgz', so delete this field, and use the 'iso2rootfs.cgz' directly. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 1 - tests/iso2rootfs | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 1a643b538ac1..4f3e9c83e6cd 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -1,7 +1,6 @@ suite: iso2rootfs category: benchmark iso2rootfs: - iso2rootfs_cgz: iso_os: iso_arch: iso_version: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index fcaadaa1ba46..761fbb8b70cf 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -43,7 +43,6 @@ check_yaml_vars() "os" "os_arch" "os_version" - "iso2rootfs_cgz" "iso_os" "iso_arch" "iso_version" @@ -191,7 +190,7 @@ check_os() get_cgz() { - local cgz_url="http://${initrd_http_host}:${initrd_http_port}/initrd/pkg/nfs/${os}/${os_arch}/${os_version}/${iso2rootfs_cgz}" + local cgz_url="http://${initrd_http_host}:${initrd_http_port}/initrd/pkg/nfs/${os}/${os_arch}/${os_version}/iso2rootfs.cgz" local cgz_name cgz_name=$(basename "${cgz_url}") local cgz_path_t=${ENV_DIR}/${cgz_name} -- 2.23.0

1 0

[PATCH v4 lkp-tests] sbin/submit: submit merge_yaml/*.yaml
by Hu Xuejiao 26 Nov '20

26 Nov '20

[why] We will use 'rspec spec/submit_spec.rb' to merge job.yaml, it support merging from external file by "<<: job.yaml" Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- spec/submit/merge_yaml/atomic.yaml | 3 +++ .../merge_yaml/atomic/atomic-defaults.yaml | 19 +++++++++++++++++++ .../merge_yaml/yaml_merge_included_files.yaml | 1 + .../yaml_merge_included_files-defaults.yaml | 19 +++++++++++++++++++ spec/submit_spec.rb | 4 ++++ 5 files changed, 46 insertions(+) create mode 100644 spec/submit/merge_yaml/atomic.yaml create mode 100644 spec/submit/merge_yaml/atomic/atomic-defaults.yaml create mode 100644 spec/submit/merge_yaml/yaml_merge_included_files.yaml create mode 100644 spec/submit/merge_yaml/yaml_merge_included_files/yaml_merge_included_files-defaults.yaml diff --git a/spec/submit/merge_yaml/atomic.yaml b/spec/submit/merge_yaml/atomic.yaml new file mode 100644 index 00000000..331cb470 --- /dev/null +++ b/spec/submit/merge_yaml/atomic.yaml @@ -0,0 +1,3 @@ +suite: atomic + +atomic: diff --git a/spec/submit/merge_yaml/atomic/atomic-defaults.yaml b/spec/submit/merge_yaml/atomic/atomic-defaults.yaml new file mode 100644 index 00000000..42b53c13 --- /dev/null +++ b/spec/submit/merge_yaml/atomic/atomic-defaults.yaml @@ -0,0 +1,19 @@ +--- +suite: atomic +atomic: +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + atomic: + cpufreq_governor: +monitors: diff --git a/spec/submit/merge_yaml/yaml_merge_included_files.yaml b/spec/submit/merge_yaml/yaml_merge_included_files.yaml new file mode 100644 index 00000000..30019c6c --- /dev/null +++ b/spec/submit/merge_yaml/yaml_merge_included_files.yaml @@ -0,0 +1 @@ +<<: atomic.yaml diff --git a/spec/submit/merge_yaml/yaml_merge_included_files/yaml_merge_included_files-defaults.yaml b/spec/submit/merge_yaml/yaml_merge_included_files/yaml_merge_included_files-defaults.yaml new file mode 100644 index 00000000..42b53c13 --- /dev/null +++ b/spec/submit/merge_yaml/yaml_merge_included_files/yaml_merge_included_files-defaults.yaml @@ -0,0 +1,19 @@ +--- +suite: atomic +atomic: +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + atomic: + cpufreq_governor: +monitors: diff --git a/spec/submit_spec.rb b/spec/submit_spec.rb index a83d2102..92d8fa3d 100644 --- a/spec/submit_spec.rb +++ b/spec/submit_spec.rb @@ -44,4 +44,8 @@ describe 'submit job spec' do it 'link matrix' do submit_job('matrix') end + + it 'merge yaml spec' do + submit_job('merge_yaml') + end end -- 2.23.0

1 0

[PATCH v3 lkp-tests] sbin/submit: submit separate_yaml/*.yaml
by Hu Xuejiao 26 Nov '20

26 Nov '20

[why] We will use 'rspec spec/submit_spec.rb' to separate job.yaml, it contains several yaml documents separated by "---" Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- spec/submit/separate_yaml/ftq.yaml | 19 +++++++++++++ .../ftq/ftq-1000-2-100000ss-add.yaml | 27 +++++++++++++++++++ .../ftq/ftq-10000-2-100000ss-add.yaml | 27 +++++++++++++++++++ .../ftq/ftq-20-2-6000ss-cache.yaml | 27 +++++++++++++++++++ spec/submit_spec.rb | 4 +++ 5 files changed, 104 insertions(+) create mode 100644 spec/submit/separate_yaml/ftq.yaml create mode 100644 spec/submit/separate_yaml/ftq/ftq-1000-2-100000ss-add.yaml create mode 100644 spec/submit/separate_yaml/ftq/ftq-10000-2-100000ss-add.yaml create mode 100644 spec/submit/separate_yaml/ftq/ftq-20-2-6000ss-cache.yaml diff --git a/spec/submit/separate_yaml/ftq.yaml b/spec/submit/separate_yaml/ftq.yaml new file mode 100644 index 00000000..fbb9dc55 --- /dev/null +++ b/spec/submit/separate_yaml/ftq.yaml @@ -0,0 +1,19 @@ +suite: ftq + +nr_task: 100% +samples: 100000ss + +ftq: + test: add + freq: 10000 + +--- +ftq: + test: add + freq: 1000 + +--- +samples: 6000ss +ftq: + test: cache + freq: 20 diff --git a/spec/submit/separate_yaml/ftq/ftq-1000-2-100000ss-add.yaml b/spec/submit/separate_yaml/ftq/ftq-1000-2-100000ss-add.yaml new file mode 100644 index 00000000..f3e2c553 --- /dev/null +++ b/spec/submit/separate_yaml/ftq/ftq-1000-2-100000ss-add.yaml @@ -0,0 +1,27 @@ +--- +suite: ftq +nr_task: 2 +samples: 100000ss +ftq: + test: add + freq: 1000 +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + ftq: + nr_task: 2 + samples: 100000ss + freq: 1000 + test: add + cpufreq_governor: +monitors: diff --git a/spec/submit/separate_yaml/ftq/ftq-10000-2-100000ss-add.yaml b/spec/submit/separate_yaml/ftq/ftq-10000-2-100000ss-add.yaml new file mode 100644 index 00000000..63bac5cf --- /dev/null +++ b/spec/submit/separate_yaml/ftq/ftq-10000-2-100000ss-add.yaml @@ -0,0 +1,27 @@ +--- +suite: ftq +nr_task: 2 +samples: 100000ss +ftq: + test: add + freq: 10000 +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + ftq: + nr_task: 2 + samples: 100000ss + freq: 10000 + test: add + cpufreq_governor: +monitors: diff --git a/spec/submit/separate_yaml/ftq/ftq-20-2-6000ss-cache.yaml b/spec/submit/separate_yaml/ftq/ftq-20-2-6000ss-cache.yaml new file mode 100644 index 00000000..a802dc1b --- /dev/null +++ b/spec/submit/separate_yaml/ftq/ftq-20-2-6000ss-cache.yaml @@ -0,0 +1,27 @@ +--- +suite: ftq +nr_task: 2 +samples: 6000ss +ftq: + test: cache + freq: 20 +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + ftq: + nr_task: 2 + samples: 6000ss + freq: 20 + test: cache + cpufreq_governor: +monitors: diff --git a/spec/submit_spec.rb b/spec/submit_spec.rb index a83d2102..169d7812 100644 --- a/spec/submit_spec.rb +++ b/spec/submit_spec.rb @@ -44,4 +44,8 @@ describe 'submit job spec' do it 'link matrix' do submit_job('matrix') end + + it 'separate yaml spec' do + submit_job('separate_yaml') + end end -- 2.23.0

1 0

[PATCH v4 lkp-tests] setup/sysctl: add system parameters
by Zhang Yu 26 Nov '20

26 Nov '20

[why] When testing need to setup system parameters. [how] Add parameters to setup/sysctl. Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- setup/sysctl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup/sysctl b/setup/sysctl index 81374939..2ace4db2 100755 --- a/setup/sysctl +++ b/setup/sysctl @@ -202,6 +202,9 @@ # - sc_netdev_max_backlog # - sc_rmem_max # - sc_wmem_max +# - sc_ipv4_tcp_rmem +# - sc_ipv4_tcp_wmem +# - sc_sched_features . $LKP_SRC/lib/reproduce-log.sh @@ -405,8 +408,11 @@ [ -n "$sc_ipv4_tcp_max_tw_buckets" ] && log_eval "echo '$sc_ipv4_tcp_max_tw_buckets' > '/proc/sys/net/ipv4/tcp_max_tw_buckets'" [ -n "$sc_ipv4_tcp_max_syn_backlog" ] && log_eval "echo '$sc_ipv4_tcp_max_syn_backlog' > '/proc/sys/net/ipv4/tcp_max_syn_backlog'" [ -n "$sc_ipv4_ip_local_port_range" ] && log_eval "echo '$sc_ipv4_ip_local_port_range' > '/proc/sys/net/ipv4/ip_local_port_range'" +[ -n "$sc_ipv4_tcp_rmem" ] && log_eval "echo '$sc_ipv4_tcp_rmem' > '/proc/sys/net/ipv4/tcp_rmem'" +[ -n "$sc_ipv4_tcp_wmem" ] && log_eval "echo '$sc_ipv4_tcp_wmem' > '/proc/sys/net/ipv4/tcp_wmem'" [ -n "$sc_netdev_max_backlog" ] && log_eval "echo '$sc_netdev_max_backlog' > '/proc/sys/net/core/netdev_max_backlog'" [ -n "$sc_rmem_max" ] && log_eval "echo '$sc_rmem_max' > '/proc/sys/net/core/rmem_max'" [ -n "$sc_wmem_max" ] && log_eval "echo '$sc_wmem_max' > '/proc/sys/net/core/wmem_max'" +[ -n "$sc_sched_features" ] && log_eval "echo '$sc_sched_features' > '/sys/kernel/debug/sched_features'" exit 0 -- 2.23.0

1 0

[PATCH v3 compass-ci] web_backend.rb: fix : compare_groups_matrices needs arg suites_list
by Li Yuanchao 26 Nov '20

26 Nov '20

compare_groups_matrices needs three args, but now only get two, lost suites_list. divide method to pass rubocop limit Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 1f7224e..90c8a58 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -147,6 +147,20 @@ def get_es_must(params) must end +def _get_group_matrices(query, dimension) + result = es_query(query) + matrices, suites_list = combine_group_query_data(result, dimension) + while matrices.empty? + from += size + break if from > total + + query[:from] = from + result = es_query(query) + matrices, suites_list = combine_group_query_data(result, dimension) + end + [matrices, suites_list] +end + def get_groups_matrices(conditions, dimension, must, size, from) must += build_mutli_field_subquery_body(conditions) count_query = { query: { bool: { must: must } } } @@ -166,27 +180,17 @@ def get_groups_matrices(conditions, dimension, must, size, from) }] } - result = es_query(query) - matrices = combine_group_query_data(result, dimension) - while matrices.empty? - from += size - break if from > total - - query[:from] = from - result = es_query(query) - matrices = combine_group_query_data(result, dimension) - end - matrices + matrices, suites_list = _get_groups_matrices(query, dimension) end def get_compare_body(params) dimension, conditions = get_dimension_conditions(params) must = get_es_must(params) - groups_matrices = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) + groups_matrices, suites_list = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) if !groups_matrices || groups_matrices.empty? body = 'No Data.' else - body = compare_group_matrices(groups_matrices, { no_print: true }) + body = compare_group_matrices(groups_matrices, suites_list, { no_print: true }) body = 'No Difference.' if !body || body.empty? end return body -- 2.23.0

1 0

[PATCH v2 compass-ci] doc: about multi-device test
by Ren Wen 26 Nov '20

26 Nov '20

Signed-off-by: Ren Wen <15991987063(a)163.com> --- doc/multi-device-test.md | 109 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 doc/multi-device-test.md diff --git a/doc/multi-device-test.md b/doc/multi-device-test.md new file mode 100644 index 0000000..1be8c79 --- /dev/null +++ b/doc/multi-device-test.md @@ -0,0 +1,109 @@ +# Multi-device Test + +在进行需要多台机器的测试的时候, 需要搭建多台机器的环境. + +## When using multi-device + + * 当被测试程序需要启动一个守护进程时 + * 当执行测试时需要访问另外的机器时 + +## How to run a multi-device test + +在要提交的job.yaml中, 需要一个**cluster**字段, 它的值是一个多机环境的配 +置文件的文件名, 配置文件的内容可以参考: *lab-z9/cluster/*. +调度器会根据配置文件的内容将该job拆分为对应机器数量的job, 然后分别提交到 +相应的队列中. + +那么如何对指定的机器执行指定的测试呢? +在job.yaml中增加这样的内容: + +``` yaml +if role <role01> + <test_script01>: + <test_script02>: + +if role <role02> + <test_script03>: + <test_script04>: + +... +``` + +在多机的配置文件中, 会给每台机器赋予一个角色. 当每台机器取得自己队列中的 +job时, 按照自己的角色去执行相应角色所对应的脚本. + +比如说: 一台多机测试中的机器的角色为*role01*, 那么它会去寻找脚本 +*test_script01*, *test_script02*, 然后去执行. + +### how to find test script + +在lkp-tests的**daemon**, **tests**目录下查找这些脚本. +假设在**daemon**下找到了*test_script01*, 则将该脚本作为`start_daemon`的参 +数运行. 如果在**tests**下找到了*test_script01*, 则作为`run_test`的参数去 +运行. 如果没有找到相应的文件, 则按变量导出. + +### about `start_daemon` and `run_test` + +用于启动脚本的这2个函数位于**lkp-tests/lib/job.sh**. +要搞清楚多机的流程, 就得明白这两个函数是怎么去做的. + +#### the processes of `start_damon` + + 1. 运行在**daemon**文件夹下找到的脚本. 如果运行成功, 向调度器报告一个状态: + `started`, 调度器记录该机器的状态为: `start`; 执行失败, 则报告: + `failed`, 调度器记录为: `abort`, 然后退出执行. 状态可以查看redis的 + key: `HGETALL sched/cluster_state`. + 2. 向调度器报告一个状态: `write_state`, 并携带一些机器相关的参数: + * `node_roles`: 机器的角色 + * `ip`: 通过命令`hostname -I`取得的第一个值 + * `direct_macs`: 多机配置文件中该机器的**macs**字段 + * `direct_ips`: 调度器根据多机配置文件中的**ip0**字段配置的直连ip, 与 + 配置文件中**macs**的值一一对应 + 此时调度器将这些参数记录为每台机器相关的信息, 可以查看redis的key: + **sched/cluster_state**. + 3. 将**direct_ips**的值一一绑定到机器中的**direct_macs**的相应设备上. + 4. 向调度器报告一个状态: `wait_ready`, 调度器将其记录为: `ready`; 与此 + 同时, 调度器查看是否多机测试中的所有机器都准备好了. 如果每一台机器 + 都准备好了, 则返回`ready`; 如果其中一台的状态为`abort`, 则返回 + `abort`, 然后退出执行; 其余情况, 则循环查询, 直到前2种情形发生. + 5. 向调度器报告一个状态: `roles_ip`, 并将调度器的返回值导出为环境变量. + 比如当返回值为: `ip=ip01`时, 执行: `export ip=ip01`. + 6. 向调度器报告一个状态: `wait_finish`, 调度器将其记录为: `finish`; 与 + 此同时, 调度器查看是否多机测试中的所有机器的状态都为`finish`. 如果 + 都是`finish`, 则返回`finish`; 如果其中一台的状态为`abort`, 则返回 + `abort`, 然后退出执行; 其余情况, 则循环查询, 直到前2种情形发生. + 7. 执行完成. + +#### the processes of `run_test` + +步骤1,2,3,4同上述2,3,4,5. + + 1. 向调度器报告一个状态: `write_state`, 并携带一些机器相关的参数: + * `node_roles`: 机器的角色 + * `ip`: 通过命令`hostname -I`取得的第一个值 + * `direct_macs`: 多机配置文件中该机器的**macs**字段 + * `direct_ips`: 调度器根据多机配置文件中的**ip0**字段配置的直连ip, 与 + 配置文件中**macs**的值一一对应 + 此时调度器将这些参数记录为每台机器相关的信息, 可以查看redis的key: + **sched/cluster_state**. + 2. 将**direct_ips**的值一一绑定到机器中的**direct_macs**的相应设备上. + 3. 向调度器报告一个状态: `wait_ready`, 调度器将其记录为: `ready`; 与此 + 同时, 调度器查看是否多机测试中的所有机器都准备好了. 如果每一台机器 + 都准备好了, 则返回`ready`; 如果其中一台的状态为`abort`, 则返回 + `abort`, 然后退出执行; 其余情况, 则循环查询, 直到前2种情形发生. + 4. 向调度器报告一个状态: `roles_ip`, 并将调度器的返回值导出为环境变量. + 比如当返回值为: `ip=ip01`时, 执行: `export ip=ip01` + 5. 运行在**tests**文件夹下找到的脚本. 运行完成, 向调度器报告一个状态: + `finished`, 调度器记录为: `finish`; 执行失败, 则报告: `failed`, 调 + 度器记录为: `abort`, 然后退出执行. + 6. 执行完成. + +#### others + +`start_daemon`和`run_test`还会报告一个`abort`状态: 循环100次向调度器发 +起请求, 查看所有机器是否全部`ready`或`finish`, 如果循环结束仍然返回的是 +`retry`, 则报告该状态, 并结束执行. + +当调度器接受到与之相关的请求时, 会阻塞(或循环)查询这些机器的状态, 因此只 +需返回`ready`(`finish`)或`abort`. 所以不会触发循环100次请求这个过程, 因 +此不会报告`abort`这个状态. -- 2.23.0

1 0

[PATCH lkp-tests] distro/*: add exit when failed download packages
by Wang Yong 26 Nov '20

26 Nov '20

now when download packages failed, echo error message and $? is 0, so add exit 1 to get last_state Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- distro/aliyun | 1 + distro/archlinux | 1 + distro/debian | 1 + 3 files changed, 3 insertions(+) diff --git a/distro/aliyun b/distro/aliyun index 79656f0e4..598243d8e 100755 --- a/distro/aliyun +++ b/distro/aliyun @@ -126,6 +126,7 @@ pack_benchmark_deps() yum_pack else echo "failed to pack-deps $BM_NAME" >&2 + exit 1 fi done } diff --git a/distro/archlinux b/distro/archlinux index 3f8d62ebc..3942080a6 100755 --- a/distro/archlinux +++ b/distro/archlinux @@ -24,6 +24,7 @@ pack_benchmark_deps() pacman_pack else echo "failed to pack-deps $BM_NAME" >&2 + exit 1 fi done } diff --git a/distro/debian b/distro/debian index 13c4e983a..8821e3ddd 100755 --- a/distro/debian +++ b/distro/debian @@ -271,6 +271,7 @@ pack_benchmark_deps() echo "$PACKAGE_LIST" > $pack_to/.${BM_NAME}.packages else echo "failed to pack-deps $BM_NAME" >&2 + exit 1 fi rm -fr "$work_dir" -- 2.23.0

1 0

[PATCH compass-ci 1/4] answerback-email: add new options
by Luan Shengde 26 Nov '20

26 Nov '20

add option for answerback-email tool -n|--name name specify my_name for user when assigning account -l|--login Y|y|N|n enable/disable user login permission -u|--update enable update configuration for assigned account -u option enable update for: my_name my_ssh_pubkey generate sshkey on jumper reset user's login permission Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/assign-account/answerback-email.rb | 147 +++++++++++++++---- 1 file changed, 120 insertions(+), 27 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index 5ddc451..bffba17 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -14,6 +14,7 @@ require 'optparse' require_relative '../defconfig' require_relative '../../lib/es_client' require_relative 'build-send-account-email' +require_relative 'build-update-email' names = Set.new %w[ JUMPER_HOST @@ -33,25 +34,61 @@ my_info = { 'my_email' => nil, 'my_name' => nil, 'my_commit_url' => nil, - 'my_uuid' => %x(uuidgen).chomp, - 'my_ssh_pubkey' => [], - 'gen_sshkey' => false + 'my_uuid' => nil, + 'my_login_name' => nil, + 'my_ssh_pubkey' => [] } -def init_info(mail_content, my_info) +# stdin_info is used to store infos added with option: +# -e email_address +# -n name +# -s pubkey_file +# email_info is used to store infos parsed from email_file with option: +# -f email_file +# my_info_es is used to store info read from ES when update user +# when use --update option +# stdin_info, email_info, my_info_es have different priority +# stdin_info > email_info > my_info_es +# when assigning account or update conf for account +# if they have the same key, my_info will use the value with higher priority. +# conf_info is used to store keys used to config the account +stdin_info = {} +email_info = {} +my_info_es = {} +conf_info = { + 'gen_sshkey' => false, + 'enable_login' => true, + 'is_update_account' => false +} + +def init_info(mail_content, email_info, my_info) my_info['my_email'] = mail_content.from[0] - my_info['my_name'] = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') + email_info['my_name'] = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') return if mail_content.attachments.empty? - my_info['my_ssh_pubkey'] << mail_content.attachments[0].body.decoded + email_info['new_email_pubkey'] = mail_content.attachments[0].body.decoded.strip +end + +def read_my_login_name(my_email, my_info_es) + my_account_info_str = %x(curl -XGET localhost:9200/accounts/_doc/#{my_email}) + my_account_info = YAML.safe_load my_account_info_str + message = "No such email found from the ES: #{my_email}" + raise message unless my_account_info['found'] + + my_info_es.update my_account_info['_source'] end options = OptionParser.new do |opts| - opts.banner = 'Usage: answerback-mail.rb [-e|--email email] ' - opts.banner += "[-s|--ssh-pubkey pub_key_file] [-f|--raw-email email_file] [-g|--gen-sshkey]\n" - opts.banner += " -e or -f is required\n" + opts.banner = 'Usage: answerback-mail.rb [-e|--email email] [-n|--name name] ' + opts.banner += "[-s|--ssh-pubkey pub_key_file] [-g|--gen-sshkey] [--login y|Y|n|N] [--update]\n" + opts.banner += ' answerback-mail.rb [-f|--raw-email email_file] ' + opts.banner += "[-g|--gen-sshkey] [--login y|Y|n|N] [--update]\n" + opts.banner += " -e|-f is required when applying account or updating account\n" + opts.banner += " -n is require when assigning account with -e\n" opts.banner += " -s is optional when use -e\n" - opts.banner += ' -g is optional, used to generate sshkey for user' + opts.banner += " -g is optional, used to generate sshkey for user\n" + opts.banner += " -u is require when updating an account\n" + opts.banner += ' -l is optional, used to enable/disable login permission' opts.separator '' opts.separator 'options:' @@ -60,18 +97,37 @@ options = OptionParser.new do |opts| my_info['my_email'] = email_address end + opts.on('-n name', '--name name', 'appoint name') do |name| + stdin_info['my_name'] = name + end + opts.on('-s pub_key_file', '--ssh-pubkey pub_key_file', \ 'ssh pub_key file, enable password-less login') do |pub_key_file| - my_info['my_ssh_pubkey'] << File.read(pub_key_file).chomp + stdin_info['new_ssh_pubkey'] = File.read(pub_key_file).strip end opts.on('-f email_file', '--raw-email email_file', 'email file') do |email_file| mail_content = Mail.read(email_file) - init_info(mail_content, my_info) + init_info(mail_content, email_info, my_info) end opts.on('-g', '--gen-sshkey', 'generate jumper ras public/private key and return pubkey') do - my_info['gen_sshkey'] = true + conf_info['gen_sshkey'] = true + end + + opts.on('-u', '--update', 'updata user account') do + read_my_login_name(my_info['my_email'], my_info_es) + conf_info['is_update_account'] = true + end + + opts.on('-l Y|y|N|n', '--login Y|y|N|n', 'enable/disable user login permission') do |value| + if %w[y Y].include? value + conf_info['enable_login'] = true + elsif %w[n N].include? value + conf_info['enable_login'] = false + else + raise 'invalid parameter, please use y|Y|n|N' + end end opts.on_tail('-h', '--help', 'show this message') do @@ -82,33 +138,70 @@ end options.parse!(ARGV) -def apply_account(my_info) - account_info_str = %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d '#{my_info.to_json}') +def apply_account(my_info, conf_info) + apply_info = {} + apply_info.update my_info + apply_info.update conf_info + + account_info_str = %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d '#{apply_info.to_json}') JSON.parse account_info_str end -def send_account(my_info) +def check_my_email(my_info) message = "No email address specified\n" - message += "use -e to add a email address\n" + message += "use -e to add a email address for applying account\n" message += 'or use -f to add a email file' - raise message if my_info['my_email'].nil? - account_info = apply_account(my_info) - my_info['my_login_name'] = account_info['my_login_name'] + raise message if my_info['my_email'].empty? +end +def build_my_info_from_input(my_info, email_info, my_info_es, stdin_info) + new_email_pubkey = email_info.delete 'new_email_pubkey' + new_stdin_pubkey = stdin_info.delete 'new_ssh_pubkey' + new_pubkey = new_stdin_pubkey || new_email_pubkey + + my_info.update my_info_es unless my_info_es.empty? + my_info.update email_info unless email_info.empty? + my_info.update stdin_info unless stdin_info.empty? + + return if new_pubkey.nil? + return if my_info['my_ssh_pubkey'].include? new_pubkey + + my_info['my_ssh_pubkey'].insert(0, new_pubkey) +end + +def build_my_info_from_account_info(my_info, account_info, conf_info) unless account_info['my_jumper_pubkey'].nil? - my_info['my_ssh_pubkey'] << account_info['my_jumper_pubkey'].chomp + return if my_info['my_ssh_pubkey'][-1] == account_info['my_jumper_pubkey'] + + my_info['my_ssh_pubkey'] << account_info['my_jumper_pubkey'] end - my_info.delete 'gen_sshkey' + my_info['my_login_name'] = account_info['my_login_name'] unless conf_info['is_update_account'] +end + +def send_account(my_info, conf_info, email_info, my_info_es, stdin_info) + check_my_email(my_info) + my_info['my_uuid'] = %x(uuidgen).chomp unless conf_info['is_update_account'] + build_my_info_from_input(my_info, email_info, my_info_es, stdin_info) + + message = 'No my_name found, please use -n to add one' + raise message if my_info['my_name'].nil? + + account_info = apply_account(my_info, conf_info) + build_my_info_from_account_info(my_info, account_info, conf_info) + store_account_info(my_info) - send_mail(my_info, account_info) + send_mail(my_info, account_info, conf_info) end -def send_mail(my_info, account_info) - message = build_message(my_info['my_email'], account_info) - +def send_mail(my_info, account_info, conf_info) + message = if conf_info['is_update_account'] + build_update_message(my_info['my_email'], account_info, conf_info) + else + build_message(my_info['my_email'], account_info) + end %x(curl -XPOST '#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end @@ -117,4 +210,4 @@ def store_account_info(my_info) es.put_source_by_id(my_info['my_email'], my_info) end -send_account(my_info) +send_account(my_info, conf_info, email_info, my_info_es, stdin_info) -- 2.23.0

2 2

[PATCH compass-ci] web_backend.rb: fix : compare_groups_matrices needs arg suites_list
by Li Yuanchao 26 Nov '20

26 Nov '20

compare_groups_matrices needs three args, but now only get two, lost suites_list. divide method to pass rubocop limit Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 1f7224e..b5b06b4 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -147,6 +147,20 @@ def get_es_must(params) must end +def _get_group_matrices(query, dimension) + result = es_query(query) + matrices, suites_list = combine_group_query_data(result, dimension) + while matrices.empty? + from += size + break if from > total + + query[:from] = from + result = es_query(query) + matrices, suites_list = combine_group_query_data(result, dimension) + end + [matrices, suites_list] +end + def get_groups_matrices(conditions, dimension, must, size, from) must += build_mutli_field_subquery_body(conditions) count_query = { query: { bool: { must: must } } } @@ -166,27 +180,18 @@ def get_groups_matrices(conditions, dimension, must, size, from) }] } - result = es_query(query) - matrices = combine_group_query_data(result, dimension) - while matrices.empty? - from += size - break if from > total - - query[:from] = from - result = es_query(query) - matrices = combine_group_query_data(result, dimension) - end - matrices + matrices, suites_list = _get_groups_matrices(query, dimension) + [matrices, suites_list] end def get_compare_body(params) dimension, conditions = get_dimension_conditions(params) must = get_es_must(params) - groups_matrices = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) + groups_matrices, suites_list = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) if !groups_matrices || groups_matrices.empty? body = 'No Data.' else - body = compare_group_matrices(groups_matrices, { no_print: true }) + body = compare_group_matrices(groups_matrices, suites_list, { no_print: true }) body = 'No Difference.' if !body || body.empty? end return body -- 2.23.0

1 0