- Compass-ci - mailweb.openeuler.org

[PATCH lab-z9] add dc-16/32g and vm-2p32g
by Cao Xueliang 10 Mar '21

10 Mar '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- hosts/dc-16g | 2 ++ hosts/dc-32g | 2 ++ hosts/vm-2p32g | 7 +++++++ 3 files changed, 11 insertions(+) create mode 100644 hosts/dc-16g create mode 100644 hosts/dc-32g create mode 100644 hosts/vm-2p32g diff --git a/hosts/dc-16g b/hosts/dc-16g new file mode 100644 index 0000000..f366892 --- /dev/null +++ b/hosts/dc-16g @@ -0,0 +1,2 @@ +arch: aarch64 +memory: 16g diff --git a/hosts/dc-32g b/hosts/dc-32g new file mode 100644 index 0000000..4006e66 --- /dev/null +++ b/hosts/dc-32g @@ -0,0 +1,2 @@ +arch: aarch64 +memory: 32g diff --git a/hosts/vm-2p32g b/hosts/vm-2p32g new file mode 100644 index 0000000..c7cc583 --- /dev/null +++ b/hosts/vm-2p32g @@ -0,0 +1,7 @@ +arch: aarch64 +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 32G +hdd_partitions: /dev/vda -- 2.23.0

1 0

[PATCH lkp-tests] hosts: add dc-32g file
by Lin Jiaxin 10 Mar '21

10 Mar '21

[Why] linux compile need big memory Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- hosts/dc-32g | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 hosts/dc-32g diff --git a/hosts/dc-32g b/hosts/dc-32g new file mode 100644 index 00000000..e9c80b37 --- /dev/null +++ b/hosts/dc-32g @@ -0,0 +1,2 @@ +nr_cpu: 8 +memory: 32g -- 2.23.0

1 0

[PATCH v3 compass-ci 1/2] sbin/result2stats: support call function from stats/script
by Lu Weitao 10 Mar '21

10 Mar '21

[Why] old logical: lkp-tests/stats/script < script_log > tmpfile lkp-tests/sbin/dump-stat < tmpfile > $script.json this flow will escape special characters [How] use return of function to keep original characters lkp-tests/stats/script.rb func(read(script_log)) --> hash compass-ci/lib/dump-stats.rb hash --> dump_stats() --> $script.json [Dependence] lkp-tests/stats/$script.rb, for example: file_name: lkp-tests/stats/proc-vmstats.rb function name: proc_vmstats() input: Array(String) ["time: 231231234", "nr_free_page: 89721", ...] output: Hash(String, Array(Number|String) | Number | String) { "time" => [231231234, 231231235], "nr_free_page" => [89721,88564], ... } or { "time" => 231231234, "nr_free_page" => 89721, ... } Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/stats.rb | 19 ++++++++++++++----- lib/stats_wrapper.rb | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/lib/stats.rb b/lib/stats.rb index c01d38d..94e370a 100644 --- a/lib/stats.rb +++ b/lib/stats.rb @@ -14,11 +14,16 @@ end def available_stats vailable_stats = [] + available_stats_func = [] Dir.open(LKP_SRC + '/stats').each do |file| - vailable_stats << file unless file.start_with?('.') + if file.end_with?('.rb') + available_stats_func << file.chomp('.rb') + else + vailable_stats << file unless file.start_with?('.') + end end - vailable_stats + [vailable_stats, available_stats_func] end def assign_default_stats(stats_list) @@ -32,13 +37,17 @@ class Stats def initialize(result_root) @result_root = result_root @job = load_yaml(File.join(result_root, 'job.yaml')) - @available_stats = available_stats + @available_stats, @available_stats_func = available_stats end def extract_stats stats_list = assign_stats_list stats_list.each do |stat| - StatsWrapper.wrapper(*stat) + if @available_stats_func.include?(stat) + StatsWrapper.wrapper_func(*stat) + else + StatsWrapper.wrapper(*stat) + end end end @@ -48,7 +57,7 @@ class Stats stats_list << ['time', @job['suite'] + '.time'] stats_list << 'stderr' @job.each_key do |k| - if @available_stats.include?(k) + if @available_stats.include?(k) || @available_stats_func.include?(k) stats_list << k end end diff --git a/lib/stats_wrapper.rb b/lib/stats_wrapper.rb index 8a6d6a9..faa81df 100644 --- a/lib/stats_wrapper.rb +++ b/lib/stats_wrapper.rb @@ -6,6 +6,11 @@ LKP_SRC ||= ENV['LKP_SRC'] || '/c/lkp-tests' require "#{LKP_SRC}/lib/log.rb" require 'tempfile' require 'English' +require_relative './dump_stat' + +Dir[File.expand_path("#{LKP_SRC}/stats/*.rb")].uniq.each do |file| + require file +end PROGRAM_DIR = "#{LKP_SRC}/stats" @@ -35,6 +40,27 @@ module StatsWrapper delete_log_package end + def self.wrapper_func(program, program_time = nil) + @program = program + @stats_group = program_time || program + @log = "#{RESULT_ROOT}/#{@stats_group}" + + return unless File.exist?("#{(a)log}.yaml") || pretreatment + + if File.exist?("#{(a)log}.yaml") + log_lines = read_log("#{(a)log}.yaml") + stat_result = parse_simple_log_yaml(log_lines) + else + log_lines = read_log(@log) + call_func_cmd = "#{(a)program.gsub('-', '_')}(log_lines)" # eg: proc_vmstats(log_lines) + stat_result = eval(call_func_cmd) + end + return unless DumpStat.dump_stat(@stats_group, stat_result) + + check_empty_json + delete_log_package + end + def self.pretreatment return unless available_program? return unless unzip_log @@ -261,3 +287,21 @@ module StatsWrapper File.delete(@tmpfile) if File.exist?(@tmpfile) end end + +# read line of log +# return Array(String) +def read_log(log_path) + return nil unless File.exist?(log_path) + + File.readlines(log_path) +end + +def parse_simple_log_yaml(log_lines) + result = Hash.new { |hash, key| hash[key] = [] } + log_lines.each do |line| + key, value = line.split(/:?\s+/) + result[key] << value + end + + result +end -- 2.23.0

2 2

[PATCH lkp-tests 05/11] feat: step 4 of iso2rootfs: add entry job
by Yu Chuan 10 Mar '21

10 Mar '21

Framework of install-iso-pre. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/install-iso-pre.yaml | 52 +++++++++++++++++++++++ tests/install-iso-pre | 86 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 jobs/install-iso-pre.yaml create mode 100755 tests/install-iso-pre diff --git a/jobs/install-iso-pre.yaml b/jobs/install-iso-pre.yaml new file mode 100644 index 000000000000..a9dbe46b4bf3 --- /dev/null +++ b/jobs/install-iso-pre.yaml @@ -0,0 +1,52 @@ +suite: install-iso-pre + +install-iso-pre: + ################# + # iso info + ################# + iso_os: + iso_arch: + iso_version: + + ################# + # http server info, which stored the installation source + ################# + HTTP_SERVER_HOST: + HTTP_SERVER_PORT: + HTTP_ROOT_DIR: + + ################# + # rsync-server info, used for upload ks + ################# + RSYNC_SERVER_HOST: + RSYNC_SERVER_PORT: + + ################# + # tftp server info, used for kickstart auto install + ################# + TFTP_SERVER_HOST: + + ################# + # nfs server info, used for store rootfs + ################# + NFS_SERVER_HOST: + NFS_ROOT_DIR: + + ################# + # result rootfs suffix, format: $(date "+%Y%m%d%H%M%S") + ################# + ROOTFS_SUFFIX: + + ################# + # name of volume group which stores the logical volume of rootfs + ################# + VG_NAME: + + ################# + # lab_z9 git repo clone url, used for find host file when submit + ################# + LAB_Z9_URL: + +secrets: + ROOTFS_ROOT_PASSWD: + MY_TOKEN: diff --git a/tests/install-iso-pre b/tests/install-iso-pre new file mode 100755 index 000000000000..4e8cbe22fcd4 --- /dev/null +++ b/tests/install-iso-pre @@ -0,0 +1,86 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +set -e + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/log.sh + +check_params() +{ + log_info "start check params" + + local required_vars=( + "iso_os" + "iso_arch" + "iso_version" + "ROOTFS_SUFFIX" + "VG_NAME" + + "HTTP_SERVER_HOST" + "HTTP_SERVER_PORT" + "HTTP_ROOT_DIR" + "ROOTFS_ROOT_PASSWD" + "rootfs_disk" + + "RSYNC_SERVER_HOST" + "RSYNC_SERVER_PORT" + "TFTP_SERVER_HOST" + "NFS_SERVER_HOST" + "NFS_ROOT_DIR" + + "LAB_Z9_URL" + "MY_TOKEN" + ) + + for i in "${required_vars[@]}" + do + [ -n "$(eval echo \$$i)" ] || die "cannot get value of var: $i" + done +} + +prepare_lv() +{ + : +} + +prepare_ks() +{ + : +} + +prepare_submit() +{ + : +} + +submit_install_iso_yaml() +{ + : +} + +submit_install_iso_firstboot_yaml() +{ + : +} + +submit_install_iso_release_yaml() +{ + : +} + +main() +{ + check_params + + prepare_lv + prepare_ks + prepare_submit + + submit_install_iso_yaml + submit_install_iso_firstboot_yaml + submit_install_iso_release_yaml +} + +main -- 2.23.0

2 1

[PATCH lkp-tests] add new type error
by Bai Jing 10 Mar '21

10 Mar '21

[input] Error: Unknown repo: 'powertools' [output] unknown-repo.powertools: 1 Signed-off-by: Bai Jing <799286817(a)qq.com> --- stats/openeuler_docker | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/stats/openeuler_docker b/stats/openeuler_docker index f85c9ca63..bcc5eb3b5 100755 --- a/stats/openeuler_docker +++ b/stats/openeuler_docker @@ -8,17 +8,23 @@ while (line = STDIN.gets) case line.chomp! # Error: Unable to find a match: docker-registry when /Error: Unable to find a match: (.+)/ - $1.split.each do |repo| - repo_set << repo + $1.split.each do |repo| + repo_set << repo end + # Error: Unknown repo: 'powertools' + when /Error: Unknown repo: (.+)/ + repo = $1.delete!("'") + puts "unknown-repo.#{repo}: 1" + puts "unknown-repo.#{repo}.message: #{line}" + # /bin/sh: passwd: command not found when /\/bin\/sh: (.+): command not found/ puts "sh.command-not-found.#{$1}: 1" puts "sh.command-not-found.#{$1}.message: #{line}" # RUN yum -y swap -- remove fakesystemd -- install systemd systemd-libs - # yum swap: error: unrecognized arguments: install systemd systemd-libs + # yum swap: error: unrecognized arguments: install systemd systemd-libs when /yum swap: error: .*: install (.+)/ $1.split.each do |sys| sys_set << sys -- 2.23.0

1 1

[PATCH v3 compass-ci] src/lib/job.cr: optimize the get_pkg_common_dir function
by Li Ping 10 Mar '21

10 Mar '21

[why] 1. rpm has no much to do with os_mount. add a judgment 2. optimize the code style Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/job.cr | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0b4bee4..f91e637 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -258,23 +258,28 @@ class Job end def get_pkg_common_dir - pkg_style = nil + tmp = nil ["cci-makepkg", "cci-depends", "build-pkg", "rpmbuild-pkg"].each do |item| - pkg_style = @hash[item]? - break if pkg_style + tmp = @hash[item]? + break if tmp end - return nil unless pkg_style + return nil unless tmp + pkg_style = Hash(String, JSON::Any).new {|h, k| h[k] = JSON::Any.new(nil)} + pkg_style.merge!(tmp.as_h? || Hash(String, JSON::Any).new) - pkg_style = JSON.parse("{}") if pkg_style == nil + tmp_os = pkg_style["os"].as_s? || "#{os}" + tmp_os_arch = pkg_style["os_arch"].as_s? || "#{os_arch}" + tmp_os_version = pkg_style["os_version"].as_s? || "#{os_version}" - tmp_os = pkg_style["os"]? == nil ? "#{os}" : pkg_style["os"] - tmp_os_arch = pkg_style["os_arch"]? == nil ? "#{os_arch}" : pkg_style["os_arch"] - tmp_os_version = pkg_style["os_version"]? == nil ? "#{os_version}" : pkg_style["os_version"] + mount_type = pkg_style["os_mount"].as_s? || "#{os_mount}" + # same usage for client + mount_type = "nfs" if mount_type == "cifs" - tmp_os_mount = pkg_style["os_mount"]? == nil ? "#{os_mount}" : pkg_style["os_mount"] - mount_type = tmp_os_mount == "cifs" ? "nfs" : tmp_os_mount.dup - - common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + if @hash["rpmbuild-pkg"]? + common_dir = "#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + else + common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + end return common_dir end -- 2.23.0

3 5

[PATCH v2 compass-ci] doc/manual: add chinese document
by Luan Shengde 10 Mar '21

10 Mar '21

Add chinese document for submit container. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- doc/manual/build-lkp-tests-container.zh.md | 63 ++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 doc/manual/build-lkp-tests-container.zh.md diff --git a/doc/manual/build-lkp-tests-container.zh.md b/doc/manual/build-lkp-tests-container.zh.md new file mode 100644 index 0000000..407ddbf --- /dev/null +++ b/doc/manual/build-lkp-tests-container.zh.md @@ -0,0 +1,63 @@ +# 前言 + +为了适应在各个版本的Linux操作系统上运行，我们提供了lkp-tests的容器化解决方案。 +你无需在您的本地服务上安装lkp-tests，也将避免缺乏依赖包而导致的安装失败。 + +# 前置准备 + + 请在您的本地服务器上完成下面步骤： + + - 安装docker + - 申请账号并配置默认的yaml文件 + - 生成本地的公私钥 + +# 构建容器 + +## 1. 下载资源 + + 本地下载lkp-tests和compass-ci + + 下载资源： + + git clone https://gitee.com/wu_fengguang/compass-ci.git + git clone https://gitee.com/wu_fengguang/lkp-tests.git + +## 2. 添加环境变量 + + 配置环境变量： + + echo "export LKP_SRC=$PWD/lkp-tests" >> ~/.${SHELL##*/}rc + echo "export CCI_SRC=$PWD/compass-ci" >> ~/.${SHELL##*/}rc + source ~/.${SHELL##*/}rc + +## 3. 构建镜像 + + 镜像构建： + + cd compass-ci/container/submit + ./build + +## 4. 添加可执行文件 + + 创建可执行文件： + + ln -s $CCI_SRC/container/submit/submit /usr/bin/submit + +# 开始第一个任务 + + 说明： + + 和在你的本地服务器上安装lkp-tests一样，您可以直接使用‘submit’命令提交任务。 + 每一个任务都会生成一个一次性的容器来执行。 + 可执行文件submit会将您下载的lkp-tests挂载到容器内的lkp-tests路径上，这样您在本地编辑lkp-tests文件，在容器内也会生效。 + + + + 提交任务命令示例： + + + submit -c -m testbox=vm-2p8g borrow-1h.yaml + + submit命令： + + 了解更多submit命令的使用方法，请参考：[submit用户手册](https://gitee.com/wu_fengguang/compass-c… -- 2.23.0

1 0

[PATCH v5 compass-ci] src/lib/job.cr: optimize the get_pkg_common_dir function
by Li Ping 10 Mar '21

10 Mar '21

[why] 1. rpm has no much to do with os_mount. add a judgment 2. optimize the code style Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/job.cr | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0b4bee4..a7674d4 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -258,23 +258,28 @@ class Job end def get_pkg_common_dir - pkg_style = nil + tmp_style = nil ["cci-makepkg", "cci-depends", "build-pkg", "rpmbuild-pkg"].each do |item| - pkg_style = @hash[item]? - break if pkg_style + tmp_style = @hash[item]? + break if tmp_style end - return nil unless pkg_style + return nil unless tmp_style + pkg_style = Hash(String, JSON::Any).new {|h, k| h[k] = JSON::Any.new(nil)} + pkg_style.merge!(tmp_style.as_h? || Hash(String, JSON::Any).new) - pkg_style = JSON.parse("{}") if pkg_style == nil + tmp_os = pkg_style["os"].as_s? || "#{os}" + tmp_os_arch = pkg_style["os_arch"].as_s? || "#{os_arch}" + tmp_os_version = pkg_style["os_version"].as_s? || "#{os_version}" - tmp_os = pkg_style["os"]? == nil ? "#{os}" : pkg_style["os"] - tmp_os_arch = pkg_style["os_arch"]? == nil ? "#{os_arch}" : pkg_style["os_arch"] - tmp_os_version = pkg_style["os_version"]? == nil ? "#{os_version}" : pkg_style["os_version"] + mount_type = pkg_style["os_mount"].as_s? || "#{os_mount}" + # same usage for client + mount_type = "nfs" if mount_type == "cifs" - tmp_os_mount = pkg_style["os_mount"]? == nil ? "#{os_mount}" : pkg_style["os_mount"] - mount_type = tmp_os_mount == "cifs" ? "nfs" : tmp_os_mount.dup - - common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + if @hash["rpmbuild-pkg"]? + common_dir = "#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + else + common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + end return common_dir end -- 2.23.0

1 0

[PATCH compass-ci] sbin/auto_submit: change the path to do git command
by Li Yuanchao 10 Mar '21

10 Mar '21

As the path of repo mirrors under /srv/git changes, the path to execute git command should change too. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/sbin/auto_submit b/sbin/auto_submit index 857af29..faebb2f 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -44,14 +44,16 @@ class AutoSubmit submit_argv end - def submit_one_job(submit_argv, git_repo, branch, commit_id) - inactive_time = %x(git -C /srv/git/#{git_repo}.git log --pretty=format:"%cr" -1 #{commit_id}) + def submit_one_job(submit_argv, git_repo, belong, branch, commit_id) + mirror_dir = "/srv/git/#{belong}/#{git_repo}.git" + inactive_time = %x(git -C #{mirror_dir} log --pretty=format:"%cr" -1 #{commit_id}) return if inactive_time =~ /(month|year)/ real_argvs = Array.new(submit_argv) real_argvs.push("upstream_branch=#{branch}") real_argvs.push("upstream_commit=#{commit_id}") - tag = %x(git -C /srv/git/#{git_repo}.git tag --points-at #{commit_id}) + real_argvs.push("upstream_dir=#{belong}") + tag = %x(git -C #{mirror_dir} tag --points-at #{commit_id}) real_argvs.push("upstream_tag=#{tag}") unless tag.empty? system(real_argvs.join(' ')) @@ -60,16 +62,17 @@ class AutoSubmit def submit_specific_branch(submit_argv, newrefs_info, branches) branches.each do |branch_name| branch = "refs/heads/#{branch_name}" - next unless newrefs_info['new_refs']['heads'][branch] + commit_id = newrefs_info['new_refs']['heads'][branch] + next unless commit_id - submit_one_job(submit_argv, newrefs_info['git_repo'], branch_name, newrefs_info['new_refs']['heads'][branch]) + submit_one_job(submit_argv, newrefs_info['git_repo'], newrefs_info['belong'], branch_name, commit_id) end end - def submit_non_specific(non_specific_commits, git_repo, submit_argv) - non_specific_commits.each do |branch, commit_id| + def submit_non_specific(newrefs_info, submit_argv) + newrefs_info['new_refs']['heads'].each do |branch, commit_id| branch = branch.delete_prefix('refs/heads/') - submit_one_job(submit_argv, git_repo, branch, commit_id) + submit_one_job(submit_argv, newrefs_info['git_repo'], newrefs_info['belong'], branch, commit_id) end end @@ -96,7 +99,7 @@ class AutoSubmit if element['branches'] submit_specific_branch(argvs, newrefs_info, element['branches']) else - submit_non_specific(newrefs_info['new_refs']['heads'], newrefs_info['git_repo'], argvs) + submit_non_specific(newrefs_info, argvs) end end end -- 2.23.0

1 0

[PATCH lkp-tests] stats/proc-vmstat.rb: add the execute permission
by Wei Jihui 10 Mar '21

10 Mar '21

[error output when submit job] 2021-03-10 10:20:04 +0800 WARN -- skip non-executable /c/lkp-tests/stats/proc-vmstat.rb Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- stats/proc-vmstat.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 stats/proc-vmstat.rb diff --git a/stats/proc-vmstat.rb b/stats/proc-vmstat.rb old mode 100644 new mode 100755 -- 2.23.0

1 0