- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci 1/3] scheduler: idle-job: fix queue when submitting job
by Ren Wen 25 Dec '20

25 Dec '20

[Before]: 1) job has a field 'idle_job' when submitting idle job. 2) field 'uuid' is used with remote user to do a rate limit. [Now]: 1) idle job has a subqueue 'idle'. 2) use email instead of 'uuid'. Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/scheduler/submit_job.cr | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/scheduler/submit_job.cr b/src/scheduler/submit_job.cr index c322d0f..c09d145 100644 --- a/src/scheduler/submit_job.cr +++ b/src/scheduler/submit_job.cr @@ -114,13 +114,6 @@ class Sched def submit_single_job(job) queue = "#{job.queue}/#{job.subqueue}" - # only single job will has "idle job" and "execute rate limiter" - if job["idle_job"].empty? - queue += "#{job.get_uuid_tag}" - else - queue = "#{queue}/idle" - end - job_id = add_task(queue, job.lab) return [{ "job_id" => "0", -- 2.23.0

1 0

[PATCH lkp-tests] lib/bootstrap.sh: judge whether has tail cmd before use it
by Wei Jihui 25 Dec '20

25 Dec '20

if is_docker is true, it will use tail cmd, so judge it before use it Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/bootstrap.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index c9a9f89f..ae902207 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -314,6 +314,8 @@ announce_bootup() redirect_stdout_stderr() { + has_cmd tail || return + if is_docker; then [ -e /tmp/stdout ] || touch /tmp/stdout [ -e /tmp/stderr ] || touch /tmp/stderr @@ -328,7 +330,6 @@ redirect_stdout_stderr() fi [ -c /dev/kmsg ] || return - has_cmd tail || return exec > /tmp/stdout exec 2> /tmp/stderr -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] lib/matrix2.rb: fix incorrect average and stddev
by Lu Weitao 25 Dec '20

25 Dec '20

[Why] We just skip useless job when make job_list --> matrix, had not did: job_list.size - 1, the average: sum($job['stats']['metric'])/job_list.size will be incorrect. standard deviation is the same reason [How] Remove useless job before job_list --> matrix, and do minor refactoring Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/matrix2.rb | 11 ++--------- lib/params_group.rb | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index 14e9558..ce64da1 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -83,7 +83,7 @@ def create_matrix(job_list) suites = [] job_list.each do |job| stats = job['stats'] - next unless stats && assign_suites(suites, job, stats) + suites << job['suite'] stats.each do |key, value| next if key.include?('timestamp') @@ -97,17 +97,10 @@ def create_matrix(job_list) return matrix, suites end -def assign_suites(suites, job, stats) - return unless job['suite'] && stats.keys.any? { |stat| stat.start_with?(job['suite']) } - - suites << job['suite'] -end - # input: query results from es_query # return: matrix def combine_query_data(query_data) - job_list = query_data['hits']['hits'] - job_list.map! { |job| job['_source'] } + job_list = extract_jobs_list(query_data['hits']['hits']) create_matrix(job_list) end diff --git a/lib/params_group.rb b/lib/params_group.rb index 57cd5c1..7bb7f5b 100644 --- a/lib/params_group.rb +++ b/lib/params_group.rb @@ -38,9 +38,21 @@ def auto_group(jobs_list, dimensions) end def extract_jobs_list(jobs_list) - jobs_list.map do |job| - job['_source'] + jobs_list.map! do |job| + job['_source'] if job_is_useful?(job) end + + jobs_list.compact +end + +def job_is_useful?(job) + stats = job['_source']['stats'] + return unless stats + + suite = job['_source']['suite'] + return unless suite && stats.keys.any? { |stat| stat.start_with?(suite) } + + true end def group(jobs_list, dimensions) -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] lib/matrix2.rb: remove duplicate code
by Lu Weitao 25 Dec '20

25 Dec '20

[Why] Had handle empty group in lib/params_group.rb def filter_groups(groups) groups.each do |group_key, value| value.each_key do |dim_key| value.delete(dim_key) if value[dim_key].empty? end groups.delete(group_key) if groups[group_key].empty? end end Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/matrix2.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index d9d86e8..14e9558 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -124,10 +124,6 @@ def combine_group_query_data(job_list, dims) suites_hash = {} groups = auto_group(job_list, dims) groups.each do |group_key, value| - if value.empty? - groups.delete(group_key) - next - end suite_list = [] value.each do |dimension_key, jobs| groups[group_key][dimension_key], suites = create_matrix(jobs) -- 2.23.0

1 0

[PATCH lkp-tests] sbin/run_job: Automatic task execution
by Hu Xuejiao 25 Dec '20

25 Dec '20

[purpose] Verify that a local job can be consumed properly Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sbin/run_job | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 sbin/run_job diff --git a/sbin/run_job b/sbin/run_job new file mode 100755 index 00000000..14e02b2f --- /dev/null +++ b/sbin/run_job @@ -0,0 +1,73 @@ +#!/bin/bash + +cci_makepkg_file="$LKP_SRC/jobs/cci-makepkg.yaml" +grep "atomic" $cci_makepkg_file || sed -i 's|benchmark:|& atomic|g' $cci_makepkg_file + +submit_one_job() +{ + cmd=( + submit -m $yaml + os=${single_arg[0]} + os_arch=${single_arg[1]} + os_version=${single_arg[2]} + os_mount=${single_arg[3]} + testbox=$testbox + queue="$tbox_group.$HOSTNAME" + ) + output=$("${cmd[@]}") + echo "$output" | grep -o -E 'job_state=finished[^\"]*' +} + +submit_job() +{ + yaml_args=("$cci_makepkg_file" "host-info.yaml") + for yaml in "${yaml_args[@]}" + do + submit_one_job & + done +} + +run_qemu() +{ + local nr_vm=4 + local testbox="vm-2p8g" + local tbox_group="vm-2p8g" + local hostname=$tbox_group.$HOSTNAME + local queues="$tbox_group.$HOSTNAME" + + $CCI_SRC/providers/multi-qemu -n "$hostname" -c "$nr_vm" -q "$queues" + + qemu_args=( + "openeuler aarch64 20.03 cifs" + "centos aarch64 7.6.1810 initramfs" + ) + for qemu_arg in "${qemu_args[@]}" + do + local single_arg=($(echo $qemu_arg)) + submit_job + done +} + +run_docker() +{ + local nr_container=2 + local tbox_group="dc-8g" + local testbox="dc-8g" + local hostname=$tbox_group.$HOSTNAME + local queues="$tbox_group.$HOSTNAME" + + $CCI_SRC/providers/multi-docker -n $hostname -c $nr_container -q $queues + + docker_args=( + "centos aarch64 7.6.1810 container" + ) + for docker_arg in "${docker_args[@]}" + do + local single_arg=($(echo $docker_arg)) + submit_job + done +} + +run_qemu +run_docker +wait -- 2.23.0

2 1

[PATCH v7 compass-ci] tutorial.md: update initramfs tutorial.md
by Wang Chenglong 25 Dec '20

25 Dec '20

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- doc/tutorial.md | 71 ++++++++++++------------------------------------- 1 file changed, 17 insertions(+), 54 deletions(-) diff --git a/doc/tutorial.md b/doc/tutorial.md index e6f8491..2409c86 100644 --- a/doc/tutorial.md +++ b/doc/tutorial.md @@ -150,61 +150,24 @@ job yaml ��[YAML](http://yaml.org/YAM #### �� - 1. �� os �� rootfs�� openEuler �� - - �� docker �� rootfs - - 1) �� openEuler �� docker �� - - ```bash - wget https://repo.openeuler.org/openEuler-20.03-LTS/docker_img/aarch64/openEuler… - ``` - 2) �� docker �� - ```bash - docker load -i openEuler-docker.aarch64 - ``` - - 3) �� openEuler �� - ```bash - docker run -id openeuler-20.03-lts - ``` - 4) �� docker �� rootfs - ```bash - docker cp -a docker run -d openeuler-20.03-lts:/ openEuler-rootfs - ``` - - �� qemu.img(qcow2��)�� rootfs (openEuler��) - - - 1) �� openEule r�� qcow2 �� - ```bash - wget https://repo.openeuler.org/openEuler-20.03-LTS/virtual_machine_img/aarch64/… - ``` - 2) ��{compass-ci}/container/qcow2rootfs ��rootfs - ```bash - cd {compass-ci}/container/qcow2rootfs - ./run openEuler-20.03-LTS.aarch64.qcow2.xz /tmp/openEuler-rootfs - ``` - 2. ��rootfs - 1. ��chroot�� rootfs (�� root ��) - ```bash - chroot openEuler-rootfs - ``` - 2. �� - - a. �� root �� - b. �� ssh �� - c. �� - d. �� docker �� osimage �� - > 1. �� - > 2. �� centos ��rpm�� - > 3. �� yum �� - > 4. �� docker �� - - 3. �� rootfs�� - ```bash - cd $rootfs - find . | coip -o -Hnewc |gzip -9 > $os_name.cgz - ``` + 1. �� openEuler �� + ```bash + cd {compass-ci}/rootfs/initramfs/openeuler/aarch64/20.03-LTS/ + ./run + ``` + + 2. ��/�� + �� {compass-ci} ��rootfs�� rootfs ��/�� + ```bash + mkdir rootfs + cd rootfs + zcat ../$os_name.cgz | cpio -idmv + find ./lib/modules/* | cpio -o -Hnewc | gzip -9 > $HOME/modules-$os_name.cgz + find ./usr/src/kernels/* | cpio -o -Hnewc | gzip -9 > $HOME/headers-$os_name.cgz + ``` + #### FAQ + 1. �� Unable to mount root fs on unknown-block�� - �� ```bash -- 2.23.0

1 0

[PATCH v5 lkp-tests] tests/build-pkg: add config file support
by Wang Yong 25 Dec '20

25 Dec '20

[Why] config file in some software need storage and recall now we can add config value in yaml and cp ${config} to ${cci_dir}/${pkgname}/ in PKGBUILD so that config file can be uploaded correctly $ cat test.yaml ... config: ${config} ... Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/build-pkg | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/build-pkg b/tests/build-pkg index addff9f7b..44bd1c10d 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -4,6 +4,7 @@ # - upstream_commit # - git_tag # - pkg_ver +# - config # upstream_url # We'll use the below global env vars, but not treat them as parameter to avoid @@ -20,6 +21,7 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/distro/common +. $LKP_SRC/lib/http.sh check_vars() { @@ -31,6 +33,24 @@ check_vars() [ -n "$os_mount" ] || die "os_mount is empty" } +get_config() +{ + local cci_http_host=${INITRD_HTTP_HOST} + local cci_http_port=${INITRD_HTTP_PORT:-8800} + export cci_dir=/cci/build-config + local remote_dir="http://${cci_http_host}:${cci_http_port}${cci_dir}/${pkgname}" + + mkdir -p ${cci_dir} + + [ -n "$config" ] || return + http_get_file "${remote_dir}/${config}" "${cci_dir}/${pkgname}/${config}" +} + +upload_config() +{ + upload_to_target_dir "${cci_dir}/${pkgname}" +} + mount_dest() { # the same image is mounted to cifs and nfs, the generated cgz files @@ -46,7 +66,7 @@ get_pkgfile() { curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' \ -d '{"git_repo": "'$1'", "git_command": ["git-show", "HEAD:'$2'"]}' -o "${2##*/}" - } +} request_pkg() { @@ -125,13 +145,17 @@ build_source_pkg() fi } replace_source PKGBUILD.src - create_cgz_name + get_config PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ $LKP_SRC/sbin/makepkg ${pkg_args} --config $LKP_SRC/etc/makepkg.conf - [ "$?" == 0 ] || exit 1 + local build_state="$?" + + upload_config + + [ "${build_state}" == "0" ] || exit 1 create_softlink upload_to_target_dir ${PKG_MNT}/${pack_to}/${pkgname} } -- 2.23.0

1 0

[PATCH v4 lkp-tests] tests/build-pkg: add config file support
by Wang Yong 25 Dec '20

25 Dec '20

[Why] config file in some software need storage and recall now we can add config value in yaml and cp ${config} to ${cci_dir}/${pkgname}/ in PKGBUILD so that config file can be uploaded correctly $ cat test.yaml ... config: ${config} ... Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/build-pkg | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/build-pkg b/tests/build-pkg index addff9f7b..06a58d498 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -4,6 +4,7 @@ # - upstream_commit # - git_tag # - pkg_ver +# - config # upstream_url # We'll use the below global env vars, but not treat them as parameter to avoid @@ -20,6 +21,7 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/distro/common +. $LKP_SRC/lib/http.sh check_vars() { @@ -31,6 +33,29 @@ check_vars() [ -n "$os_mount" ] || die "os_mount is empty" } +check_config_var() +{ + [ -n "$config" ] || return +} + +get_config() +{ + local cci_http_host=${INITRD_HTTP_HOST} + local cci_http_port=${INITRD_HTTP_PORT:-8800} + export cci_dir=/cci/build-config + local remote_dir="http://${cci_http_host}:${cci_http_port}${cci_dir}/${pkgname}" + + mkdir -p ${cci_dir} + + check_config_var + http_get_file "${remote_dir}/${config}" "${cci_dir}/${pkgname}/${config}" +} + +upload_config() +{ + upload_to_target_dir "${cci_dir}/${pkgname}" +} + mount_dest() { # the same image is mounted to cifs and nfs, the generated cgz files @@ -46,7 +71,7 @@ get_pkgfile() { curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' \ -d '{"git_repo": "'$1'", "git_command": ["git-show", "HEAD:'$2'"]}' -o "${2##*/}" - } +} request_pkg() { @@ -125,13 +150,17 @@ build_source_pkg() fi } replace_source PKGBUILD.src - create_cgz_name + get_config PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ $LKP_SRC/sbin/makepkg ${pkg_args} --config $LKP_SRC/etc/makepkg.conf - [ "$?" == 0 ] || exit 1 + local build_state="$?" + + upload_config + + [ "${build_state}" == "0" ] || exit 1 create_softlink upload_to_target_dir ${PKG_MNT}/${pack_to}/${pkgname} } -- 2.23.0

2 2

[PATCH v2 compass-ci] doc: update README.md
by Liu Yinsi 24 Dec '20

24 Dec '20

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/README.md | 113 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 87 insertions(+), 26 deletions(-) diff --git a/sparrow/README.md b/sparrow/README.md index ccb1d15..a9a4bb0 100644 --- a/sparrow/README.md +++ b/sparrow/README.md @@ -1,26 +1,87 @@ -Deploy a mini instance in one single machine. - -Prepare: - 1. hardware - Server: at least prepare a server - ProductType: ThaiShan200-2280 - Arch: aarch64 - Memory: 8G - CPU: 64 nuclear - DiskSpace: 500G - - 2. software - OS: openEuler-aarch64-20.03 LTS - git: suggest 2.23.0 - - 3. network - Internet is available - - 4. /os for store rootfs - >= 300G - -Steps: - umask 002 - git clone https://gitee.com/wu_fengguang/compass-ci.git - cd compass-ci/sparrow - ./install-tiny +# 本地搭建 compass-ci 服务器节点 + +在 openEuler 系统一键部署 compass-ci 环境，当前已支持 openEuler-aarch64-20.03-LTS 系统环境，以下配置仅供参考。 + +## 准备工作 +- 硬件 + 服务器类型：ThaiShan200-2280 (建议) + 架构：aarch64 + 内存：>= 32GB + CPU：64 nuclear (建议) + 硬盘：>= 500G (建议划分独立分区) + + >**说明：** + >划分较小独立分区详细操作请参考[添加测试用例](https://gitee.com/wu_fengguang/compass-ci/blob/mast… + >划分较大独立分区详细操作请参考[添加测试用例](https://gitee.com/wu_fengguang/compass-ci/blob/mast… + +- 软件 + OS：openEuler-aarch64-20.03 LTS + git：2.23.0版本 (建议) + 预留空间：>= 300G + 网络：可以访问互联网 + + >**说明：** + >openEuler 系统安装详细操作请参考[添加测试用例](https://openeuler.org/zh/docs/20.03_LTS/docs/Installati… + +### 操作指导 +1. 登录 openEuler 系统 + +2. 创建工作目录并设置文件权限 + ```bash + mkdir demo && cd demo && umask 002 + ``` +3. 克隆 compass-ci 项目代码到 demo 目录 + ```bash + git clone https://gitee.com/wu_fengguang/compass-ci.git + ``` +4. 执行一键部署脚本 install-tiny + ```bash + cd compass-ci/sparrow && ./install-tiny + ``` +5. 生成lkp-aarch64.cgz压缩包 + ```bash + cd /c/compass-ci/container/lkp-initrd && ./run + ``` +6. 验证账号 + ```bash + cd /c/compass-ci/sbin && ./build-my-info.rb + 按照提示输入账号名和邮箱即可 + +7. 测试环境是否可以提交job测试 + ```bash + submit iperf.yaml testbox=vm-2p8g + ``` + + 执行上述命令正常情况下会提示信息如下： + submit /c/lkp-tests/jobs/iperf.yaml failed, got job_id=0, error: Error resolving real path of '/srv/os/openeuler/aarch64/20.03/boot/vmlinuz': No such file or directory + submit /c/lkp-tests/jobs/iperf.yaml failed, got job_id=0, error: Error resolving real path of '/srv/os/openeuler/aarch64/20.03/boot/vmlinuz': No such file or directory + compass-ci搭建完毕，下面就可以开始进行测试了。 + +8. 下载rootfs（以下载openeuler/aarch64/20.03为例，需要在哪个系统上测试就去对应的/srv/os/目录下使用wget命令下载cgz文件包） + ```bash + mkdir -p /srv/os/openeuler/aarch64/ + cd /srv/os/openeuler/aarch64 + wget http://124.90.34.227:11300/os/test/openeuler/aarch64/20.03.cgz + ``` + +9. 解压cgz包 + ```bash + gzip -dc 20.03.cgz | cpio -idv + ``` + +10. [使用 compass-ci 平台测试开源项目](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/… + +11. [编写测试用例](https://gitee.com/wu_fengguang/lkp-tests/blob/master/doc/add-testca… + +12. 使用[submit命令](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manu… + 注意：os_mount 必须指定为cifs + +13. 运行测试任务 + ```bash + cd /c/compass-ci/providers/ && ./my-qemu.sh + ``` + +14. [查看任务结果](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/br… id查看output文件 + ```bash + cd /srv/result/iperf/$job_id/output + ``` -- 2.23.0

2 1

[PATCH lkp-tests] bootstrap.sh: need a blank line after single return
by Xu Xijian 24 Dec '20

24 Dec '20

A blank line is needed after single return, more than 1 return cmd together doesn't need. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- lib/bootstrap.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index ad1c4a2f2..bd168a62a 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -235,7 +235,6 @@ run_ntpdate() [ -z "$NO_NETWORK" ] || return [ -n "$LKP_SERVER" ] || return is_clearlinux && clearlinux_timesync && return - [ -x '/usr/sbin/ntpdate' ] || return local hour="$(date +%H)" @@ -371,6 +370,7 @@ install_deb() # solved the dependent relationship, so here we ignore the dependency errors echo "install debs round one: dpkg -i --force-confdef --force-depends $files" dpkg -i --force-confdef --force-depends $files 2>/tmp/dpkg_error && return + grep -v "$filter_info" /tmp/dpkg_error # round two, install all debs one by one accroding to keep-deb which is in sequence @@ -397,6 +397,7 @@ install_deb() install_rpms() { [ -d /opt/rpms ] || return + echo "install $(ls /opt/rpms/*.rpm)" rpm -ivh --force --ignoresize /opt/rpms/*.rpm } @@ -458,6 +459,7 @@ fixup_packages() mount_debugfs() { is_docker && return + check_mount debug /sys/kernel/debug -t debugfs } @@ -467,6 +469,7 @@ cleanup_pkg_cache() local pkg_cache=$1 local cleanup_stamp=$pkg_cache/cleanup_stamp/$(date +%U) [ -d "$cleanup_stamp" ] && return + mkdir $cleanup_stamp -p find "$pkg_cache" $ -type f -mtime +7 -delete $ -or $ -type d -ctime +7 -empty -delete $ @@ -497,6 +500,7 @@ mount_rootfs() set_job_state 'load_disk_fail' return 1 } + ROOTFS_DIR=/opt/rootfs mkdir -p $ROOTFS_DIR mount $rootfs_partition $ROOTFS_DIR || { @@ -536,7 +540,6 @@ netconsole_init() { # don't init netconsole at local run [ "$LKP_LOCAL_RUN" = "1" ] && return - [ -z "$netconsole_port" ] && return [ -n "$NO_NETWORK" ] && return @@ -545,6 +548,7 @@ netconsole_init() # Select the interface that can access netconsole server netconsole_interface=$(show_default_interface) [ -n "$netconsole_server" ] || return + # eth0 is default interface if netconsole_interface is null. modprobe netconsole netconsole=@/$netconsole_interface,$netconsole_port@$netconsole_server/ } @@ -755,6 +759,7 @@ boot_init() install_pkgs() { [ -d /opt/pkgs ] || return + echo "install $(ls /opt/pkgs/*.pkg.tar*)" pacman -U --needed --noconfirm /opt/pkgs/*.pkg.tar* } -- 2.23.0

1 0