December 2020 - Compass-ci - mailweb.openeuler.org

[PATCH v5 lkp-tests] lkp-tests/tests: add rpmbuild-pkg to build RPM
by Zhang Yale 10 Dec '20

10 Dec '20

Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- tests/rpmbuild-pkg | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100755 tests/rpmbuild-pkg diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg new file mode 100755 index 00000000..156ca0f6 --- /dev/null +++ b/tests/rpmbuild-pkg @@ -0,0 +1,59 @@ +#!/bin/bash +# - rpm_repo + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/upload.sh + +[ -n "$rpm_repo" ] || die "rpm_repo is empty" +benchmark=${rpm_repo##*/} +sync_dest="/initrd/rpmbuild-pkg/${os_mount}/${os}/${os_arch}/${os_version}/${benchmark}" + +init_workspace() +{ + # generate workspace in ${HOME} + rpmdev-setuptree +} + +get_pkgfile() +{ + curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' \ + -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-show", "HEAD:'$1'"]}' -o "${2}" +} + +download_rpm_repo() +{ + filelist=$(curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' \ + -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-ls-files", "."]}') + for pkgfile in ${filelist[*]} + do + local dir="SOURCES" + echo $pkgfile | egrep "\.spec$" && dir="SPECS" + get_pkgfile "$pkgfile" "${HOME}/rpmbuild/${dir}/$pkgfile" + done +} + +build_rpm() +{ + spec_dir=${HOME}/rpmbuild/SPECS/$benchmark.spec + sed -i 's/^$Source[^ ]*:[ \t]*$https/\1http/g' `grep http -rl $benchmark.spec` + # Install build depends + yum-builddep -y $spec_dir + # Download tar.gz to default path ${HOME}/rpmbuild/SOURCE + spectool -g -R $spec_dir + # Building rpm or srpm packages + rpmbuild -ba $spec_dir +} + +upload_rpm_pkg() +{ + # rpm package will be generated in "${HOME}/rpmbuild/SRPMS" and "${HOME}/rpmbuild/RPMS" + for file in $(find ${HOME}/rpmbuild/ -type f -name "*.rpm") + do + upload_one_curl ${file} ${sync_dest} + done +} + +init_workspace +download_rpm_repo +build_rpm +upload_rpm_pkg -- 2.23.0

1 0

[PATCH v2 lkp-tests 3/3] iso2rootfs: allow install some packages to result rootfs
by Yu Chuan 10 Dec '20

10 Dec '20

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 4 ++++ tests/iso2rootfs | 34 ++++++++++++++++++++++++++++------ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index c67faf44147d..a84f7583de94 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -34,6 +34,10 @@ iso2rootfs: # - no_fstab: will comment all line in /etc/fstab. config_rootfs: + ## install pkgs for result rootfs + ## - example: vim, git, xxx + rootfs_install_pkgs: + ################# # dailybuild related fields ################# diff --git a/tests/iso2rootfs b/tests/iso2rootfs index cd64044f0833..8625b6484d89 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -203,17 +203,24 @@ get_cgz() tar -xf "${cgz_path_t}" -C "${CGZ_PATH}" } -install_pkgs() +get_pkg_installer() { - local pkg_installer - command -v yum && pkg_installer="yum" + local installer + + has_cmd yum && installer=yum + has_cmd apt && installer=apt - [[ -n "$pkg_installer" ]] || die "cannot find pkg installer." + [ -n "$installer" ] || die "cannot find pkg installer." + echo $installer +} + +install_pkgs() +{ + local installer=$(get_pkg_installer) - local pt for pt do - "$pkg_installer" install -y "$pt" || die "cannot install pkg: $pt." + "$installer" install -y "$pt" || die "cannot install pkg: $pt." done } @@ -367,11 +374,26 @@ disable_fstab() sed -i 's/^$[^#]$/# \1/g' "${ROOTFS_DES_DIR}/etc/fstab" } +install_pkgs_for_rootfs() +{ + [ -n "$rootfs_install_pkgs" ] || return 0 + + local installer=$(get_pkg_installer) + + if [ "$installer" == "yum" ]; then + "$installer" install -y --skip-broken --installroot=${ROOTFS_DES_DIR} ${rootfs_install_pkgs//,/ } + elif [ "$installer" == "apt" ]; then + "$installer" install -y --ignore-missing --fix-missing --fix-broken -o RootDir=${ROOTFS_DES_DIR} ${rootfs_install_pkgs//,/ } + fi +} + config_rootfs() { config_dns_resolver disable_selinux disable_fstab + + install_pkgs_for_rootfs } ############ test rootfs ############ -- 2.23.0

1 0

[PATCH v2 lkp-tests 2/3] iso2rootfs: add three required fields when submit test job.yaml
by Yu Chuan 10 Dec '20

10 Dec '20

[Why] Since the ability `verify account information` was added on submit job.yaml, so add three required fields when submit test job.yaml. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 52 ++++++++++++++++++++++++++++++++++---------- tests/iso2rootfs | 14 ++++++++++-- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 9123bef24ae5..c67faf44147d 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -1,35 +1,63 @@ suite: iso2rootfs category: benchmark iso2rootfs: + ################# # iso related fields to be used to generate rootfs + ################# + iso_os: iso_arch: iso_version: + ################# # place the result rootfs related fields - ## 1. result rootfs will be placed in the following localtion on the - ## remote file server: - ## - {remote_file_server}/{rootfs_path}/{iso_os}/{iso_arch}/ - ## 2. remote file server protocols current supported: - ## - nfs - ## - cifs + ################# + + # 1. Result rootfs will be placed in the following location on the + # remote file server: + # - {remote_file_server}/{rootfs_path}/{iso_os}/{iso_arch}/ + # 2. Remote file server protocols current supported: + # - nfs + # - cifs rootfs_protocol: rootfs_server: rootfs_path: + ################# # config rootfs related fields - ## supported fields: - ## - dns: will config /etc/resolv.conf. - ## - no_selinux: will disable selinux in /etc/selinux/config. - ## - no_fstab: will comment all line in /etc/fstab. + ################# + + # you can config add some configurations of the result rootfs. + # supported fields: + # - dns: will config /etc/resolv.conf. + # - no_selinux: will disable selinux in /etc/selinux/config. + # - no_fstab: will comment all line in /etc/fstab. config_rootfs: + ################# # dailybuild related fields + ################# + + # dailybuild_iso_url_file: + # - The `dailybuild iso url file` content is the url of a iso. + # - The iso checksum file also exists on the network, and the checksum file path is "{dailybuild_iso_url}.check256sum" + # - demo: + # root@localhost ~% curl http://1.1.1.1/dailybuilds/openEuler-20.03-LTS-aarch64-LTS/release_iso + # http://1.1.1.1//dailybuilds/1970-01-01-00-00-00/openEuler-20.03-LTS-aarch64… dailybuild_iso_url_file: + ################# # submit test yaml related fields - ## 1. you can add as many jobs as you like. - ## 2. the following three fields is requied for every test job. + ################# + + # three required fields when submit test yaml + submit_email: + submit_name: + submit_uuid: + + # submit target tests yaml + ## 1. You can add as many jobs as you like. + ## 2. The following three fields is required for every test job. test1_yaml: test1_os_mount: test1_testbox: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index ce02080875ce..cd64044f0833 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -448,11 +448,21 @@ EOF ############ submit test job yaml ############ generate_global_yaml() { + [ -n "${submit_email}" ] || die "cannot find value of var: submit_email" + [ -n "${submit_name}" ] || die "cannot find value of var: submit_name" + [ -n "${submit_uuid}" ] || die "cannot find value of var: submit_uuid" + local config_yaml="/etc/compass-ci/defaults/sparrow.yaml" mkdir -p "$(dirname "${config_yaml}")" - echo "SCHED_HOST: ${SCHED_HOST}" >> "${config_yaml}" - echo "SCHED_PORT: ${SCHED_PORT}" >> "${config_yaml}" + + cat <<-EOF > "${config_yaml}" + SCHED_HOST: ${SCHED_HOST} + SCHED_PORT: ${SCHED_PORT} + my_email: ${submit_email} + my_name: ${submit_name} + my_uuid: ${submit_uuid} + EOF } generate_submit_append_str() -- 2.23.0

1 0

[PATCH v2 lkp-tests 1/3] iso2rootfs: result rootfs support disable fstab
by Yu Chuan 10 Dec '20

10 Dec '20

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 1 + tests/iso2rootfs | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 2c4d3a93ff62..9123bef24ae5 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -21,6 +21,7 @@ iso2rootfs: ## supported fields: ## - dns: will config /etc/resolv.conf. ## - no_selinux: will disable selinux in /etc/selinux/config. + ## - no_fstab: will comment all line in /etc/fstab. config_rootfs: # dailybuild related fields diff --git a/tests/iso2rootfs b/tests/iso2rootfs index 9adbc930b48f..ce02080875ce 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -361,10 +361,17 @@ disable_selinux() sed -i 's/^SELINUX=.*/SELINUX=disabled/g' "${ROOTFS_DES_DIR}/etc/selinux/config" } +disable_fstab() +{ + echo "${config_rootfs}" | grep -qw 'no_fstab' || return 0 + sed -i 's/^$[^#]$/# \1/g' "${ROOTFS_DES_DIR}/etc/fstab" +} + config_rootfs() { config_dns_resolver disable_selinux + disable_fstab } ############ test rootfs ############ -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] scheduler: subqueue: consume jobs from subqueue
by Ren Wen 10 Dec '20

10 Dec '20

Find subqueues related to the host default queues of all users, and consume according to random sequence. Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/scheduler/constants.cr | 2 ++ src/scheduler/find_job_boot.cr | 11 ++++++++++- src/scheduler/redis_client.cr | 4 ++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/scheduler/constants.cr b/src/scheduler/constants.cr index b2ea831..1146b21 100644 --- a/src/scheduler/constants.cr +++ b/src/scheduler/constants.cr @@ -1,6 +1,8 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +require "../taskqueue/constants" + JOB_REDIS_HOST = "172.17.0.1" JOB_REDIS_PORT = 6379 JOB_REDIS_PORT_DEBUG = 6380 diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 3e33604..fb94e91 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -56,7 +56,16 @@ class Sched queues << item.strip end - return rand_queues(queues) + sub_queues = [] of String + queues.each do |queue| + matched_queues = @redis.keys("#{QUEUE_NAME_BASE}/sched/#{queue}/*/ready") + matched_queues.each do |mq| + match_data = "#{mq}".match(%r(^#{QUEUE_NAME_BASE}/sched/(#{queue}/.+)/ready$)) + sub_queues << match_data[1] if match_data + end + end + + return rand_queues(sub_queues) end def get_job_from_queues(queues, testbox) diff --git a/src/scheduler/redis_client.cr b/src/scheduler/redis_client.cr index b816d67..1253108 100644 --- a/src/scheduler/redis_client.cr +++ b/src/scheduler/redis_client.cr @@ -16,6 +16,10 @@ class Redis::Client @client = Redis::PooledClient.new(host: host, port: port, pool_size: 25, pool_timeout: 0.01) end + def keys(pattern) + @client.keys(pattern) + end + def hash_set(key : String, field, value) @client.hset(key, field.to_s, value.to_s) end -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] scheduler: subqueue: submit user's job to subqueue
by Ren Wen 10 Dec '20

10 Dec '20

Every user has a subqueue, user's email by default. Queue will be "queue/subqueue" when submitting job. Example: queue : vm-2p8g.aarch64 subqueue: zs(a)xxx.com will submit to: vm-2p8g.aarch64/zs(a)xxx.com Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/lib/job.cr | 9 +++++++++ src/scheduler/submit_job.cr | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/lib/job.cr b/src/lib/job.cr index 7ded6d8..5e56fff 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -80,6 +80,7 @@ class Job testbox lab queue + subqueue initrd_pkg initrd_deps initrds_uri @@ -161,6 +162,7 @@ class Job set_lkp_server() set_sshr_info() set_queue() + set_subqueue() end private def set_kernel @@ -266,6 +268,13 @@ class Job end end + private def set_subqueue + return unless self["subqueue"].empty? + + # set default subqueue + self["subqueue"] = self["my_email"] + end + # if not assign tbox_group, set it to a match result from testbox # ?if job special testbox, should we just set tbox_group=textbox private def update_tbox_group_from_testbox diff --git a/src/scheduler/submit_job.cr b/src/scheduler/submit_job.cr index f93ad2b..7841736 100644 --- a/src/scheduler/submit_job.cr +++ b/src/scheduler/submit_job.cr @@ -108,6 +108,8 @@ class Sched # failure: [{"job_id" => "0", "message" => err_msg, job_state => "submit"}] def submit_single_job(job) queue = job.queue + subqueue = job.subqueue + queue += "/#{subqueue}" # only single job will has "idle job" and "execute rate limiter" if job["idle_job"].empty? -- 2.23.0

1 0

[PATCH compass-ci] minio/start: add localtime
by Cui Yili 10 Dec '20

10 Dec '20

Signed-off-by: cuiyili <2268260388(a)qq.com> --- container/minio/start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/minio/start b/container/minio/start index 348bff7..1798cfa 100755 --- a/container/minio/start +++ b/container/minio/start @@ -9,6 +9,7 @@ cmd=( docker run -d -p 11321:9000 --name minio + -v /etc/localtime:/etc/localtime:ro -v /srv/minio/:/srv/minio/ -e "MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY}" -e "MINIO_SECRET_KEY=${MINIO_SECRET_KEY}" -- 2.23.0

1 0

Re: [PATCH lkp-tests] jobs: rename pub_key to ssh_pub_key
by Zhang Yale 10 Dec '20

10 Dec '20

On Thu, Dec 10, 2020 at 05:10:47PM +0800, Li Ping wrote: >On Thu, Dec 10, 2020 at 03:31:54PM +0800, Zhang Yale wrote: >>As top level job key, the name should be more exact. > > if you make changes in these files, > you should also synchronize the changes in those > documents. > and remind everyone. > > Thanks, > Ping The relevant document has been modified by someone else. Thanks, Yale > >> >>Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> >>--- >> jobs/borrow-10d.yaml | 2 +- >> jobs/borrow-1d.yaml | 2 +- >> jobs/borrow-1h.yaml | 2 +- >> 3 files changed, 3 insertions(+), 3 deletions(-) >> >>diff --git a/jobs/borrow-10d.yaml b/jobs/borrow-10d.yaml >>index a2b50025..e0e35e17 100644 >>--- a/jobs/borrow-10d.yaml >>+++ b/jobs/borrow-10d.yaml >>@@ -1,7 +1,7 @@ >> suite: borrow >> testcase: borrow >> >>-pub_key: <%= >>+ssh_pub_key: <%= >> begin >> File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp >> rescue >>diff --git a/jobs/borrow-1d.yaml b/jobs/borrow-1d.yaml >>index 7767f2d7..c31d7ebd 100644 >>--- a/jobs/borrow-1d.yaml >>+++ b/jobs/borrow-1d.yaml >>@@ -3,7 +3,7 @@ testcase: borrow >> >> timeout: 1d >> >>-pub_key: <%= >>+ssh_pub_key: <%= >> begin >> File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp >> rescue >>diff --git a/jobs/borrow-1h.yaml b/jobs/borrow-1h.yaml >>index f2fedd20..d6b72c43 100644 >>--- a/jobs/borrow-1h.yaml >>+++ b/jobs/borrow-1h.yaml >>@@ -1,7 +1,7 @@ >> suite: borrow >> testcase: borrow >> >>-pub_key: <%= >>+ssh_pub_key: <%= >> begin >> File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp >> rescue >>-- >>2.23.0 >>

1 0

Re: [PATCH lkp-tests] jobs: rename pub_key to ssh_pub_key
by Zhang Yale 10 Dec '20

10 Dec '20

On Thu, Dec 10, 2020 at 05:10:47PM +0800, Li Ping wrote: >On Thu, Dec 10, 2020 at 03:31:54PM +0800, Zhang Yale wrote: >>As top level job key, the name should be more exact. > > if you make changes in these files, > you should also synchronize the changes in those > documents. > and remind everyone. > > Thanks, > Ping Ok, I got it. Thanks, Yale > >> >>Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> >>--- >> jobs/borrow-10d.yaml | 2 +- >> jobs/borrow-1d.yaml | 2 +- >> jobs/borrow-1h.yaml | 2 +- >> 3 files changed, 3 insertions(+), 3 deletions(-) >> >>diff --git a/jobs/borrow-10d.yaml b/jobs/borrow-10d.yaml >>index a2b50025..e0e35e17 100644 >>--- a/jobs/borrow-10d.yaml >>+++ b/jobs/borrow-10d.yaml >>@@ -1,7 +1,7 @@ >> suite: borrow >> testcase: borrow >> >>-pub_key: <%= >>+ssh_pub_key: <%= >> begin >> File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp >> rescue >>diff --git a/jobs/borrow-1d.yaml b/jobs/borrow-1d.yaml >>index 7767f2d7..c31d7ebd 100644 >>--- a/jobs/borrow-1d.yaml >>+++ b/jobs/borrow-1d.yaml >>@@ -3,7 +3,7 @@ testcase: borrow >> >> timeout: 1d >> >>-pub_key: <%= >>+ssh_pub_key: <%= >> begin >> File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp >> rescue >>diff --git a/jobs/borrow-1h.yaml b/jobs/borrow-1h.yaml >>index f2fedd20..d6b72c43 100644 >>--- a/jobs/borrow-1h.yaml >>+++ b/jobs/borrow-1h.yaml >>@@ -1,7 +1,7 @@ >> suite: borrow >> testcase: borrow >> >>-pub_key: <%= >>+ssh_pub_key: <%= >> begin >> File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp >> rescue >>-- >>2.23.0 >>

1 0

[PATCH lkp-tests] lib/job.rb: delete useless var
by Wei Jihui 10 Dec '20

10 Dec '20

@job2 is {} all the time, so it is useless. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/job.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/job.rb b/lib/job.rb index e94b6c85..b41d55a7 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -433,7 +433,6 @@ class Job i = include_files job = deepcopy(@job) - revise_hash(job, deepcopy(@job2), true) job['___'] = nil @jobx = job if @job.include?(:expand_params) @@ -498,8 +497,6 @@ class Job return unless first_time - revise_hash(@job, @job2, true) - return if @overrides.empty? key = comment_to_symbol('user overrides') @@ -729,7 +726,6 @@ class Job def each_jobs(&block) each_job_init job = deepcopy(@job) - @job2 = {} load_defaults load_hosts_config each_job_init @@ -738,7 +734,6 @@ class Job @load_hosts_done = false @job = deepcopy(job) @job.merge!(hash) - @job2 = {} load_defaults load_hosts_config each_job_init -- 2.23.0

1 0

2024

2023

2022

2021

2020

Compass-ci December 2020