- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci] sbin/auto_submit: add support for different branch different job
by Li Yuanchao 08 Feb '21

08 Feb '21

Now can support some branch use specific submit commands and others use common submit commands for the same repo. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 78 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/sbin/auto_submit b/sbin/auto_submit index 410445d..6d29543 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -30,21 +30,6 @@ class AutoSubmit return pkgbuild_repo end - def submit(newrefs_info, submit_argv) - newrefs_info['new_refs']['heads'].each do |branch, commit_id| - inactive_time = %x(git -C /srv/git/#{newrefs_info['git_repo']}.git log --pretty=format:"%cr" -1 #{commit_id}) - next if inactive_time =~ /(month|year)/ - - real_argvs = Array.new(submit_argv) - real_argvs.push("upstream_branch=#{branch.delete_prefix('refs/heads/')}") - real_argvs.push("upstream_commit=#{commit_id}") - tag = %x(git -C /srv/git/#{newrefs_info['git_repo']}.git tag --points-at #{commit_id}) - real_argvs.push("upstream_tag=#{tag}") unless tag.empty? - - system(real_argvs.join(' ')) - end - end - def get_argvs(newrefs_info) git_repo = newrefs_info['git_repo'] puts "git_repo : #{git_repo}" @@ -60,16 +45,77 @@ class AutoSubmit submit_argv end + def submit_one_job(submit_argv, git_repo, branch, commit_id) + inactive_time = %x(git -C /srv/git/#{git_repo}.git log --pretty=format:"%cr" -1 #{commit_id}) + return if inactive_time =~ /(month|year)/ + + real_argvs = Array.new(submit_argv) + real_argvs.push("upstream_branch=#{branch}") + real_argvs.push("upstream_commit=#{commit_id}") + tag = %x(git -C /srv/git/#{git_repo}.git tag --points-at #{commit_id}) + real_argvs.push("upstream_tag=#{tag}") unless tag.empty? + + system(real_argvs.join(' ')) + end + + def get_branch_commands(submit_commands) + submit_commands.each do |element| + return element['branch_commands'] if element['branch_commands'] + end + return nil + end + + def submit_specific_branch(submit_argv, newrefs_info, branch_commands) + return newrefs_info['new_refs']['heads'] if branch_commands.nil? + + non_specific_commits = {} + newrefs_info['new_refs']['heads'].each do |branch, commit_id| + branch_name = branch.delete_prefix('refs/heads/') + if branch_commands[branch_name].nil? + non_specific_commits[branch] = commit_id + next + end + argvs = Array.new(submit_argv) + argvs.push(branch_commands[branch_name]) + submit_one_job(argvs, newrefs_info['git_repo'], branch_name, commit_id) + end + return non_specific_commits + end + + def submit_non_specific(non_specific_commits, git_repo, submit_argv) + non_specific_commits.each do |branch, commit_id| + branch = branch.delete_prefix('refs/heads/') + submit_one_job(submit_argv, git_repo, branch, commit_id) + end + end + + # Add support of different branch different job + # newrefs_info['submit'] is like: + # ["submit command 1", + # "submit command 2", + # ..., + # { "branch_commands" => { "branch_1" => "submit command 1", + # ..., + # "branch_n" => "submit command n" + # } + # } + # ] def submit_job(newrefs_info) submit_argv = get_argvs(newrefs_info) return unless submit_argv return unless newrefs_info['submit'] + branch_commands = get_branch_commands(newrefs_info['submit']) + non_specific_commits = submit_specific_branch(submit_argv, newrefs_info, branch_commands) + return if non_specific_commits.empty? + newrefs_info['submit'].each do |argv_config| + next if argv_config.is_a? Hash + argvs = Array.new(submit_argv) argvs.push(argv_config) - submit(newrefs_info, argvs) + submit_non_specific(non_specific_commits, newrefs_info['git_repo'], argvs) end end -- 2.23.0

1 0

[PATCH v4 compass-ci 1/3] container/defconfig: add function docker_skip_rebuild
by Liu Yinsi 08 Feb '21

08 Feb '21

When execute `sparrow/4-docker/buildall run-only`, if docker image already exists, just exit build script, not rebuild container. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/defconfig.rb | 6 ++++++ container/defconfig.sh | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/container/defconfig.rb b/container/defconfig.rb index f9b0824..7dc9f33 100755 --- a/container/defconfig.rb +++ b/container/defconfig.rb @@ -47,3 +47,9 @@ def get_available_memory # take the middle value according to the system memory size. [1024, 30720, Math.sqrt(memtotal) * 1024].sort[1].to_i end + +def docker_skip_rebuild(tag) + return unless ENV['action'].include? 'run-only' + + exit 1 if system "docker image inspect #{tag} > /dev/null 2>&1" +end diff --git a/container/defconfig.sh b/container/defconfig.sh index 54e65b5..7c575ef 100755 --- a/container/defconfig.sh +++ b/container/defconfig.sh @@ -41,3 +41,11 @@ push_image() docker tag "$src_tag" "$dst_tag" docker push "$dst_tag" } + +docker_skip_rebuild() +{ + tag=$1 + [ "$action" != "run-only" ] && return + docker image inspect $tag > /dev/null 2>&1 + [ "$?" == "0" ] && exit 1 +} -- 2.23.0

1 0

[PATCH v4 compass-ci] container/register-accounts: fix failed to register account
by Liu Yinsi 08 Feb '21

08 Feb '21

[error] root@taishan200-2280-2s64p-256g--a9 /c/compass-ci/container/register-accounts# ./start register-accounts register-accounts /c/compass-ci/container/register-accounts # ./register-accounts awk: /etc/compass-ci/defaults/*.yaml: No such file or directory "option: {\"my_email\"=>\"\", \"my_name\"=>\"\", \"lab\"=>\"\"}" Traceback (most recent call last): 10: from ./register-accounts:8:in `<main>' 9: from ./register-accounts:8:in `require_relative' 8: from /c/compass-ci/sbin/build-my-info.rb:57:in `<top (required)>' 7: from /c/compass-ci/sbin/build-my-info.rb:48:in `build_my_info' 6: from /c/compass-ci/lib/build_my_info_client.rb:30:in `search_my_info' 5: from /c/compass-ci/lib/es_query.rb:58:in `query_by_id' 4: from /usr/lib/ruby/gems/2.7.0/gems/elasticsearch-api-7.10.1/lib/elasticsearch/api/actions/search.rb:103:in `search' 3: from /usr/lib/ruby/gems/2.7.0/gems/elasticsearch-transport-7.10.1/lib/elasticsearch/transport/client.rb:182:in `perform_request' 2: from /usr/lib/ruby/gems/2.7.0/gems/elasticsearch-transport-7.10.1/lib/elasticsearch/transport/transport/http/faraday.rb:37:in `perform_request' 1: from /usr/lib/ruby/gems/2.7.0/gems/elasticsearch-transport-7.10.1/lib/elasticsearch/transport/transport/base.rb:347:in `perform_request' /usr/lib/ruby/gems/2.7.0/gems/elasticsearch-transport-7.10.1/lib/elasticsearch/transport/transport/base.rb:218:in `__raise_transport_error': [400] {"error":{"root_cause":[{"type":"query_shard_exception","reason":"failed to create query: {\\n \\"bool\\" : {\\n \\"must\\" : [\\n {\\n \\"term\\" : {\\n \\"_id\\" : {\\n \\"value\\" : \\"\\",\\n \\"boost\\" : 1.0\\n }\\n }\\n }\\n ],\\n \\"adjust_pure_negative\\" : true,\\n \\"boost\\" : 1.0\\n }\\n}","index_uuid":"MH_AlDBvTYa9sQhwsESXgQ","index":"accounts"}],"type":"search_phase_execution_exception","reason":"all shards failed","phase":"query","grouped":true,"failed_shards":[{"shard":0,"index":"accounts","node":"I1usO2axQ9-XZhzcX3oHxw","reason":{"type":"query_shard_exception","reason":"failed to create query: {\\n \\"bool\\" : {\\n \\"must\\" : [\\n {\\n \\"term\\" : {\\n \\"_id\\" : {\\n \\"value\\" : \\"\\",\\n \\"boost\\" : 1.0\\n }\\n }\\n }\\n ],\\n \\"adjust_pure_negative\\" : true,\\n \\"boost\\" : 1.0\\n }\\n}","index_uuid":"MH_AlDBvTYa9sQhwsESXgQ","index":"accounts","caused_by":{"type":"illegal_argument_exception","reason":"Ids can't be empty"}}}]},"status":400} (Elasticsearch::Transport::Transport::Errors::BadRequest) refer code: /c/compass-ci/sbin/build-my-info.rb: ... option = { 'my_email' => `git config --global user.email`.chomp, 'my_name' => `git config --global user.name`.chomp, 'lab' => `awk '/^lab:\s/ {print $2; exit}' /etc/compass-ci/defaults/*.yaml`.chomp } ... build_my_info(option) ... [why] when call build-my-info.rb, we just want call function build_my_info(), no need to execute option, in caller script already define option values [how] use `if $PROGRAM_NAME == __FILE__`, when sbin/build-my-info.rb is not executed independently but invoked by other files, avoid to execute some part code. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/build-my-info.rb | 72 ++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/sbin/build-my-info.rb b/sbin/build-my-info.rb index dbe093d..c7a7798 100755 --- a/sbin/build-my-info.rb +++ b/sbin/build-my-info.rb @@ -7,49 +7,51 @@ require_relative '../lib/es_client' require_relative '../lib/build_my_info_client' require 'optparse' -option = { - 'my_email' => `git config --global user.email`.chomp, - 'my_name' => `git config --global user.name`.chomp, - 'lab' => `awk '/^lab:\s/ {print $2; exit}' /etc/compass-ci/defaults/*.yaml`.chomp -} +def build_my_info(option) + build_my_info = BuildMyInfo.new(option['my_email']) + info_es = build_my_info.search_my_info + info_es.update option + info_es['my_token'] = %x(uuidgen).chomp if info_es['my_token'].nil? -options = OptionParser.new do |opts| - opts.banner = "Usage: build-my-info [-e email] [-n name] [-l lab] [-t]\n" + build_my_info.config_my_info(info_es) +end - opts.separator '' - opts.separator 'options:' +if $PROGRAM_NAME == __FILE__ + option = { + 'my_email' => `git config --global user.email`.chomp, + 'my_name' => `git config --global user.name`.chomp, + 'lab' => `awk '/^lab:\s/ {print $2; exit}' /etc/compass-ci/defaults/*.yaml`.chomp + } - opts.on('-e email', 'my_email') do |email| - option['my_email'] = email - end + options = OptionParser.new do |opts| + opts.banner = "Usage: build-my-info [-e email] [-n name] [-l lab] [-t]\n" - opts.on('-n name', 'my_name') do |name| - option['my_name'] = name - end + opts.separator '' + opts.separator 'options:' - opts.on('-l lab', 'lab') do |lab| - option['lab'] = lab - end + opts.on('-e email', 'my_email') do |email| + option['my_email'] = email + end - opts.on('-t', 'my_token') do - option['my_token'] = %x(uuidgen).chomp - end + opts.on('-n name', 'my_name') do |name| + option['my_name'] = name + end - opts.on_tail('-h', 'show this message') do - puts opts - exit - end -end + opts.on('-l lab', 'lab') do |lab| + option['lab'] = lab + end -options.parse! + opts.on('-t', 'my_token') do + option['my_token'] = %x(uuidgen).chomp + end -def build_my_info(option) - build_my_info = BuildMyInfo.new(option['my_email']) - info_es = build_my_info.search_my_info - info_es.update option - info_es['my_token'] = %x(uuidgen).chomp if info_es['my_token'].nil? + opts.on_tail('-h', 'show this message') do + puts opts + exit + end + end - build_my_info.config_my_info(info_es) -end + options.parse! -build_my_info(option) + build_my_info(option) +end -- 2.23.0

1 0

[PATCH lkp-tests] sbin/install-dependencies.sh: delete redundant code
by Wang Yong 08 Feb '21

08 Feb '21

write_shell_profile function will source profile, no need source twice Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- sbin/install-dependencies.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/sbin/install-dependencies.sh b/sbin/install-dependencies.sh index ed8aeb927..b687dc0d6 100755 --- a/sbin/install-dependencies.sh +++ b/sbin/install-dependencies.sh @@ -65,7 +65,6 @@ set_env() write_host write_shell_profile "export LKP_SRC=$PWD" write_shell_profile "export PATH=\$PATH:\$LKP_SRC/sbin:\$LKP_SRC/bin" - source $(shell_profile) } set_env -- 2.23.0

1 0

[PATCH v3 lkp-tests] sbin/install-dependencies.sh: change install way
by Wang Yong 08 Feb '21

08 Feb '21

[before] install packages from $DISTRO detect [after] install packages from Package Manager it's not convenient when install packages in a new $DISTRO Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- sbin/install-dependencies.sh | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/sbin/install-dependencies.sh b/sbin/install-dependencies.sh index b5b070f01..ed8aeb927 100755 --- a/sbin/install-dependencies.sh +++ b/sbin/install-dependencies.sh @@ -5,28 +5,36 @@ PROJECT_DIR=$(dirname $SCRIPT_DIR) . $PROJECT_DIR/lib/env.sh -# choose install function base on DISTRIBUTION +# choose install function base on common Package Manager linux_dep() { - source /etc/os-release - case $ID in - ubuntu|debian) + get_package_manager + + case "$installer" in + apt-get) export DEBIAN_FRONTEND=noninteractive - sudo apt-get install -yqm ruby-dev libssl-dev gcc g++ uuid-runtime + sudo "$installer" install -yqm ruby-dev libssl-dev gcc g++ uuid-runtime + ;; + dnf|yum) + sudo "$installer" install -y --skip-broken ruby rubygems gcc gcc-c++ make ruby-devel git lftp util-linux ;; - openEuler|fedora|rhel|centos) - if type dnf > /dev/null 2>&1; then - sudo dnf install -y --skip-broken ruby rubygems gcc gcc-c++ make ruby-devel git lftp util-linux - else - sudo yum install -y --skip-broken ruby rubygems gcc gcc-c++ make ruby-devel git lftp util-linux - fi + pacman) + sudo "$installer" -Sy --noconfirm --needed ruby rubygems gcc make git lftp util-linux ;; *) - echo "$ID not support! please install dependencies manually." && exit 1 + echo "Unknown Package Manager! please install dependencies manually." && exit 1 ;; esac } +get_package_manager() +{ + has_cmd "yum" && installer="yum" + has_cmd "dnf" && installer="dnf" && return + has_cmd "apt-get" && installer="apt-get" && return + has_cmd "pacman" && installer="pacman" && return +} + mac_dep() { brew install ruby -- 2.23.0

2 2

[PATCH compass-ci] sbin/build-my-info: fix no key: my_ssh_pubkey
by Luan Shengde 08 Feb '21

08 Feb '21

[why] my_ssh_pubkey is needed sometimes when submit jobs: $ submit -m -c job.yaml -i ~/lkp-tests/jobs/ssh-on-fail.yaml submit_id=f87d2951-c226-4e51-a3e4-c71818a4c974 submit job.yaml failed, got job id=0, error: Missing hash key: "my_ssh_pubkey" [how] add initialize my_ssh_pubkey when building my_info. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- sbin/build-my-info.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/sbin/build-my-info.rb b/sbin/build-my-info.rb index dbe093d..6be08a4 100755 --- a/sbin/build-my-info.rb +++ b/sbin/build-my-info.rb @@ -48,6 +48,7 @@ def build_my_info(option) info_es = build_my_info.search_my_info info_es.update option info_es['my_token'] = %x(uuidgen).chomp if info_es['my_token'].nil? + info_es['my_ssh_pubkey'] = [] build_my_info.config_my_info(info_es) end -- 2.23.0

1 0

[PATCH v2 lkp-tests] lib/common.rb: build_pkg_error_id function add filtering rule
by Lin Jiaxin 08 Feb '21

08 Feb '21

Some software use tempfile when build. The directory is random, thus every error will be a new error_id. In fact, they are same. Now remove the random part except /tmp/lkp (makepkg argument: BUILDDIR). /tmp/gitstatus-build.BOpUcJeETC/libgit2/src/futils.c:1143:16: warning: comparison of integer expressions of different signedness: 'uint64_t' {aka 'const long unsigned int'} and 'long int' [-Wsign-compare] => libgit2/src/futils.c:warning:comparison-of-integer-expressions-of-different-signedness:'uint64_t'{aka'const-long-unsigned-int'}and'long-int'[-Wsign-compare] Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- lib/common.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/common.rb b/lib/common.rb index cc64eb17..7deee62e 100755 --- a/lib/common.rb +++ b/lib/common.rb @@ -339,5 +339,6 @@ def build_pkg_error_id(line) line = line.tr(" ", "-") line = line.gsub(/[-_.,;:#!\[(]+$/, "") line = line.gsub(/([-_.,;:#!]){3,}/, ":") + line = line.gsub(/^\/tmp\/(?!lkp).*?\//, "") line end -- 2.23.0

1 0

[PATCH lab-z9] add host-info of b5
by Zhang Dewan 08 Feb '21

08 Feb '21

Signed-off-by: Zhang Dewan <513619464(a)qq.com> --- hosts/2288hv5-2s44p-384g--b5 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 hosts/2288hv5-2s44p-384g--b5 diff --git a/hosts/2288hv5-2s44p-384g--b5 b/hosts/2288hv5-2s44p-384g--b5 new file mode 100644 index 0000000..5dbd623 --- /dev/null +++ b/hosts/2288hv5-2s44p-384g--b5 @@ -0,0 +1,14 @@ +memory: 384G +nr_ssd_partitions: 1 +ssd_partitions: + - /dev/disk/by-id/ata-MTFDDAK480TDC-1AT1ZABYY_20032653CBD6 +mac_addr: + - 6c:eb:b6:50:c0:a9 + - 6c:eb:b6:50:c0:aa + - 64:2c:ac:de:ce:51 + - 64:2c:ac:de:ce:d3 +arch: x86_64 +nr_node: 2 +nr_cpu: 88 +model_name: Intel(R) Xeon(R) Gold 6266C CPU @ 3.00GHz +ipmi_ip: 9.3.3.10 -- 2.23.0

1 0

[PATCH lkp-tests] distro/depends/cci-makepkg: add git depend
by Wei Jihui 08 Feb '21

08 Feb '21

user maybe use git cmd when write PKGBUILD Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- distro/depends/cci-makepkg | 1 + 1 file changed, 1 insertion(+) diff --git a/distro/depends/cci-makepkg b/distro/depends/cci-makepkg index 4ed4f19e..855ac8ce 100644 --- a/distro/depends/cci-makepkg +++ b/distro/depends/cci-makepkg @@ -1,3 +1,4 @@ +git curl gnupg gettext -- 2.23.0

1 0

[PATCH lkp-tests] add tests script for openeuler os migration
by Bai Jing 08 Feb '21

08 Feb '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 113 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100755 tests/openeuler_docker diff --git a/tests/openeuler_docker b/tests/openeuler_docker new file mode 100755 index 000000000..69d8a58a6 --- /dev/null +++ b/tests/openeuler_docker @@ -0,0 +1,113 @@ +#!/bin/sh +# - dockerfile_url + +. $LKP_SRC/lib/debug.sh + +[ -n "$dockerfile_url" ] || die "dockerfile_url is empty " + +cd $BENCHMARK_ROOT + +fit_repo() +{ + yum reinstall -y git + git clone "$dockerfile_url" || exit 0 + file_dir=$(find ./ -type f -name Dockerfile | xargs grep -E '^FROM centos:(centos|)[78]') + [ -n "$file_dir" ] || exit 0 +} + +openeuler_ready() +{ + docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar +} + +centos_docker() +{ + flag=0 + for file in $(find ./ -name Dockerfile | xargs grep -E '^FROM centos:(centos|)[78]'| awk -F: '{print $1}') + do + file=${file#*/} + image=$(echo "$file" | awk -F'/' '{print $2}' | tr 'A-Z' 'a-z') + + ( + cd $BENCHMARK_ROOT/${file%/*} + if [ -f "build.sh" ] && [ -x "build.sh" ] + then + ./build.sh > /dev/null 2> /dev/null && handle_dockerfile + else + docker_build > /dev/null 2> /dev/null && handle_dockerfile + fi + ) + done +} + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" Dockerfile + +} + +handle_epel() +{ + grep -qw "epel-release" Dockerfile && { + sed -i "s/epel-release/bash/g" Dockerfile + } +} + +handle_epelrpm() +{ + + grep -qw "https://.*epel-release.*rpm" Dockerfile && { + sed -i "s|https://.*epel-release.*rpm|bash|g" Dockerfile + + } +} + +handle_groupadd() +{ + grep -qw "groupadd*" Dockerfile && { + sed -i '/groupadd*/i\ yum -y install shadow && \\' Dockerfile + } + +} + +handle_dockerfile() +{ + flag=1 + handle_FROM + handle_epel + handle_groupadd + docker_build "-openeuler" + [ $? -ne 0 ] || docker_run "-openeuler" +} + +docker_build() +{ + docker build -t "${image}$1" . + [ $? -ne 0 ] && { + [ "$flag" == "1" ] && echo "${image}$1.build.fail: 1" + return 1 + } + [ "$flag" == "1" ] && echo "${image}$1.build.success: 1" + flag=0 +} + +docker_run() +{ + docker run -d --name "${image}" "${image}$1" + docker_stats=$(docker inspect ${image} | grep Running | awk -F ' ' '{print $2}' | tr -d ',') + [ $docker_stats == false ] && { + echo "${image}.run.fail: 1" + return 1 + } + + echo "${image}.run.success: 1" +} + +docker_test() +{ + fit_repo + openeuler_ready + centos_docker +} + +docker_test -- 2.23.0

2 1