- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci] container/logging-es: remove redundant code
by Liu Yinsi 17 Mar '21

17 Mar '21

[why] these deleted code optimize the bulk write performance, but affect the real-time performance of data. At now, our fluentd architecture already been optimized, the bulk write is no longer a bottleneck, so delete codes. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/logging-es/start | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/container/logging-es/start b/container/logging-es/start index efd89de..9be7a5b 100755 --- a/container/logging-es/start +++ b/container/logging-es/start @@ -19,14 +19,3 @@ cmd=( ) "${cmd[@]}" - -# set index -check_service_ready 9202 -curl -sSH "Content-Type: application/json" -XPUT "localhost:9202/_settings" -d ' -{ - "index": { - "refresh_interval": "120s", - "translog.durability": "async", - "translog.sync_interval": "120s", - "translog.flush_threshold_size": "1024mb"} -}' -- 2.23.0

1 0

[PATCH compass-ci] dracut-initrd: add xz-utils
by Wang Chenglong 17 Mar '21

17 Mar '21

[Why] wcl@crystal ~/compass-ci/container/dracut-initrd% ./run /srv/os/centos/x86_64/6.10/lib/modules/3.10.0-957.1.3.el7.x86_64 dracut: Executing: /usr/bin/dracut -o iscsi --omit-drivers iscsi --force --kver 3.10.0-957.1.3.el7.x86_64 -k /lib/modules/3.10.0-957.1.3.el7.x86_64 /boot/initramfs.lkp-3.10.0-957.1.3.el7.x86_64.img dracut: Kernel modules are compressed with xz, but xz is not available. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 616183f..b8eb6f5 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -6,7 +6,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y --fix-missing --no-install-recommends -qq apt-utils \ -nfs-common netbase cifs-utils kmod rsync dracut dracut-network \ +nfs-common netbase cifs-utils kmod rsync dracut dracut-network xz-utils \ dracut-config-generic apt-get clean -- 2.23.0

1 0

[PATCH v2 lkp-tests] jobs: add secrets_info.yaml
by Wei Jihui 17 Mar '21

17 Mar '21

secrets_info.yaml is for user use secret field Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- jobs/secrets_info.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 jobs/secrets_info.yaml diff --git a/jobs/secrets_info.yaml b/jobs/secrets_info.yaml new file mode 100644 index 00000000..4026b3b7 --- /dev/null +++ b/jobs/secrets_info.yaml @@ -0,0 +1,6 @@ +# This file is for user add secret field, so that it's not visible from outside(job.yaml), but can be used at testbox. +# Usage: submit job.yaml -i secrets_info.yaml +# Use at testbox: testbox has env with: "secrets_" + secret field, such as $secrets_my_token + +secrets: + my_token: {{my_token}} -- 2.23.0

1 0

[PATCH lkp-tests] jobs: add secrets_info.yaml
by Wei Jihui 17 Mar '21

17 Mar '21

secrets_info.yaml is for user use secret field Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- jobs/secrets_info.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 jobs/secrets_info.yaml diff --git a/jobs/secrets_info.yaml b/jobs/secrets_info.yaml new file mode 100644 index 00000000..196cf5ed --- /dev/null +++ b/jobs/secrets_info.yaml @@ -0,0 +1,6 @@ +# This file is for user add secret field, so that it;s not visible from outside, but can be use at testbox. +# Usage: submit job.yaml -i secrets_info.yaml +# Use at testbox: testbox has env with: "secrets_" + secret field, such as $secrets_my_token + +secrets: + my_token: {{my_token}} -- 2.23.0

2 1

[PATCH compass-ci] sbin/es-jobs: add -e to get fail result
by Wu Zhende 17 Mar '21

17 Mar '21

es-jobs -e group_id=xxxx error.result.path=/srv/result/openeuler_docker/xxxx: ["Error: unable to find a match"] Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/es_jobs.rb | 24 +++++++++++++++++++++++- sbin/es-jobs | 7 ++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index b0d0fb2..e4e9530 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -12,13 +12,14 @@ require_relative './es_query' # deal jobs search from es class ESJobs - def initialize(es_query, my_refine = [], fields = [], stats_filter = []) + def initialize(es_query, my_refine = [], fields = [], stats_filter = [], error_result = false) @es_query = es_query @jobs = query_jobs_from_es @refine = my_refine @fields = fields @stats_filter = stats_filter @refine_jobs = [] + @error_result = error_result set_jobs_summary end @@ -98,6 +99,7 @@ class ESJobs 'max.stats' => {}, 'min.stats' => {} } + result['error.result'] = {} if @error_result metrics.each { |metric| result['raw.stats'][metric] = [] } result end @@ -119,7 +121,27 @@ class ESJobs result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 result['raw.id']["[#{keyword}=#{job[keyword]}]"] ||= [] result['raw.id']["[#{keyword}=#{job[keyword]}]"] << job['id'] + + next unless @error_result && job[keyword] == "failed" + + get_failed_result(result, job) + end + end + + def get_failed_result(result, job) + path = "/srv#{job['result_root']}/#{job['suite']}" + res = find_error_lines(path) + result['error.result']["path=#{path}"] = res + end + + def find_error_lines(path) + res = [] + return res unless File.exist?(path) + + File.open(path).each do |line| + res << line.to_s if line.match(/error:/i) end + res end def stats_count(result) diff --git a/sbin/es-jobs b/sbin/es-jobs index ff0d540..3804dc9 100755 --- a/sbin/es-jobs +++ b/sbin/es-jobs @@ -22,6 +22,7 @@ end opt_refine = [-1] opt_fields = [] opt_stats_filter = [] +opt_error_result = false opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: es-jobs [options] search_key1=val1[,val2..] ..' @@ -45,11 +46,15 @@ opt_parser = OptionParser.new do |opts| opts.on('-s fields', '--stats-filter fields', 'return data contains fields in stats') do |fields| opt_stats_filter = fields.split(',') end + + opts.on('-e', '--error.result', 'obtain the error information about job failure') do + opt_error_result = true + end end opt_parser.parse!(ARGV) items = parse_argv raise 'Please enter a query' if items.empty? -es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter) +es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter, opt_error_result) es_jobs.output -- 2.23.0

1 0

[PATCH compass-ci] container/git-mirror: change mount dir
by Li Yuanchao 17 Mar '21

17 Mar '21

As several repos are supported as upstream-repos, to avoid confusion, change the dir name of /c/upstream-repos to /c/git-repos. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/git-mirror/start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/git-mirror/start b/container/git-mirror/start index ec42595..f01789c 100755 --- a/container/git-mirror/start +++ b/container/git-mirror/start @@ -6,7 +6,7 @@ require_relative '../defconfig.rb' DOCKER_CCI = '/c/compass-ci' -DOCKER_REPO = '/c/upstream-repos' +DOCKER_REPO = '/c/git-repos' docker_rm "git-mirror" lkp = %x(id -u lkp) -- 2.23.0

1 0

[PATCH compass-ci] container/web-backend|-nginx: update SRV_HTTP_RESULT_PORT
by Lu Weitao 17 Mar '21

17 Mar '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend-nginx/nginx.conf | 2 +- container/web-backend/start | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/container/web-backend-nginx/nginx.conf b/container/web-backend-nginx/nginx.conf index 381888a..a3313fb 100644 --- a/container/web-backend-nginx/nginx.conf +++ b/container/web-backend-nginx/nginx.conf @@ -29,7 +29,7 @@ http { proxy_set_header X-Forward-For $proxy_add_x_forwarded_for; # srv-http server - proxy_pass http://172.17.0.1:11300; + proxy_pass http://172.17.0.1:20007; } } } diff --git a/container/web-backend/start b/container/web-backend/start index e680255..cebf949 100755 --- a/container/web-backend/start +++ b/container/web-backend/start @@ -11,7 +11,7 @@ docker_rm web-backend load_cci_defaults [[ $SRV_HTTP_RESULT_HOST ]] || SRV_HTTP_RESULT_HOST=$(curl ip.sb) -[[ $SRV_HTTP_RESULT_PORT ]] || SRV_HTTP_RESULT_PORT=11300 +[[ $SRV_HTTP_RESULT_PORT ]] || SRV_HTTP_RESULT_PORT=20007 cmd=( docker run -- 2.23.0

1 0

[PATCH v2 compass-ci] container/web-backend: add API /get_table_error
by Lu Weitao 17 Mar '21

17 Mar '21

[Example] curl -X GET 'localhost:30000/get_job_error?group_id=wcl_ansible-openeuler-03-10' return: { "filter":{"group_id":"wcl_ansible-openeuler-03-10"}, "table_fields":["job_id","error_id","error_message","link to result"], "data_table":[ [ "crystal.1354921", "ansible_test.error.Unable-to-start-service-httpd-Job-for-httpdservice-xxx.fail", "{\"changed\": false, \"msg\": \"Unable to start service httpd: Job for httpd.service failed because a timeout was exceeded.}", "http://172.17.0.1:11300/result/ansible_test/2021-03-10/vm-2p16g--wcl1/opene… ate/crystal.1354921" ], [ "crystal.1354929", ... ] ... ] } Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend/web-backend | 19 +++++++++ src/lib/web_backend.rb | 68 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 74eda85..fa2a53d 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -146,3 +146,22 @@ end get '/get_repo_statistics' do new_refs_statistics(params) end + +# GET /get_job_error?suite=virttest&tbox_group=vm-2p8g +# Response +# { +# "filter":{"group_id":"wcl_ansible-openeuler-03-10"}, +# "table_fields":["job_id","error_id","error_message","link to result"], +# "data_table":[ +# [ +# "crystal.1354921", +# "ansible_test.error.Unable-to-start-service-httpd-Job-for-httpdservice-failed.message", +# "{\"changed\": false, \"msg\": \"Unable to start service httpd: Job for httpd.service failed because a timeout was exceeded.} +# "http://172.17.0.1:11300/result/ansible_test/2021-03-10/vm-2p16g--wcl1/opene…" +# ], +# ... +# ] +# } +get '/get_job_error' do + get_job_error(params) +end diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 89c0f3c..48b7ef2 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -12,6 +12,7 @@ require "#{CCI_SRC}/lib/compare.rb" require "#{CCI_SRC}/lib/constants.rb" require "#{CCI_SRC}/lib/es_query.rb" require "#{CCI_SRC}/lib/matrix2.rb" +require "#{CCI_SRC}/lib/params_group.rb" require "#{CCI_SRC}/lib/compare_data_format.rb" UPSTREAM_REPOS_PATH = ENV['UPSTREAM_REPOS_PATH'] || '/c/upstream-repos' @@ -548,3 +549,70 @@ def new_refs_statistics(params) end [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] end + + +# ------------------------------------------------------------------------------------------- +# job error table like: +# job_id error_id error_message link to result +# ------------------------------------------------------------------------------------- +# crystal.630608 "stderr.xxx" "messag:xxxx" https://$host:$port/$result_root +# ... +# ------------------------------------------------------------------------------------------- + +def get_job_error(params) + begin + body = job_error_body(params) + rescue StandardError => e + warn e.message + return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'get error table error'] + end + + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] +end + +def job_error_body(params) + error_table = get_error_table(params) + { + filter: params, + table_fields: ["job_id", "error_id", "error_message", "link to result"], + data_table: error_table, + }.to_json +end + +def get_error_table(filter_items) + error_table = [] + + job_list = get_job_list(filter_items) + job_list.each do |job| + error_table += get_error_from_job(job) + end + + error_table +end + +def get_job_list(items) + es = ESQuery.new(ES_HOST, ES_PORT) + query_results = es.multi_field_query(items) + + extract_jobs_list(query_results['hits']['hits']) +end + +# get all error_id from one job +def get_error_from_job(job) + job_error = [] + job['stats'].each do |metric, value| + next unless metric.end_with?('.message') + + result_host = ENV['SRV_HTTP_RESULT_HOST'] || SRV_HTTP_RESULT_HOST + result_port = ENV['SRV_HTTP_RESULT_PORT'] || SRV_HTTP_RESULT_PORT + error_id = metric.sub('.message', '.fail') + job_error << [ + job['id'], + error_id, + value, + "http://#{result_host}:#{result_port}#{job['result_root']}" + ] + end + + job_error +end -- 2.23.0

2 2

[PATCH v4 lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 17 Mar '21

17 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 81 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 tests/ansible_test diff --git a/tests/ansible_test b/tests/ansible_test new file mode 100755 index 000000000..bf75b19b7 --- /dev/null +++ b/tests/ansible_test @@ -0,0 +1,81 @@ +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + ansible_host=localhost + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] || { + echo "Ansible failed to install." + exit 1 + } + } +} + +download_ansible_role() +{ + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git + if [ $? -eq 0 ]; then + echo "$benchmark is successfully cloned" + else + echo "Please check whether your ansible_repo is correct!" + exit 1 + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: $ansible_host + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook is running failed!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

1 0

[PATCH lkp-tests] rpmbuild for openeuler
by Li Ping 16 Mar '21

16 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- jobs/rpmbuild-pkg.yaml | 2 +- tests/rpmbuild-pkg | 46 +++++++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/jobs/rpmbuild-pkg.yaml b/jobs/rpmbuild-pkg.yaml index 53e145b3b..a70590afe 100644 --- a/jobs/rpmbuild-pkg.yaml +++ b/jobs/rpmbuild-pkg.yaml @@ -1,5 +1,5 @@ suite: rpmbuild-pkg category: functional -rpm_repo: +upstream_repo: rpmbuild-pkg: diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index b641e97d1..034e4f3e1 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -1,12 +1,16 @@ #!/bin/bash -# - rpm_repo +# - upstream_repo +# - compatible_os . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh -[ -n "$rpm_repo" ] || die "rpm_repo is empty" -package_name=${rpm_repo##*/} -sync_dest="/initrd/rpmbuild-pkg/${os}/${os_arch}/${os_version}/${package_name}" +: "${compatible_os:=compat-openeuler}" + +[ -n "$upstream_repo" ] || die "upstream_repo is empty" +package_name=${upstream_repo##*/} +rpm_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compatible_os}/${os_arch}/Packages" +source_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compatible_os}/source/Packages" init_workspace() { @@ -14,21 +18,16 @@ init_workspace() rpmdev-setuptree } -get_pkgfile() +download_upstream_repo() { - curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-show", "HEAD:'$1'"]}' -o "${2}" -} - -download_rpm_repo() -{ - local filelist=$(curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-ls-files", "."]}') - for pkgfile in "${filelist[@]}" + git clone "git://$GIT_SERVER/openeuler/${upstream_repo}" || die "clone git repo ${package_name} failed: git://$GIT_SERVER/openeuler/${upstream_repo}" + cd $package_name || exit + filelist=$(git ls-files) + for pkgfile in ${filelist[@]} do local dir="SOURCES" echo $pkgfile | egrep "\.spec$" && dir="SPECS" - get_pkgfile "$pkgfile" "${HOME}/rpmbuild/${dir}/$pkgfile" + mv "$pkgfile" "${HOME}/rpmbuild/${dir}/" done } @@ -50,19 +49,28 @@ build_rpm() show_rpm_files() { find ${HOME}/rpmbuild/RPMS -type f -name "*.rpm" +} + +show_src_rpm_files() +{ find ${HOME}/rpmbuild/SRPMS -type f -name "*.rpm" } upload_rpm_pkg() { - local file - for file in $(show_rpm_files) + local rpm_file + for rpm_file in $(show_rpm_files) + do + upload_one_curl ${rpm_file} ${rpm_dest} + done + local src_rpm_file + for src_rpm_file in $(show_src_rpm_files) do - upload_one_curl ${file} ${sync_dest} + upload_one_curl ${src_rpm_file} ${source_dest} done } init_workspace -download_rpm_repo +download_upstream_repo build_rpm upload_rpm_pkg -- 2.23.0

1 0