April 2021 - Compass-ci - mailweb.openeuler.org

[PATCH v2 lkp-tests] tests/rpmbuild-pkg: post data to rpm-repo docker
by Wang Yong 13 Apr '21

13 Apr '21

post rpm data to rpm-repo in order to createrepo ref: commit 5dd22c1e010734bc5e610f4abe46baec0f36f5bb Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/rpmbuild-pkg | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index a91ae4a08..312582d16 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -108,6 +108,32 @@ upload_rpm_pkg() done } +handle_rpm_list() +{ + rpm_list=($(show_rpm_files | xargs basename -a)) + srpm_list=($(show_src_rpm_files | xargs basename -a)) + full_list=(${rpm_list[@]} ${srpm_list[@]}) + j=0 + + for rpm_name in ${full_list[@]} + do + full_list[$j]="\"/srv${rpm_dest}/${rpm_name}\"" + [[ ${rpm_name} =~ "src.rpm" ]] && full_list[$j]="\"/srv${src_rpm_dest}/${rpm_name}\"" + ((j++)) + done + + full_list=$(echo ${full_list[@]} | sed 's; ;,;g') +} + +update_repo_mq() +{ + handle_rpm_list + + curl -H 'Content-Type: Application/json' -XPOST "${LKP_SERVER}:10003/upload" \ + -d "{\"upload_rpms\": ["${full_list}"]}" +} + [ -n "$repo_name" ] && from_srpm || from_git build_rpm upload_rpm_pkg +update_repo_mq -- 2.23.0

1 0

[PATCH compass-ci] container/scheduler: fix Missing ENV key: NODE_PORT
by Cao Xueliang 13 Apr '21

13 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/scheduler/my-start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/scheduler/my-start b/container/scheduler/my-start index 8d800f9..e5855af 100755 --- a/container/scheduler/my-start +++ b/container/scheduler/my-start @@ -65,6 +65,7 @@ cmd = %W[ -e CCI_REPOS=#{CCI_REPOS} -e LKP_SRC=#{DEFAULT_LKP} -e CCI_SRC=#{DEFAULT_CCI} + -e NODE_PORT=#{SCHED_PORT} -v #{ENV['LKP_SRC']}:#{DEFAULT_LKP} -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} -v #{CCI_REPOS}/lab-#{LAB}:#{CCI_REPOS}/lab-#{LAB} -- 2.23.0

1 0

[PATCH compass-ci] scheduler: use NODE_PORT to run kemal
by Cao Xueliang 13 Apr '21

13 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/scheduler/start | 5 +++-- src/scheduler.cr | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/container/scheduler/start b/container/scheduler/start index b13b396..598b263 100755 --- a/container/scheduler/start +++ b/container/scheduler/start @@ -7,6 +7,7 @@ require 'set' require_relative '../defconfig.rb' names = Set.new %w[ + SCHED_HOST SCHED_PORT SCHED_NODES REDIS_HOST @@ -38,7 +39,7 @@ defaults['SCHED_PORT'] ||= '3000' defaults['MASTER_FLUENTD_PORT'] ||= '24224' defaults['SRV_HTTP_CCI_PORT'] ||= '11307' sched_nodes = defaults.delete('SCHED_NODES') -SCHED_PORT = defaults.delete('SCHED_PORT') +SCHED_PORT = defaults['SCHED_PORT'] sched_nodes.each do |node| docker_rm "s001-alpine-#{node['port']}" @@ -69,7 +70,7 @@ sched_nodes.each do |node| -e CCI_REPOS=#{CCI_REPOS} -e LKP_SRC=#{DEFAULT_LKP} -e CCI_SRC=#{DEFAULT_CCI} - -e SCHED_PORT=#{node_port} + -e NODE_PORT=#{node_port} -v /srv/tmp/public:/c/cci/scheduler/public -v #{ENV['LKP_SRC']}:#{DEFAULT_LKP} -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} diff --git a/src/scheduler.cr b/src/scheduler.cr index 07b38a3..4e4bf24 100644 --- a/src/scheduler.cr +++ b/src/scheduler.cr @@ -9,7 +9,7 @@ module Scheduler log = JSONLogger.new begin - Kemal.run(SCHED_PORT) + Kemal.run(ENV["NODE_PORT"].to_i32) rescue e log.error(e) end -- 2.23.0

2 2

[PATCH compass-ci 2/2] src/extract-stats/stats_worker.cr: optimize output
by Lu Weitao 13 Apr '21

13 Apr '21

[Why] standardized output log for kibana Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- src/extract-stats/stats_worker.cr | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index e371bf3..47c3a59 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -2,6 +2,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. require "../lib/etcd_client" +require "../lib/json_logger" require "../scheduler/elasticsearch_client" require "../scheduler/redis_client" require "../scheduler/constants" @@ -16,6 +17,7 @@ class StatsWorker @es = Elasticsearch::Client.new @etcd = EtcdClient.new @rc = RegressionClient.new + @log = JSONLogger.new end def handle(queue_path, channel) @@ -30,7 +32,7 @@ class StatsWorker @etcd.delete(queue_path) rescue e channel.send(queue_path) - STDERR.puts e.message + @log.error(e.message) # incase of many error message when task-queue, ES does not work sleep(10) ensure @@ -95,16 +97,16 @@ class StatsWorker new_error_ids = check_new_error_ids(error_ids, job_id) unless new_error_ids.empty? sample_error_id = new_error_ids.sample - STDOUT.puts "send a delimiter task: job_id is #{job_id}" + @log.info("send a delimiter task: job_id is #{job_id}") queue = "#{DELIMITER_TASK_QUEUE}/#{job_id}" value = {"job_id" => job_id, "error_id" => sample_error_id} @etcd.put(queue, value) msg = %({"job_id": "#{job_id}", "new_error_id": "#{sample_error_id}"}) - system "echo '#{msg}'" + @log.info(msg) end msg = %({"job_id": "#{job_id}", "job_state": "extract_finished"}) - system "echo '#{msg}'" + @log.info(msg) end def check_new_error_ids(error_ids : Array, job_id : String) @@ -113,7 +115,7 @@ class StatsWorker begin is_exists = @rc.check_error_id error_id rescue e - STDERR.puts e.message + @log.error(e.message) next end next if is_exists -- 2.23.0

1 0

[PATCH compass-ci 1/2] lib/dump_stat.rb: optimize useless special chars
by Lu Weitao 13 Apr '21

13 Apr '21

[Why] we already use strip, so needn't chomp Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/dump_stat.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dump_stat.rb b/lib/dump_stat.rb index 383ff09..04a7632 100644 --- a/lib/dump_stat.rb +++ b/lib/dump_stat.rb @@ -38,7 +38,7 @@ module DumpStat stat_result.each do |key, value| key = key.resolve_invalid_bytes - key = key.chomp.strip + key = key.strip next if key[0] == '#' next if value.empty? || value == 0 next if monitor =~ /^(dmesg|kmsg)$/ && key =~ /^(message|pattern):/ -- 2.23.0

1 0

[PATCH lkp-tests] sbin/submit: delete useless require
by Wei Jihui 13 Apr '21

13 Apr '21

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- sbin/submit | 1 - 1 file changed, 1 deletion(-) diff --git a/sbin/submit b/sbin/submit index d2d9bec7..edb207cc 100755 --- a/sbin/submit +++ b/sbin/submit @@ -5,7 +5,6 @@ LKP_SRC = ENV['LKP_SRC'] || File.dirname(File.dirname(File.realpath($PROGRAM_NAME))) require "#{LKP_SRC}/lib/job2sh" -require "#{LKP_SRC}/lib/distro_info" require "#{LKP_SRC}/lib/scheduler_client" require 'optparse' require 'yaml' -- 2.23.0

1 0

[PATCH lkp-tests] tests/rpmbuild-pkg: post data to rpm-repo docker
by Wang Yong 13 Apr '21

13 Apr '21

post rpm data to rpm-repo in order to createrepo ref: commit 5dd22c1e010734bc5e610f4abe46baec0f36f5bb Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/rpmbuild-pkg | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index a91ae4a08..3d63bebae 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -108,6 +108,33 @@ upload_rpm_pkg() done } +update_repo_mq() +{ + rpm_list=($(find ${HOME}/rpmbuild/RPMS -type f -name "*.rpm" -exec basename {} \;)) + srpm_list=($(find ${HOME}/rpmbuild/SRPMS -type f -name "*.rpm" -exec basename {} \;)) + j=0 + k=0 + + for rpm_name in ${rpm_list[@]} + do + rpm_list[$j]="\"/srv${rpm_dest}/${rpm_name}\"" + ((j++)) + done + + for src_rpm_name in ${srpm_list[@]} + do + srpm_list[$k]="\"/srv${src_rpm_dest}/${src_rpm_name}\"" + ((k++)) + done + + full_list=(${rpm_list[@]} ${srpm_list[@]}) + full_list=$(echo ${full_list[@]} | sed 's; ;,;g') + + curl -H 'Content-Type: Application/json' -XPOST "${LKP_SERVER}:10003/upload" \ + -d "{\"upload_rpms\": ["${full_list}"]}" +} + [ -n "$repo_name" ] && from_srpm || from_git build_rpm upload_rpm_pkg +update_repo_mq -- 2.23.0

2 2

[PATCH compass-ci 2/2] sbin/download-rootfs: not to download if machine is virtual
by Liu Yinsi 12 Apr '21

12 Apr '21

[why] we not support to run qemu in a virtual machine, so download os rootfs is unnecessary. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/download-rootfs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sbin/download-rootfs b/sbin/download-rootfs index f8c245d..68a6803 100755 --- a/sbin/download-rootfs +++ b/sbin/download-rootfs @@ -37,4 +37,9 @@ download_os_rootfs() rm -rf ${os_version}.cgz } -download_os_rootfs +is_virtual() +{ + dmidecode -s system-product-name | grep -iq "virtual" +} + +is_virtual || download_os_rootfs -- 2.23.0

1 0

[PATCH compass-ci 1/2] sbin/download-rootfs: download os rootfs according to arch
by Liu Yinsi 12 Apr '21

12 Apr '21

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/download-rootfs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/download-rootfs b/sbin/download-rootfs index 74afbf6..f8c245d 100755 --- a/sbin/download-rootfs +++ b/sbin/download-rootfs @@ -10,7 +10,7 @@ # ./download-rootfs debian aarch64 sid os=${1:-openeuler} -os_arch=${2:-aarch64} +os_arch=${2:-$(arch)} os_version=${3:-20.03} . ${CCI_SRC}/container/defconfig.sh -- 2.23.0

1 0

[PATCH lkp-tests] add script for fix dockerfile file
by Bai Jing 12 Apr '21

12 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- lib/handle_dockerfile.sh | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 lib/handle_dockerfile.sh diff --git a/lib/handle_dockerfile.sh b/lib/handle_dockerfile.sh new file mode 100644 index 000000000..b71f9865d --- /dev/null +++ b/lib/handle_dockerfile.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" +} + +handle_epel() +{ + sed -i "s/epel-release/bash/g" "$dockerfile_name" +} + +handle_epelrpm() +{ + sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" +} + +handle_rpm_gpg() +{ + sed -i "s|RPM-GPG-KEY-CentOS-7|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" + sed -i "s|RPM-GPG-KEY-centosofficial|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" +} + +add_base_commands() +{ + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" +} + +handle_epel_repo() +{ + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" +} + +handle_error_exit() +{ + sed '/^RUN/RUN set -e;/g' "$dockerfile_name" +} + +handle_dockerfile() +{ + dockerfile_name=$1 + handle_FROM + handle_epel + handle_epelrpm + handle_rpm_gpg + add_base_commands + handle_epel_repo + handle_error_exit +} -- 2.23.0

2 3