April 2021 - Compass-ci - mailweb.openeuler.org

[PATCH compass-ci 2/3] sbin/mail-job: mail compare result to author and committer
by Lu Weitao 24 Apr '21

24 Apr '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/mail_job_result.rb | 55 +++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/lib/mail_job_result.rb b/lib/mail_job_result.rb index e5884705..752b8dcf 100644 --- a/lib/mail_job_result.rb +++ b/lib/mail_job_result.rb @@ -4,6 +4,7 @@ require_relative 'mail_client.rb' require_relative 'es_query.rb' +require_relative 'compare.rb' require_relative 'constants.rb' require 'json' @@ -16,27 +17,44 @@ class MailJobResult end def send_mail - json = compose_mail.to_json - MailClient.new.send_mail(json) + data = compose_mail + return nil unless data + MailClient.new.send_mail_encode(data) end def compose_mail set_submitter_info - subject = "[Compass-CI] job: #{@job_id} result" + context = get_compare_result(@job) + return nil unless context + + subject = "[Compass-CI] job: #{@job_id} compare result" signature = "Regards\nCompass-CI\nhttps://gitee.com/openeuler/compass-ci" - body = "Hi, - Thanks for your participation in Kunpeng and software ecosystem! + + data = <<~BODY + To: #{@email_to} + Bcc: #{@email_cc} + Subject: #{subject} + + Hi, + + html_part: Thanks for your participation in Kunpeng and software ecosystem! Your Job: #{@job_id} had finished. - Please check job result: http://#{@result_host}:#{@result_port}#{@result_root}\n\n#{signature}" - { 'to' => @submitter_email, 'body' => body, 'subject' => subject } + Bellow are comparsion result of the base_commit: + #{context.to_s} + \n\n#{signature}" + BODY + + data end def set_submitter_info - job = query_job - exit unless job && job['email'] + sleep 10 + @job = query_job + exit unless @job && @job['author_email'] - @submitter_email = job['email'] - @result_root = job['result_root'] + @email_to = @job['author_email'] + @email_cc = @job['committer_email'] + @result_root = @job['result_root'] end def query_job @@ -50,3 +68,18 @@ class MailJobResult query_result['hits']['hits'][0]['_source'] end end + +def get_compare_result(job) + min_samples = job['nr_num'].to_i + base_commit = job['base_commit'] + commit_id = job['commit_id'] + return nil unless base_commit && commit_id + + condition_list = [{'base_commit' => base_commit}, {'commit_id' => commit_id}] + options = { :min_samples => min_samples, :no_print => true} + + matrices_list, suite_list = create_matrices_list(condition_list, options[:min_samples]) + return nil if matrices_list.size < 2 + + compare_matrixes(matrices_list, suite_list, options: options) +end -- 2.23.0

1 0

[PATCH compass-ci 1/3] sbin/compare: support min_samples mode
by Lu Weitao 24 Apr '21

24 Apr '21

[Example] sbin/compare id=crystal.1686878,crystal.1686879 id=crystal.1686880,crystal.1686881 --min_samples 3 Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/compare.rb | 22 +++++++++++++++++++--- lib/matrix2.rb | 4 +++- sbin/compare | 5 +++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/lib/compare.rb b/lib/compare.rb index 7386caea..a6fd9f98 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -20,7 +20,13 @@ require 'yaml' def compare_matrices_list(argv, common_conditions, options) condition_list = parse_argv(argv, common_conditions) - matrices_list, suite_list = create_matrices_list(condition_list) + matrices_list, suite_list = create_matrices_list(condition_list, options[:min_samples]) + if matrices_list.size < 2 + return nil if options[:no_print] + + warn "matrix less than min_samples" + end + compare_matrixes(matrices_list, suite_list, options: options) end @@ -35,13 +41,15 @@ def parse_argv(argv, common_conditions) conditions end -def create_matrices_list(conditions) +def create_matrices_list(conditions, min_samples) matrices_list = [] suite_list = [] es = ESQuery.new conditions.each do |condition| query_results = es.multi_field_query(condition, desc_keyword: 'start_time') - matrix, suites = combine_query_data(query_results) + matrix, suites = combine_query_data(query_results, min_samples) + next unless matrix + matrices_list << matrix suite_list.concat(suites) end @@ -49,6 +57,14 @@ def create_matrices_list(conditions) return matrices_list, suite_list end +def matrices_verification(matrices, options) + if matrices.size < 2 + return nil if options[:no_print] + + warn "matrix less than min_samples" + end +end + # ------------------------------------------------------------------------------------------- # compare_group # - one condition only diff --git a/lib/matrix2.rb b/lib/matrix2.rb index 12245a80..2fd63243 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -101,8 +101,10 @@ end # input: query results from es_query # return: matrix -def combine_query_data(query_data) +def combine_query_data(query_data, min_sample) job_list = extract_jobs_list(query_data['hits']['hits']) + return nil if job_list.size < min_sample + create_matrix(job_list) end diff --git a/sbin/compare b/sbin/compare index 2ee32688..bcb66ccf 100755 --- a/sbin/compare +++ b/sbin/compare @@ -22,6 +22,7 @@ is_group = false dimensions = nil colorful = nil options = {} +options[:min_samples] = 1 template = nil opt_parser = OptionParser.new do |opts| @@ -48,6 +49,10 @@ opt_parser = OptionParser.new do |opts| colorful = theme end + opts.on('--min_samples min_samples', 'give a number to litmit the min_samples of matrix which used to compare') do |min_samples| + options[:min_samples] = min_samples.to_i + end + opts.on('-t', '--template template', 'compare with user-defined template') do |t| template = t end -- 2.23.0

1 0

[PATCH lkp-tests] test: optimize mysql start command
by Luan Shengde 23 Apr '21

23 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/mysql-8-0-20 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mysql-8-0-20 b/tests/mysql-8-0-20 index 0c60ed9dd..990a053d7 100755 --- a/tests/mysql-8-0-20 +++ b/tests/mysql-8-0-20 @@ -54,8 +54,8 @@ create_database() start_mysql() { - "$mysqld" --defaults-file=/etc/my.cnf --port="$mysql_port" - sleep 20 + "$mysql_server" start & + wait } main() -- 2.23.0

1 0

[PATCH v3 compass-ci] sbin/download-rootfs: download osimage
by Liu Yinsi 23 Apr '21

23 Apr '21

the default os_mount=initramfs, so download osimage to run qemu kvm. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/download-rootfs | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/sbin/download-rootfs b/sbin/download-rootfs index 68a68039..cea416c0 100755 --- a/sbin/download-rootfs +++ b/sbin/download-rootfs @@ -19,22 +19,35 @@ os_version=${3:-20.03} load_cci_defaults rootfs_dir=/srv/os/${os}/${os_arch}/${os_version} +osimage_dir=/srv/initrd/osimage/${os}/${os_arch}/${os_version} -download_os_rootfs() -{ - mkdir -p $rootfs_dir - cd $rootfs_dir +mkdir -p $rootfs_dir $osimage_dir - download_url="http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" - wget $download_url +wget_download() +{ + local download_url=$1 + wget $download_url + + [ "$?" -ne 0 ] && { + log_error "failed to download os rootfs: $download_url" + exit 1 + } +} - [ "$?" -ne 0 ] && { - log_error "failed to download os rootfs: $download_url" - exit 1 - } +download_os_rootfs() +{ + cd $rootfs_dir + wget_download "http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" + + gzip -dc ${os_version}.cgz | cpio -id + rm -rf ${os_version}.cgz +} - gzip -dc ${os_version}.cgz | cpio -id - rm -rf ${os_version}.cgz +download_osimage() +{ + cd $osimage_dir + wget_download "http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/initrd/osimage/…" + wget_download "http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/initrd/deps/nfs…" } is_virtual() @@ -42,4 +55,4 @@ is_virtual() dmidecode -s system-product-name | grep -iq "virtual" } -is_virtual || download_os_rootfs +is_virtual || ( download_os_rootfs && download_osimage ) -- 2.23.0

1 0

[PATCH v3 compass-ci] container/scheduler-http: fix unfriendly error
by Liu Yinsi 23 Apr '21

23 Apr '21

before: /c/compass-ci/container/scheduler-https# ./start Traceback (most recent call last): ./start:14:in `<main>': CA certificate is missing. Ignore if the HTTPS scheduler is not required (RuntimeError) after: /c/compass-ci/container/scheduler-https# ./start CA certificate is missing. Ignore if the HTTPS scheduler is not required Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/scheduler-https/start | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/container/scheduler-https/start b/container/scheduler-https/start index 4ec84a28..5e79d2be 100755 --- a/container/scheduler-https/start +++ b/container/scheduler-https/start @@ -10,8 +10,10 @@ ca_path = '/etc/compass-ci/ca' crt_path = "#{ca_path}/server.crt" key_path = "#{ca_path}/server.key" -warn_msg = 'CA certificate is missing. Ignore if the HTTPS scheduler is not required' -raise warn_msg unless File.exist?(crt_path) && File.exist?(key_path) +unless File.exist?(crt_path) && File.exist?(key_path) + puts 'CA certificate is missing. Ignore if the HTTPS scheduler is not required' + exit +end names = Set.new %w[ SCHED_HOST -- 2.23.0

1 0

[PATCH v5 lkp-tests] tests/qcow2rootfs-x86_64: fix failed to build container using absolute path
by Wang Chenglong 23 Apr '21

23 Apr '21

[Error] root@2288hv5-2s44p-384g--b6 /c/lkp-tests/tests# /c/compass-ci/container/qcow2rootfs/build /c/compass-ci/container/qcow2rootfs/build: line 5: ../defconfig.sh: No such file or directory /c/compass-ci/container/qcow2rootfs/build: line 7: docker_skip_rebuild: command not found Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/qcow2rootfs-x86_64 | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/qcow2rootfs-x86_64 b/tests/qcow2rootfs-x86_64 index f698fb7ec..a783dc1a6 100755 --- a/tests/qcow2rootfs-x86_64 +++ b/tests/qcow2rootfs-x86_64 @@ -9,12 +9,17 @@ set_qcow2rootfs_tools() { export QCOW2_PATH="/c/compass-ci/container/qcow2rootfs" - git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci || return 1 + export CCI_SRC="/c/compass-ci" + git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci rm ${QCOW2_PATH}/root/etc/yum.repos.d/* wget -O ${QCOW2_PATH}/root/etc/yum.repos.d/CentOS-Base.repo https://repo.huaweicloud.com/repository/conf/CentOS-7-reg.repo - ${QCOW2_PATH}/build + cd ${QCOW2_PATH} + ./build || { + echo "build container failed~" + exit 1 + } sed -i 's#$EXTRACT_ROOT/bin/extract $QCOW2_FILE $tar_out_file#$EXTRACT_ROOT/bin/extract $EXTRACT_ROOT/qcow2-dir/$QCOW2_NAME $tar_out_file#' \ - ${QCOW2_PATH}/bin/mail + ${QCOW2_PATH}/bin/main } @@ -52,7 +57,13 @@ extract_rootfs() export ROOTFS_DIR="${qcow2_version}-$(date +%Y-%m-%d-%H-%M-%S)" export ROOTFS_PATH="${MOUNT_POINT}/openeuler/${qcow2_arch}/automatic-build/${ROOTFS_DIR}" mkdir -p $ROOTFS_PATH - ${QCOW2_PATH}/run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} + cd $CCI_SRC/container/dracut-initrd + ./build || { + echo "build container failed~" + exit 1 + } + cd ${QCOW2_PATH} + ./run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} } set_resolv_config() -- 2.23.0

1 0

[PATCH v4 lkp-tests] tests/qcow2rootfs-x86_64: fix failed to build container using absolute path
by Wang Chenglong 23 Apr '21

23 Apr '21

[Error] root@2288hv5-2s44p-384g--b6 /c/lkp-tests/tests# /c/compass-ci/container/qcow2rootfs/build /c/compass-ci/container/qcow2rootfs/build: line 5: ../defconfig.sh: No such file or directory /c/compass-ci/container/qcow2rootfs/build: line 7: docker_skip_rebuild: command not found Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/qcow2rootfs-x86_64 | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/qcow2rootfs-x86_64 b/tests/qcow2rootfs-x86_64 index f698fb7ec..792a80fb9 100755 --- a/tests/qcow2rootfs-x86_64 +++ b/tests/qcow2rootfs-x86_64 @@ -9,12 +9,17 @@ set_qcow2rootfs_tools() { export QCOW2_PATH="/c/compass-ci/container/qcow2rootfs" - git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci || return 1 + export CCI_SRC="/c/compass-ci" + git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci rm ${QCOW2_PATH}/root/etc/yum.repos.d/* wget -O ${QCOW2_PATH}/root/etc/yum.repos.d/CentOS-Base.repo https://repo.huaweicloud.com/repository/conf/CentOS-7-reg.repo - ${QCOW2_PATH}/build + cd ${QCOW2_PATH} + ./build || { + echo "build container failed~" + exit 1 + } sed -i 's#$EXTRACT_ROOT/bin/extract $QCOW2_FILE $tar_out_file#$EXTRACT_ROOT/bin/extract $EXTRACT_ROOT/qcow2-dir/$QCOW2_NAME $tar_out_file#' \ - ${QCOW2_PATH}/bin/mail + ${QCOW2_PATH}/bin/main } @@ -51,15 +56,20 @@ extract_rootfs() { export ROOTFS_DIR="${qcow2_version}-$(date +%Y-%m-%d-%H-%M-%S)" export ROOTFS_PATH="${MOUNT_POINT}/openeuler/${qcow2_arch}/automatic-build/${ROOTFS_DIR}" + cd $CCI_SRC/container/dracut-initrd + ./build || { + echo "build container failed~" + exit 1 + } mkdir -p $ROOTFS_PATH - ${QCOW2_PATH}/run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} + ${QCOW2_PATH}/run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} } set_resolv_config() { - cat >${ROOTFS_PATH}/etc/resolv.conf<<-EOF - nameserver 114.114.114.114 - EOF + cat >${ROOTFS_PATH}/etc/resolv.conf<<-EOF + nameserver 114.114.114.114 + EOF } set_link() @@ -71,6 +81,7 @@ set_link() echo $ROOTFS_PATH } + set_qcow2rootfs_tools check_url download_qcow2_img -- 2.23.0

1 1

[PATCH compass-ci] sbin: configuring ES authentication information
by Wu Zhende 23 Apr '21

23 Apr '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- sbin/es-accounts-mapping.sh | 8 ++++++-- sbin/es-jobs-mapping.sh | 10 +++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/sbin/es-accounts-mapping.sh b/sbin/es-accounts-mapping.sh index c4ac2fc..580b459 100755 --- a/sbin/es-accounts-mapping.sh +++ b/sbin/es-accounts-mapping.sh @@ -2,15 +2,19 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +. $CCI_SRC/container/defconfig.sh + +load_service_authentication + # check whether accounts index has created -status_code=$(curl -sSIL -w "%{http_code}\\n" -o /dev/null http://localhost:9200/accounts) +status_code=$(curl -sSIL -u "${ES_USER}:${ES_PASSWORD}" -w "%{http_code}\\n" -o /dev/null http://localhost:9200/accounts) if [ "$status_code" -eq 200 ] then echo "accounts index has been created, exit." else echo "begin create index." - curl -sSH 'Content-Type: Application/json' -XPUT 'http://localhost:9200/accounts' -d '{ + curl -sSH 'Content-Type: Application/json' -XPUT 'http://localhost:9200/accounts' -u "${ES_USER}:${ES_PASSWORD}" -d '{ "mappings": { "dynamic": false, "properties": { diff --git a/sbin/es-jobs-mapping.sh b/sbin/es-jobs-mapping.sh index 67b1ed1..15000d9 100755 --- a/sbin/es-jobs-mapping.sh +++ b/sbin/es-jobs-mapping.sh @@ -2,6 +2,10 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +. $CCI_SRC/container/defconfig.sh + +load_service_authentication + # Determine whether curl is installed. If not, install curl. if ! [ -x "$(command -v curl)" ] then @@ -28,14 +32,14 @@ then fi # Determine whether jobs index has created -status_code=$(curl -sSIL -w "%{http_code}\n" -o /dev/null http://localhost:9200/jobs) +status_code=$(curl -sSIL -u "${ES_USER}:${ES_PASSWORD}" -w "%{http_code}\n" -o /dev/null http://localhost:9200/jobs) if [ $status_code -eq 200 ] then echo "jobs index has create, exit." else echo "jobs index not exists, begin create index." - curl -sSH 'Content-Type: Application/json' -XPUT 'http://localhost:9200/jobs' -d '{ + curl -sSH 'Content-Type: Application/json' -XPUT 'http://localhost:9200/jobs' -u "${ES_USER}:${ES_PASSWORD}" -d '{ "mappings": { "dynamic": false, "dynamic_templates": [ @@ -185,7 +189,7 @@ else echo "create jobs index failed." else echo "set index.mapping.total_fields.limit: 10000" - curl -sS -XPUT 127.0.0.1:9200/jobs/_settings -H 'Content-Type: application/json' \ + curl -sS -XPUT 127.0.0.1:9200/jobs/_settings -u "${ES_USER}:${ES_PASSWORD}" -H 'Content-Type: application/json' \ -d '{"index.mapping.total_fields.limit": 10000}' fi fi -- 2.23.0

1 0

[PATCH lkp-tests 3/3] tests/qcow2rootfs-x86_64: fix failed to build container using absolute path
by Wang Chenglong 23 Apr '21

23 Apr '21

[Error] root@2288hv5-2s44p-384g--b6 /c/lkp-tests/tests# /c/compass-ci/container/qcow2rootfs/build /c/compass-ci/container/qcow2rootfs/build: line 5: ../defconfig.sh: No such file or directory /c/compass-ci/container/qcow2rootfs/build: line 7: docker_skip_rebuild: command not found Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/qcow2rootfs-x86_64 | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/qcow2rootfs-x86_64 b/tests/qcow2rootfs-x86_64 index f698fb7ec..6826e39ef 100755 --- a/tests/qcow2rootfs-x86_64 +++ b/tests/qcow2rootfs-x86_64 @@ -9,12 +9,17 @@ set_qcow2rootfs_tools() { export QCOW2_PATH="/c/compass-ci/container/qcow2rootfs" + export CCI_SRC="/c/compass-ci" git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci || return 1 rm ${QCOW2_PATH}/root/etc/yum.repos.d/* wget -O ${QCOW2_PATH}/root/etc/yum.repos.d/CentOS-Base.repo https://repo.huaweicloud.com/repository/conf/CentOS-7-reg.repo - ${QCOW2_PATH}/build + cd ${QCOW2_PATH} + ./build || { + echo "build container failed~" + exit 1 + } sed -i 's#$EXTRACT_ROOT/bin/extract $QCOW2_FILE $tar_out_file#$EXTRACT_ROOT/bin/extract $EXTRACT_ROOT/qcow2-dir/$QCOW2_NAME $tar_out_file#' \ - ${QCOW2_PATH}/bin/mail + ${QCOW2_PATH}/bin/main } @@ -51,15 +56,20 @@ extract_rootfs() { export ROOTFS_DIR="${qcow2_version}-$(date +%Y-%m-%d-%H-%M-%S)" export ROOTFS_PATH="${MOUNT_POINT}/openeuler/${qcow2_arch}/automatic-build/${ROOTFS_DIR}" + cd $CCI_SRC/container/dracut-initrd + ./build || { + echo "build container failed~" + exit 1 + } mkdir -p $ROOTFS_PATH - ${QCOW2_PATH}/run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} + ${QCOW2_PATH}/run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} || exit 1 } set_resolv_config() { - cat >${ROOTFS_PATH}/etc/resolv.conf<<-EOF - nameserver 114.114.114.114 - EOF + cat >${ROOTFS_PATH}/etc/resolv.conf<<-EOF + nameserver 114.114.114.114 + EOF } set_link() @@ -71,6 +81,7 @@ set_link() echo $ROOTFS_PATH } + set_qcow2rootfs_tools check_url download_qcow2_img -- 2.23.0

2 2

[PATCH lkp-tests 1/3] install-iso-pre: add depend pkg: wget
by Yu Chuan 23 Apr '21

23 Apr '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- distro/depends/install-iso-pre | 1 + 1 file changed, 1 insertion(+) diff --git a/distro/depends/install-iso-pre b/distro/depends/install-iso-pre index 4569e1e3cdc0..7a66195e44fd 100644 --- a/distro/depends/install-iso-pre +++ b/distro/depends/install-iso-pre @@ -1,3 +1,4 @@ lvm2 ruby git +wget -- 2.23.0

1 0