- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci] lib/job.cr: temporarily cancel account verification
by Wu Zhende 22 Dec '20

22 Dec '20

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/job.cr | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index e6f7e6a..2554db1 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -347,13 +347,14 @@ class Job error_msg = "Failed to verify the account.\n" error_msg += "Please refer to https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/apply-acco…" account_info = @es.get_account(self["my_email"]) - raise account_info unless account_info.is_a?(JSON::Any) + # raise account_info unless account_info.is_a?(JSON::Any) - @account_info = account_info.as_h + # @account_info = account_info.as_h + @account_info = account_info.as_h if account_info.is_a?(JSON::Any) - raise error_msg if @account_info["found"]? == false - raise error_msg unless self["my_name"] == @account_info["my_name"].to_s - raise error_msg unless self["my_token"] == @account_info["my_token"] + # raise error_msg if @account_info["found"]? == false + # raise error_msg unless self["my_name"] == @account_info["my_name"].to_s + # raise error_msg unless self["my_token"] == @account_info["my_token"] @hash.delete("my_uuid") @hash.delete("my_token") -- 2.23.0

1 0

[PATCH lkp-tests] adaptation/archlinux: add archlinux mapping
by Wang Yong 22 Dec '20

22 Dec '20

add mapping in order to run cci-depends job for build-pkg Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- distro/adaptation/archlinux | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/distro/adaptation/archlinux b/distro/adaptation/archlinux index 89163cf41..c52212e08 100644 --- a/distro/adaptation/archlinux +++ b/distro/adaptation/archlinux @@ -72,3 +72,11 @@ libtirpc: libtirpc1 initramfs-tools: openssh-server: openssh openssh-client: openssh +autopoint: gettext +g++: gcc +libarchive13: libarchive +libbz2-1.0: bzip2 +libssl: openssl +libtool-bin: libtool +linux-perf: perf +ruby-dev: ruby -- 2.23.0

1 0

[PATCH v2 lkp-tests] tests: fix return value
by Zhang Yu 22 Dec '20

22 Dec '20

[why] if /dev/ipmi0 or /dev/ipmi/0 or /dev/ipmidev/0 both not exit, the value of return should be 0 not 1 Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- tests/host-info | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/host-info b/tests/host-info index 3984727b4..fe55d1ff2 100755 --- a/tests/host-info +++ b/tests/host-info @@ -102,7 +102,7 @@ get_ipmi_ip() # 802.1q VLAN ID : Disabled # RMCP+ Cipher Suites : 0,1,2,3,17 # Cipher Suite Priv Max : XuuaXXXXXXXXXXX - [ -e /dev/ipmi0 ] || [ -e /dev/ipmi/0 ] || [ -e /dev/ipmidev/0 ] || return + [ -e /dev/ipmi0 ] || [ -e /dev/ipmi/0 ] || [ -e /dev/ipmidev/0 ] || return 0 local ip=$(ipmitool lan print | awk 'NR==3{print $4}') [ -n "$ip" ] && echo "ipmi_ip: $ip" } -- 2.23.0

1 0

[PATCH v4 compass-ci 1/6] fetch-mail: config file for fetchmail
by Luan Shengde 22 Dec '20

22 Dec '20

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/fetch-mail/fetchmailrc.template | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 container/fetch-mail/fetchmailrc.template diff --git a/container/fetch-mail/fetchmailrc.template b/container/fetch-mail/fetchmailrc.template new file mode 100644 index 0000000..8c7e82b --- /dev/null +++ b/container/fetch-mail/fetchmailrc.template @@ -0,0 +1,5 @@ +poll imap.qq.com with proto IMAP +username "FETCH_MAIL_ADDRESS" +password "FETCH_MAIL_AUTH_CODE" +mda "/usr/bin/procmail -f %F -d %T" +nokeep -- 2.23.0

2 2

[PATCH v4 compass-ci 2/6] fetch-mail: config file for procmail
by Luan Shengde 22 Dec '20

22 Dec '20

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/fetch-mail/procmailrc.template | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 container/fetch-mail/procmailrc.template diff --git a/container/fetch-mail/procmailrc.template b/container/fetch-mail/procmailrc.template new file mode 100644 index 0000000..9cea4bc --- /dev/null +++ b/container/fetch-mail/procmailrc.template @@ -0,0 +1,25 @@ +DROPPRIVS=yes +PATH=$HOME/bin:/usr/local/bin:/usr/bin:/bin +MAILDIR=FETCH_MAIL_DIR +MAILLIST=${MAILDIR} +DEFAULT=${MAILDIR}/FETCH_MAIL_BOX/ +ORGMAIL=${DEFAULT}/FETCH_MAIL_BOX/ +LOGFILE=${MAILDIR}/log/procmail # recommended + +UMASK=002 + +DISPLAY=:0 +XAUTHORITY=$HOME/.Xauthority + +# One often receives duplicate messages for cross-postings to mailing +# lists, or explicit replies to a post from you in addition to the +# mailing list message. +# +# The following recipe eliminates all duplicate messages. It tells +# formail(1) to keep a cache file of the Message-IDs of the most recent +# messages you've received. Since Message-IDs are guaranteed unique for +# distinct messages, they are ideally suited to weed out duplicates. + +# LOCKFILE=$MAILDIR/.msgid.lock +:0 Wh: $MAILDIR/.msgid.lock +| formail -D 655360 $MAILDIR/log/msgid -- 2.23.0

2 2

[PATCH v2 compass-ci] lib/matrix2.rb: fix rubocop warning
by Lu Weitao 22 Dec '20

22 Dec '20

[Why] lib/matrix2.rb:81:1: C: Metrics/AbcSize: Assignment Branch Condition size for create_matrix is too high. [<5, 16, 5> 17.49/15] def create_matrix(job_list) ... ^^^^^^^^^^^^^^^^^^^^^^^^^^^ lib/matrix2.rb:81:1: C: Metrics/CyclomaticComplexity: Cyclomatic complexity for create_matrix is too high. [9/7] def create_matrix(job_list) ... ^^^^^^^^^^^^^^^^^^^^^^^^^^^ lib/matrix2.rb:81:1: C: Metrics/MethodLength: Method has too many lines. [18/15] def create_matrix(job_list) ... ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/matrix2.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index a7b83c6..19eb8f1 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -83,13 +83,7 @@ def create_matrix(job_list) suites = [] job_list.each do |job| stats = job['stats'] - next unless stats - - if job['suite'] - next unless stats.keys.any? { |stat| stat.start_with?(job['suite']) } - - suites << job['suite'] - end + next unless stats && assign_suites(suites, job, stats) stats.each do |key, value| next if key.include?('timestamp') @@ -103,6 +97,12 @@ def create_matrix(job_list) return matrix, suites end +def assign_suites(suites, job, stats) + return unless job['suite'] && stats.keys.any? { |stat| stat.start_with?(job['suite']) } + + suites << job['suite'] +end + # input: query results from es_query # return: matrix def combine_query_data(query_data) -- 2.23.0

1 0

[PATCH compass-ci] lib/matrix2.rb: fix rubocop offenses
by Lu Kaiyi 22 Dec '20

22 Dec '20

[error] Inspecting 1 file C Offenses: matrix2.rb:160:35: C: Style/TrailingUnderscoreVariable: Do not use trailing _s in parallel assignment. Prefer groups[group_key][dim_key], = create_matrix(jobs). groups[group_key][dim_key], _ = create_matrix(jobs) ^^ Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/matrix2.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index dfc606e..5f538bd 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -157,7 +157,7 @@ def combine_group_jobs_list(query_data, groups_params, dimensions, metrics) groups = auto_group_by_template(job_list, groups_params, dimensions, metrics) groups.each do |group_key, dims| dims.each do |dim_key, jobs| - groups[group_key][dim_key], _ = create_matrix(jobs) + groups[group_key][dim_key], = create_matrix(jobs) end end -- 2.23.0

1 0

[PATCH compass-ci] lib/matrix2.rb: fix rubocop warning
by Lu Weitao 22 Dec '20

22 Dec '20

[Why] lib/matrix2.rb:81:1: C: Metrics/AbcSize: Assignment Branch Condition size for create_matrix is too high. [<5, 16, 5> 17.49/15] def create_matrix(job_list) ... ^^^^^^^^^^^^^^^^^^^^^^^^^^^ lib/matrix2.rb:81:1: C: Metrics/CyclomaticComplexity: Cyclomatic complexity for create_matrix is too high. [9/7] def create_matrix(job_list) ... ^^^^^^^^^^^^^^^^^^^^^^^^^^^ lib/matrix2.rb:81:1: C: Metrics/MethodLength: Method has too many lines. [18/15] def create_matrix(job_list) ... ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/matrix2.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index a7b83c6..fc805d8 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -85,12 +85,7 @@ def create_matrix(job_list) stats = job['stats'] next unless stats - if job['suite'] - next unless stats.keys.any? { |stat| stat.start_with?(job['suite']) } - - suites << job['suite'] - end - + assign_suites(suites, job, stats) stats.each do |key, value| next if key.include?('timestamp') @@ -103,6 +98,12 @@ def create_matrix(job_list) return matrix, suites end +def assign_suites(suites, job, stats) + return unless job['suite'] && stats.keys.any? { |stat| stat.start_with?(job['suite']) } + + suites << job['suite'] +end + # input: query results from es_query # return: matrix def combine_query_data(query_data) -- 2.23.0

1 1

[PATCH v2 compass-ci] lib/matrix2.rb remove the *timestamp* from matrix
by Lu Weitao 22 Dec '20

22 Dec '20

[Why] In compare results: the value of *timestamp*(metric) doesn't mean the number of occurrences, it may confuse user like bellow: openeuler centos archLinux metric -------------------- ------------------------------ ------------------------------ ------------------------------ fails:runs change fails:runs change fails:runs | | | | | 3585:13 +6913.7% 344:1 -2.6e+4% 11:1 dmesg.timestamp:last 2350:13 +1.6e+4% 344:1 -1.7e+4% 10:1 kmsg.timestamp:last 2:13 -15.4% 0:1 +84.6% 1:1 last_state.exit_fail [How] remove the metric that include "timestamp" from matrix Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/matrix2.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index c712e32..2769f7c 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -87,6 +87,8 @@ def create_matrix(job_list) next unless stats stats.each do |key, value| + next if key.include?('timestamp') + matrix[key] = [] unless matrix[key] matrix[key] << value end -- 2.23.0

3 3

[PATCH compass-ci] lib/matrix2.rb: remove duplicate code
by Lu Weitao 22 Dec '20

22 Dec '20

[Why] Had handle empty group in lib/params_group.rb def filter_groups(groups) groups.each do |group_key, value| value.each_key do |dim_key| value.delete(dim_key) if value[dim_key].empty? end groups.delete(group_key) if groups[group_key].empty? end end Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/matrix2.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index dfc606e..a7b83c6 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -124,10 +124,6 @@ def combine_group_query_data(job_list, dims) suites_hash = {} groups = auto_group(job_list, dims) groups.each do |group_key, value| - if value.empty? - groups.delete(group_key) - next - end suite_list = [] value.each do |dimension_key, jobs| groups[group_key][dimension_key], suites = create_matrix(jobs) -- 2.23.0

1 0

[PATCH v3 lkp-tests] tests/build-pkg: add config file support
by Wang Yong 22 Dec '20

22 Dec '20

[Why] config file in some software need storage and recall now we can add config value in yaml and cp ${config} to ${cci_dir}/${pkgname}/ in PKGBUILD so that config file can be uploaded correctly $ cat test.yaml ... config: ${config} ... Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/build-pkg | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/build-pkg b/tests/build-pkg index addff9f7b..06a58d498 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -4,6 +4,7 @@ # - upstream_commit # - git_tag # - pkg_ver +# - config # upstream_url # We'll use the below global env vars, but not treat them as parameter to avoid @@ -20,6 +21,7 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/distro/common +. $LKP_SRC/lib/http.sh check_vars() { @@ -31,6 +33,29 @@ check_vars() [ -n "$os_mount" ] || die "os_mount is empty" } +check_config_var() +{ + [ -n "$config" ] || return +} + +get_config() +{ + local cci_http_host=${INITRD_HTTP_HOST} + local cci_http_port=${INITRD_HTTP_PORT:-8800} + export cci_dir=/cci/build-config + local remote_dir="http://${cci_http_host}:${cci_http_port}${cci_dir}/${pkgname}" + + mkdir -p ${cci_dir} + + check_config_var + http_get_file "${remote_dir}/${config}" "${cci_dir}/${pkgname}/${config}" +} + +upload_config() +{ + upload_to_target_dir "${cci_dir}/${pkgname}" +} + mount_dest() { # the same image is mounted to cifs and nfs, the generated cgz files @@ -46,7 +71,7 @@ get_pkgfile() { curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' \ -d '{"git_repo": "'$1'", "git_command": ["git-show", "HEAD:'$2'"]}' -o "${2##*/}" - } +} request_pkg() { @@ -125,13 +150,17 @@ build_source_pkg() fi } replace_source PKGBUILD.src - create_cgz_name + get_config PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ $LKP_SRC/sbin/makepkg ${pkg_args} --config $LKP_SRC/etc/makepkg.conf - [ "$?" == 0 ] || exit 1 + local build_state="$?" + + upload_config + + [ "${build_state}" == "0" ] || exit 1 create_softlink upload_to_target_dir ${PKG_MNT}/${pack_to}/${pkgname} } -- 2.23.0

1 0

[PATCH v2 compass-ci] lib/job.cr: check account info
by Wu Zhende 22 Dec '20

22 Dec '20

If submit yaml in /srv/result, will not check account info. Fix it Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/job.cr | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 4d0b5df..e6f7e6a 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -51,17 +51,23 @@ class Job def initialize(job_content : JSON::Any, id) @hash = job_content.as_h + @es = Elasticsearch::Client.new + @account_info = Hash(String, JSON::Any).new # init job with "-1", or use the original job_content["id"] id = "-1" if "#{id}" == "" if initialized? - return if @hash["id"] == "#{id}" + if @hash["id"] == "#{id}" + return unless @hash.has_key?("my_uuid") || @hash.has_key?("my_token") + + check_account_info() + return + end end - @es = Elasticsearch::Client.new @hash["id"] = JSON::Any.new("#{id}") - @account_info = Hash(String, JSON::Any).new + check_required_keys() check_account_info() set_defaults() -- 2.23.0

2 1

[PATCH v2 lkp-tests] daemon: fix ssh login testbox for docker
by Xiao Shenwei 22 Dec '20

22 Dec '20

[why] systemctl is not supported when use docker [how] use the sshd command Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- daemon/sshd | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/daemon/sshd b/daemon/sshd index 41fa99adf..6bfda467e 100755 --- a/daemon/sshd +++ b/daemon/sshd @@ -6,19 +6,31 @@ # sshr_port_base . $LKP_SRC/lib/http.sh +. $LKP_SRC/lib/env.sh sshr_ip=$LKP_SERVER [ -n "$sshr_port" ] || sshr_port=5050 [ -n "$sshr_port_len" ] || sshr_port_len=2000 [ -n "$sshr_port_base" ] || sshr_port_base=50000 +generate_key_file() +{ + echo -e "y\n" | ssh-keygen -t rsa -b 2048 -N '' -f /etc/ssh/ssh_host_rsa_key +} + run_ssh() { [ -n "$my_ssh_pubkey" ] || return umask 0077 mkdir -p /root/.ssh echo "$my_ssh_pubkey" > /root/.ssh/authorized_keys - systemctl start sshd + if is_docker; then + generate_key_file + sshd_path=$(cmd_path sshd) + $sshd_path -q + else + systemctl start sshd + fi } data_success() -- 2.23.0

4 5

Re: [PATCH lkp-tests] fix lkp make install error
by Wu Fengguang 22 Dec '20

22 Dec '20

TO ALL Best practice is to use realpath. >-. $LKP_SRC/lib/env.sh >-. $LKP_SRC/sbin/set-env.sh >+. lib/env.sh >+. sbin/set-env.sh SCRIPT_DIR=$(dirname $(realpath $0)) PROJECT_DIR=$(dirname $SCRIPT_DIR) $PROJECT_DIR/lib/env.sh $PROJECT_DIR/sbin/set-env.sh Thanks, Fengguang

2 1

[PATCH lkp-tests] daemon/multi-qemu: add $CACHE_DIR judgment
by Hu Xuejiao 22 Dec '20

22 Dec '20

$CACHE_DIR is not created in crystal Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- daemon/multi-qemu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/multi-qemu b/daemon/multi-qemu index 90197dae..87b40758 100755 --- a/daemon/multi-qemu +++ b/daemon/multi-qemu @@ -13,7 +13,7 @@ multi_qemu() export CCI_SRC=${CCI_SRC:-"/c/compass-ci"} local hostname=$tbox_group.$HOSTNAME - cd "$CACHE_DIR" + [ -n "$CACHE_DIR" ] && cd "$CACHE_DIR" $CCI_SRC/providers/multi-qemu -n "$hostname" -c "$nr_vm" -q "$queues" } -- 2.23.0

3 2

[PATCH compass-ci] lib: add assign_account_client
by Luan Shengde 22 Dec '20

22 Dec '20

add assign_account_client for code calling to assign account Signed-off-by: Luan Shengde <shdluan(a)163.com> --- lib/assign_account_client.rb | 83 ++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100755 lib/assign_account_client.rb diff --git a/lib/assign_account_client.rb b/lib/assign_account_client.rb new file mode 100755 index 0000000..d7e0623 --- /dev/null +++ b/lib/assign_account_client.rb @@ -0,0 +1,83 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'mail' +require 'set' +require 'optparse' +require_relative '../container/defconfig' +require_relative 'es_client' +require_relative '../container/mail-robot/lib/assign-account-email' + +names = Set.new %w[ + JUMPER_HOST + JUMPER_PORT + SEND_MAIL_HOST + SEND_MAIL_PORT +] + +defaults = relevant_defaults(names) +JUMPER_HOST = defaults['JUMPER_HOST'] +JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 +SEND_MAIL_HOST = defaults['SEND_MAIL_HOST'] || 'localhost' +SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 49000 +LAB = ENV['lab'] + +# used for other codes calling to assign account for user +class AutoAssignAccount + def initialize(user_info) + @my_info = user_info + + @my_info_es = {} + end + + def update_from_es + account_infos = ESQuery.new(index: 'accounts') + my_account_info = account_infos.query_by_id(@my_info['my_email']) || {} + + my_account_info.update @my_info + @my_info.update my_account_info + @my_info['my_ssh_pubkey'] = [] unless my_account_info['my_ssh_pubkey'] + + @my_info['my_token'] = %x(uuidgen).chomp if @my_info['my_login_name'].nil? + end + + def apply_account + apply_info = {} + + apply_info.update @my_info + apply_info['enable_login'] = true + apply_info['is_update_account'] = true unless apply_info['my_login_name'].nil? + apply_info['lab'] = LAB + + account_info_str = %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d '#{apply_info.to_json}') + JSON.parse account_info_str + end + + def update_my_info_from_account_info + account_info = apply_account + + @my_info['my_login_name'] = account_info['my_login_name'] + @my_info['my_ssh_pubkey'] << account_info['my_jumper_pubkey'] unless account_info['my_jumper_pubkey'].nil? + end + + def store_account_info + es = ESClient.new(index: 'accounts') + es.put_source_by_id(@my_info['my_email'], @my_info) + end + + def send_mail + message = build_apply_account_email(@my_info) + %x(curl -XPOST "#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text" -d "#{message}") + end + + def send_account + update_from_es + update_my_info_from_account_info + + store_account_info + send_mail + end +end -- 2.23.0

2 2

[PATCH v2 lkp-tests 1/4] bin/install-dependencies.sh: refactor file
by Wei Jihui 22 Dec '20

22 Dec '20

all dependent functions is move to lib/env.sh, sbin/set-env.sh in not required. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- sbin/install-dependencies.sh | 40 +++++++++++++++++------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/sbin/install-dependencies.sh b/sbin/install-dependencies.sh index 7605a29f..b4d3ad22 100755 --- a/sbin/install-dependencies.sh +++ b/sbin/install-dependencies.sh @@ -1,30 +1,28 @@ #!/bin/bash . $LKP_SRC/lib/env.sh -. $LKP_SRC/sbin/set-env.sh -# choose install function base on -# DISTRIBUTION +# choose install function base on DISTRIBUTION linux_dep() { -source /etc/os-release -case $ID in -ubuntu|debian) - export DEBIAN_FRONTEND=noninteractive - apt-get install -yqm ruby-git ruby-activesupport ruby-rest-client - ;; -openEuler|fedora|rhel|centos) - if type dnf > /dev/null 2>&1; then - dnf install -y --skip-broken ruby rubygems gcc gcc-c++ make ruby-devel git - else - yum install -y --skip-broken ruby rubygems gcc gcc-c++ make ruby-devel git - fi - sudo gem install -f git activesupport rest-client - ;; -*) - echo "$ID not support! please install dependencies manually." - ;; -esac + source /etc/os-release + case $ID in + ubuntu|debian) + export DEBIAN_FRONTEND=noninteractive + apt-get install -yqm ruby-git ruby-activesupport ruby-rest-client + ;; + openEuler|fedora|rhel|centos) + if type dnf > /dev/null 2>&1; then + dnf install -y --skip-broken ruby rubygems gcc gcc-c++ make ruby-devel git + else + yum install -y --skip-broken ruby rubygems gcc gcc-c++ make ruby-devel git + fi + sudo gem install -f git activesupport rest-client + ;; + *) + echo "$ID not support! please install dependencies manually." + ;; + esac } -- 2.23.0

3 3

[PATCH v2 lkp-tests] tests/build-pkg: add config file support
by Wang Yong 21 Dec '20

21 Dec '20

[Why] config file in some software need storage and recall now we can add config value in yaml and cp ${config} to ${cci_dir}/${benchmark}/ in PKGBUILD so that config file can be uploaded correctly $ cat test.yaml ... config: ${config} ... Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/build-pkg | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/tests/build-pkg b/tests/build-pkg index addff9f7b..e539be923 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -4,6 +4,7 @@ # - upstream_commit # - git_tag # - pkg_ver +# - config # upstream_url # We'll use the below global env vars, but not treat them as parameter to avoid @@ -20,6 +21,7 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/distro/common +. $LKP_SRC/lib/http.sh check_vars() { @@ -31,6 +33,29 @@ check_vars() [ -n "$os_mount" ] || die "os_mount is empty" } +check_config_var() +{ + [ -n "$config" ] || return +} + +get_config() +{ + local cci_http_host=${INITRD_HTTP_HOST} + local cci_http_port=${INITRD_HTTP_PORT:-8800} + export cci_dir=/cci/config + local remote_dir="http://${cci_http_host}:${cci_http_port}${cci_dir}/${pkgname}" + + mkdir -p ${cci_dir} + + check_config_var + http_get_file "${remote_dir}/${config}" "${cci_dir}/${pkgname}/${config}" +} + +upload_config() +{ + upload_to_target_dir "${cci_dir}/${pkgname}" +} + mount_dest() { # the same image is mounted to cifs and nfs, the generated cgz files @@ -46,7 +71,7 @@ get_pkgfile() { curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' \ -d '{"git_repo": "'$1'", "git_command": ["git-show", "HEAD:'$2'"]}' -o "${2##*/}" - } +} request_pkg() { @@ -125,13 +150,17 @@ build_source_pkg() fi } replace_source PKGBUILD.src - create_cgz_name + get_config PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ $LKP_SRC/sbin/makepkg ${pkg_args} --config $LKP_SRC/etc/makepkg.conf - [ "$?" == 0 ] || exit 1 + local build_state="$?" + + upload_config + + [ "${build_state}" == "0" ] || exit 1 create_softlink upload_to_target_dir ${PKG_MNT}/${pack_to}/${pkgname} } -- 2.23.0

1 0

[PATCH lkp-tests] daemon/multi-docker: add default value of queues
by Hu Xuejiao 21 Dec '20

21 Dec '20

Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- daemon/multi-docker | 2 ++ 1 file changed, 2 insertions(+) diff --git a/daemon/multi-docker b/daemon/multi-docker index 257645b8..7759fdf3 100755 --- a/daemon/multi-docker +++ b/daemon/multi-docker @@ -6,6 +6,8 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +: ${queues:="$tbox_group.$HOSTNAME,$tbox_group.$(arch)"} + multi_docker() { export CCI_SRC=${CCI_SRC:-"/c/compass-ci"} -- 2.23.0

1 0

[PATCH v4 compass-ci 3/3] lib: use suites_hash instead of suites_list to keep consistent
by Lu Kaiyi 21 Dec '20

21 Dec '20

[why] suite_list maybe not consistent with group key if sort group matrixes. [how] use suites_hash instead of suites_list to keep correct congruent relationship with same group_key. Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/compare.rb | 4 ++-- lib/compare_matrixes.rb | 10 +++++----- lib/matrix2.rb | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/compare.rb b/lib/compare.rb index da761ef..9090c3b 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -62,8 +62,8 @@ end def compare_group(argv, dimensions, options) conditions = parse_conditions(argv) dims = dimensions.split(' ') - groups_matrices, suites_list = create_groups_matrices_list(conditions, dims) - compare_group_matrices(groups_matrices, suites_list, options) + groups_matrices, suites_hash = create_groups_matrices_list(conditions, dims) + compare_group_matrices(groups_matrices, suites_hash, options) end def create_groups_matrices_list(conditions, dims) diff --git a/lib/compare_matrixes.rb b/lib/compare_matrixes.rb index 0727797..6e4c2aa 100644 --- a/lib/compare_matrixes.rb +++ b/lib/compare_matrixes.rb @@ -387,25 +387,25 @@ end # input: group matrices # output: pre compare result of each group # the result with more comparison objects first -def compare_group_matrices(group_matrices, suites_list, options) +def compare_group_matrices(group_matrices, suites_hash, options) result_str = '' group_matrices_array = sort_by_matrix_size(group_matrices) group_matrices_array.each do |matrice_kv| - result_str += get_matrix_str(matrice_kv[0], matrice_kv[1], suites_list, options) + result_str += get_matrix_str(matrice_kv[0], matrice_kv[1], suites_hash[matrice_kv[0]], options) end result_str end -def get_matrix_str(matrice_key, matrice_value, suites_list, options) +def get_matrix_str(matrice_key, matrice_value, suite_list, options) m_list = [] m_titles = [] matrice_value.each do |dim, matrix| m_titles << dim m_list << matrix end - return compare_matrixes(m_list, suites_list.shift, m_titles, matrice_key, options: options) if options[:no_print] + return compare_matrixes(m_list, suite_list, m_titles, matrice_key, options: options) if options[:no_print] - print compare_matrixes(m_list, suites_list.shift, m_titles, matrice_key, options: options) + print compare_matrixes(m_list, suite_list, m_titles, matrice_key, options: options) return '' end diff --git a/lib/matrix2.rb b/lib/matrix2.rb index 4393a65..dfc606e 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -121,7 +121,7 @@ end # ... # } def combine_group_query_data(job_list, dims) - suites_list = [] + suites_hash = {} groups = auto_group(job_list, dims) groups.each do |group_key, value| if value.empty? @@ -133,10 +133,10 @@ def combine_group_query_data(job_list, dims) groups[group_key][dimension_key], suites = create_matrix(jobs) suite_list.concat(suites) end - suites_list << suite_list + suites_hash[group_key] = suite_list end - return groups, suites_list + return groups, suites_hash end # input: -- 2.23.0

1 0

[PATCH v4 compass-ci 2/3] lib/compare_matrixes.rb: don't remove the field that start with $suite
by Lu Kaiyi 21 Dec '20

21 Dec '20

if the metric field start with $suite, even if the field is unchanged, still keep it in compare result. Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/compare_matrixes.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/compare_matrixes.rb b/lib/compare_matrixes.rb index b7ea5a4..0727797 100644 --- a/lib/compare_matrixes.rb +++ b/lib/compare_matrixes.rb @@ -176,11 +176,15 @@ def get_matrixes_values(matrixes_list, options) matrixes_values end -def remove_unchanged_field(matrixes_values) +def remove_unchanged_field(matrixes_values, suite_list) # remove unchanged field from matrixes values and remove :changed key # matrixes_values.each_key do |success| matrixes_values[success].delete_if do |field| + if suite_list.any? { |suite| field.start_with?(suite) } + matrixes_values[success][field][:changed] = true + end + !matrixes_values[success][field].delete(:changed) end end @@ -203,7 +207,7 @@ def compare_matrixes(matrixes_list, suite_list, matrixes_titles = nil, group_key options = { 'perf-profile': 5, theme: :none, no_print: false }.merge(options) matrixes_values = get_matrixes_values(matrixes_list, options) - remove_unchanged_field(matrixes_values) if matrixes_list.length > 1 + remove_unchanged_field(matrixes_values, suite_list) if matrixes_list.length > 1 no_print = options[:no_print] result_str = group_key ? "\n\n\n\n\n" + group_key : '' result_str += get_all_result_str( -- 2.23.0

1 0

[PATCH v4 compass-ci 1/3] lib/matrix2.rb: discard the no $suite related job
by Lu Kaiyi 21 Dec '20

21 Dec '20

if job has suite, but no $suite related stats metric, then discard this job when create matrix. Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/matrix2.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index bff4eee..4393a65 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -82,10 +82,15 @@ def create_matrix(job_list) matrix = {} suites = [] job_list.each do |job| - suites << job['suite'] if job['suite'] stats = job['stats'] next unless stats + if job['suite'] + next unless stats.keys.any? { |stat| stat.start_with?(job['suite']) } + + suites << job['suite'] + end + stats.each do |key, value| next if key.include?('timestamp') -- 2.23.0

1 0

[PATCH compass-ci] service/scheduler: move the service logic to the Sched class
by Wu Zhende 21 Dec '20

21 Dec '20

Keep scheduler.cr neat and do not perform operations related to env. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 40 +++++++++++++++++++++++++++------- src/scheduler/find_job_boot.cr | 2 +- src/scheduler/scheduler.cr | 32 ++++----------------------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index a1ef28e..f7220f9 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -51,20 +51,44 @@ class Sched mac.gsub(":", "-") end - def set_host_mac(mac : String, hostname : String) - @redis.hash_set("sched/mac2host", normalize_mac(mac), hostname) + def set_host_mac + if (hostname = @env.params.query["hostname"]?) && (mac = @env.params.query["mac"]?) + @redis.hash_set("sched/mac2host", normalize_mac(mac), hostname) + + "Done" + else + "No yet!" + end end - def del_host_mac(mac : String) - @redis.hash_del("sched/mac2host", normalize_mac(mac)) + def del_host_mac + if mac = @env.params.query["mac"]? + @redis.hash_del("sched/mac2host", normalize_mac(mac)) + + "Done" + else + "No yet!" + end end - def set_host2queues(hostname : String, queues : String) - @redis.hash_set("sched/host2queues", hostname, queues) + def set_host2queues + if (queues = @env.params.query["queues"]?) && (hostname = @env.params.query["host"]?) + @redis.hash_set("sched/host2queues", hostname, queues) + + "Done" + else + "No yet!" + end end - def del_host2queues(hostname : String) - @redis.hash_del("sched/host2queues", hostname) + def del_host2queues + if hostname = @env.params.query["host"]? + @redis.hash_del("sched/host2queues", hostname) + + "Done" + else + "No yet!" + end end def update_tbox_wtmp diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 7565375..9b3db88 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -29,7 +29,7 @@ class Sched # then hostname will be "sut-00-01-02-03-04" !!! def submit_host_info_job(mac) host = "sut-#{mac}" - set_host_mac(mac, host) + @redis.hash_set("sched/mac2host", normalize_mac(mac), host) Jobfile::Operate.auto_submit_job( "#{ENV["LKP_SRC"]}/jobs/host-info.yaml", diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr index 7ec149c..2103f7c 100644 --- a/src/scheduler/scheduler.cr +++ b/src/scheduler/scheduler.cr @@ -79,46 +79,22 @@ module Scheduler # # curl -X PUT "http://localhost:3000/set_host_mac?hostname=wfg&mac=00-01-02-03-04-05" put "/set_host_mac" do |env| - if (client_hostname = env.params.query["hostname"]?) && (client_mac = env.params.query["mac"]?) - env.sched.set_host_mac(client_mac, client_hostname) - - "Done" - else - "No yet!" - end + env.sched.set_host_mac end # curl -X PUT "http://localhost:3000/set_host2queues?queues=vm-2p8g.aarch64&host=vm-2p8g.a…" put "/set_host2queues" do |env| - if (client_queues = env.params.query["queues"]?) && (client_host = env.params.query["host"]?) - env.sched.set_host2queues(client_host, client_queues) - - "Done" - else - "No yet" - end + env.sched.set_host2queues end # curl -X PUT "http://localhost:3000/del_host_mac?mac=00-01-02-03-04-05" put "/del_host_mac" do |env| - if client_mac = env.params.query["mac"]? - env.sched.del_host_mac(client_mac) - - "Done" - else - "No yet!" - end + env.sched.del_host_mac end # curl -X PUT "http://localhost:3000/del_host2queues?host=vm-2p8g.aarch64" put "/del_host2queues" do |env| - if client_host = env.params.query["host"]? - env.sched.del_host2queues(client_host) - - "Done" - else - "No yet!" - end + env.sched.del_host2queues end # client(runner) report job's status -- 2.23.0

1 0

[PATCH compass-ci] lib/job.cr: check account info
by Wu Zhende 21 Dec '20

21 Dec '20

If submit yaml in /srv/result, will not check account info. Fix it Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/job.cr | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 4d0b5df..e6f7e6a 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -51,17 +51,23 @@ class Job def initialize(job_content : JSON::Any, id) @hash = job_content.as_h + @es = Elasticsearch::Client.new + @account_info = Hash(String, JSON::Any).new # init job with "-1", or use the original job_content["id"] id = "-1" if "#{id}" == "" if initialized? - return if @hash["id"] == "#{id}" + if @hash["id"] == "#{id}" + return unless @hash.has_key?("my_uuid") || @hash.has_key?("my_token") + + check_account_info() + return + end end - @es = Elasticsearch::Client.new @hash["id"] = JSON::Any.new("#{id}") - @account_info = Hash(String, JSON::Any).new + check_required_keys() check_account_info() set_defaults() -- 2.23.0

1 0

[PATCH compass-ci 4/4] web_backend.rb: optimize the way of query data for compare
by Li Yuanchao 21 Dec '20

21 Dec '20

Before: When do compare with dimension, first query 100(default size) jobs by filters, then group these jobs by dimension. But if one kind of job is done too many, other kinds of jobs may can not be got. After: Fisrt query how many kinds of dimensions in es, take dimension = 'os' as an example, the result can be ["openeuler", "archlinux", "centos", "debian", "kylin", "archLinux"] Then query 100 jobs by each os, then group all these jobs. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/compare.rb | 2 +- lib/matrix2.rb | 3 +- src/lib/web_backend.rb | 64 ++++++++++++++++++++++++++++-------------- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/lib/compare.rb b/lib/compare.rb index db264f8..da761ef 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -69,7 +69,7 @@ end def create_groups_matrices_list(conditions, dims) es = ESQuery.new(ES_HOST, ES_PORT) query_results = es.multi_field_query(conditions) - combine_group_query_data(query_results, dims) + combine_group_query_data(query_results['hits']['hits'], dims) end # ------------------------------------------------------------------------------------------- diff --git a/lib/matrix2.rb b/lib/matrix2.rb index 2507b77..b6cfa79 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -109,9 +109,8 @@ end # group2_key => {...} # ... # } -def combine_group_query_data(query_data, dims) +def combine_group_query_data(job_list, dims) suites_list = [] - job_list = query_data['hits']['hits'] groups = auto_group(job_list, dims) groups.each do |group_key, value| if value.empty? diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index c4182a3..a680b8a 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -34,7 +34,7 @@ ALL_FIELDS = FIELDS + NOT_SHOW_FIELDS NOT_NEED_EXIST_FIELDS = %w[error_ids upstream_repo].freeze PREFIX_SEARCH_FIELDS = ['tbox_group'].freeze ES_CLIENT = Elasticsearch::Client.new(url: "http://#{ES_HOST}:#{ES_PORT}") -COMPARE_RECORDS_NUMBER = 50 +COMPARE_RECORDS_NUMBER = 100 def es_query(query) ES_CLIENT.search index: 'jobs*', body: query @@ -148,30 +148,21 @@ def get_es_must(params) must end -def do_get_groups_matrices(query, dimension, total, size, from) - result = es_query(query) - matrices, suites_list = combine_group_query_data(result, dimension) - while matrices.empty? - from += size - break if from > total - - query[:from] = from - result = es_query(query) - matrices, suites_list = combine_group_query_data(result, dimension) - end - [matrices, suites_list] +def get_dimension_list(dimension) + query = { size: 0, aggs: { dims: { terms: { size: 10000, field: dimension[0] } } } } + buckets = es_query(query)['aggregations']['dims']['buckets'] + dimension_list = [] + buckets.each { |dims_agg| dimension_list << dims_agg['key'] } + return dimension_list end -def get_groups_matrices(conditions, dimension, must, size, from) - must += build_multi_field_subquery_body(conditions) - count_query = { query: { bool: { must: must } } } - total = es_count(count_query) - return {} if total < 1 - +def query_dimension(dim_field, dim_value, must, size, from) + must_dim = Array.new(must) + must_dim << { term: { dim_field => dim_value } } query = { query: { bool: { - must: must + must: must_dim } }, size: size, @@ -180,8 +171,39 @@ def get_groups_matrices(conditions, dimension, must, size, from) start_time: { order: 'desc' } }] } + es_query(query)['hits']['hits'] +end + +def get_dimension_job_list(dimension, must, size, from) + dimension_list = get_dimension_list(dimension) + job_list = [] + dimension_list.each do |dim| + job_list += query_dimension(dimension[0], dim, must, size, from) + end + job_list +end + +def do_get_groups_matrices(must, dimension, total, size, from) + job_list = get_dimension_job_list(dimension, must, size, from) + + matrices, suites_list = combine_group_query_data(job_list, dimension) + while matrices.empty? + from += size + break if from > total + + job_list = get_dimension_job_list(dimension, must, size, from) + matrices, suites_list = combine_group_query_data(job_list, dimension) + end + [matrices, suites_list] +end + +def get_groups_matrices(conditions, dimension, must, size, from) + must += build_multi_field_subquery_body(conditions) + count_query = { query: { bool: { must: must } } } + total = es_count(count_query) + return {} if total < 1 - do_get_groups_matrices(query, dimension, total, size, from) + do_get_groups_matrices(must, dimension, total, size, from) end def get_compare_body(params) -- 2.23.0

2 1