- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci 1/3] sparrow/1-storage: add work dir and permission for /c/git-repos
by Liu Yinsi 23 Mar '21

23 Mar '21

the URL of the repository to be tested will store in /c/git-repos, set lkp:committer permission for git-mirror. refer: [PATCH compass-ci] container/git-mirror: change mount dir Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/1-storage/permission | 2 ++ sparrow/1-storage/tiny | 1 + 2 files changed, 3 insertions(+) diff --git a/sparrow/1-storage/permission b/sparrow/1-storage/permission index 26086d5..3d9bdfa 100755 --- a/sparrow/1-storage/permission +++ b/sparrow/1-storage/permission @@ -11,6 +11,7 @@ chmod_dir=( /srv/cci/serial/fluentd-pos /srv/cci/Maildir /srv/rabbitmq + /c/git-repos ) chmod 775 "${chmod_dir[@]}" @@ -29,6 +30,7 @@ chown_dir=( chown lkp:lkp "${chown_dir[@]}" chown lkp:committer /srv/git +chown lkp:committer /c/git-repos chown mailer:team /srv/cci/Maildir chgrp team /srv/cci/serial/logs diff --git a/sparrow/1-storage/tiny b/sparrow/1-storage/tiny index 9939d93..0eb7d39 100755 --- a/sparrow/1-storage/tiny +++ b/sparrow/1-storage/tiny @@ -20,6 +20,7 @@ dirs=( /srv/cci/libvirt-xml /tftpboot /c + /c/git-repos /etc/qemu /srv/cci/Maildir/.compass-ci/new /srv/cci/Maildir/.compass-ci/cur -- 2.23.0

1 0

[PATCH lkp-tests] daemon/sshd: change sshr service port
by Wu Zhende 23 Mar '21

23 Mar '21

Restarting the sshr service will disconnect the existing sshr connection and the machine that has been applied for can't be accessed. Therefore, we can't restart the sshr service. Instead, we start a new sshr service and use a new port. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- daemon/sshd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon/sshd b/daemon/sshd index 50a7677ab..e0616ea8b 100755 --- a/daemon/sshd +++ b/daemon/sshd @@ -13,7 +13,7 @@ PROJECT_DIR=$(dirname $SCRIPT_DIR) . $PROJECT_DIR/lib/email.sh sshr_ip=$LKP_SERVER -[ -n "$sshr_port" ] || sshr_port=5050 +[ -n "$sshr_port" ] || sshr_port=5051 [ -n "$sshr_port_len" ] || sshr_port_len=2000 [ -n "$sshr_port_base" ] || sshr_port_base=21000 -- 2.23.0

1 0

[PATCH compass-ci] docker/docker.rb: fix failed to borrow and run multi-docker
by Lin Jiaxin 23 Mar '21

23 Mar '21

job id=z9.1471951 ==> /tmp/stderr <== /usr/share/rubygems/rubygems/core_ext/kernel_require.rb:59:in `require': cannot load such file -- ../lib/mq_client (LoadError) from /usr/share/rubygems/rubygems/core_ext/kernel_require.rb:59:in `require' from /c/compass-ci/providers/docker/docker.rb:11:in `<top (required)>' from /c/compass-ci/providers/multi-docker:7:in `require_relative' from /c/compass-ci/providers/multi-docker:7:in `<main>' Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- providers/docker/docker.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/docker/docker.rb b/providers/docker/docker.rb index 9d6854f..51017f3 100755 --- a/providers/docker/docker.rb +++ b/providers/docker/docker.rb @@ -8,7 +8,7 @@ require 'json' require 'set' require 'fileutils' -require '../lib/mq_client' +require_relative '../../lib/mq_client' require_relative '../../container/defconfig' BASE_DIR = '/srv/dc' -- 2.23.0

1 0

[PATCH compass-ci 2/2] provides: fix require error
by Wu Zhende 23 Mar '21

23 Mar '21

[Error] can't load such file "../lib/mq_client" [How] use absolute path Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- providers/docker/docker.rb | 4 +++- providers/multi-qemu | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/providers/docker/docker.rb b/providers/docker/docker.rb index 9d6854f..b372a11 100755 --- a/providers/docker/docker.rb +++ b/providers/docker/docker.rb @@ -3,12 +3,14 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +CCI_SRC = ENV['CCI_SRC'] || File.dirname(File.dirname(File.realpath($PROGRAM_NAME))) + require 'open-uri' require 'json' require 'set' require 'fileutils' -require '../lib/mq_client' +require "#{CCI_SRC}/lib/mq_client" require_relative '../../container/defconfig' BASE_DIR = '/srv/dc' diff --git a/providers/multi-qemu b/providers/multi-qemu index d3f8406..70675d1 100755 --- a/providers/multi-qemu +++ b/providers/multi-qemu @@ -3,11 +3,13 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +CCI_SRC = ENV['CCI_SRC'] || File.dirname(File.dirname(File.realpath($PROGRAM_NAME))) + require 'fileutils' require 'optparse' require 'json' -require '../lib/mq_client' +require "#{CCI_SRC}/lib/mq_client" opt = {} options = OptionParser.new do |opts| -- 2.23.0

1 0

[PATCH v4 lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 23 Mar '21

23 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 81 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 tests/ansible_test diff --git a/tests/ansible_test b/tests/ansible_test new file mode 100755 index 000000000..bf75b19b7 --- /dev/null +++ b/tests/ansible_test @@ -0,0 +1,81 @@ +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + ansible_host=localhost + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] || { + echo "Ansible failed to install." + exit 1 + } + } +} + +download_ansible_role() +{ + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git + if [ $? -eq 0 ]; then + echo "$benchmark is successfully cloned" + else + echo "Please check whether your ansible_repo is correct!" + exit 1 + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: $ansible_host + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook is running failed!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

3 6

[PATCH compass-ci 3/3] container/mail-robot: optimize git operations
by Luan Shengde 22 Mar '21

22 Mar '21

Use gem resource ruby-git instead of system command. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/parse-apply-account-email.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/container/mail-robot/lib/parse-apply-account-email.rb b/container/mail-robot/lib/parse-apply-account-email.rb index e6c35f7..ce01fec 100755 --- a/container/mail-robot/lib/parse-apply-account-email.rb +++ b/container/mail-robot/lib/parse-apply-account-email.rb @@ -3,9 +3,13 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +# set default system encoding +Encoding.default_external = Encoding::UTF_8 + require 'json' require 'mail' require 'rest-client' +require 'git' # used to parse mail_content for my_commit_url and my_ssh_pubkey # be called by AssignAccount when it needs to extract required data: @@ -110,14 +114,15 @@ class ParseApplyAccountEmail repo_dir = repo_url.split('/')[-1] commit_id = url.split('/')[-1] - %x(/usr/bin/git -C /tmp clone --bare #{repo_url} #{repo_dir}) + Git.clone(repo_url, repo_dir, path: '/tmp/', bare: true) # get all commit IDs and check commit id exists - repo_commits = %x(/usr/bin/git -C /tmp/#{repo_dir} log --pretty=format:'%H').split(/\n/) + repo = Git.bare("/tmp/#{repo_dir}") + repo_commits = repo.lib.log_commits check_commit_exist(commit_id, repo_commits) # get the auther's email for the commit - author_email = %x(/usr/bin/git -C /tmp/#{repo_dir} show -s --format=%aE #{commit_id}).chomp + author_email = repo.gcommit(commit_id).author.email check_commit_email(author_email) FileUtils.rm_rf "/tmp/#{repo_dir}" -- 2.23.0

1 0

[PATCH compass-ci 2/3] add use ENV variable
by Luan Shengde 22 Mar '21

22 Mar '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/apply-account.rb | 2 +- container/mail-robot/start | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/container/mail-robot/lib/apply-account.rb b/container/mail-robot/lib/apply-account.rb index 17705ec..97c3989 100755 --- a/container/mail-robot/lib/apply-account.rb +++ b/container/mail-robot/lib/apply-account.rb @@ -56,7 +56,7 @@ class ApplyAccount @es_port = ES_PORT # email info file for account issuers. - @account_issuer = YAML.load_file '/etc/compass-ci/account-issuer.yaml' + @account_issuer = YAML.load_file ENV['ACCOUNT_ISSUER'] @my_info = {} end diff --git a/container/mail-robot/start b/container/mail-robot/start index bd2a62d..ce18498 100755 --- a/container/mail-robot/start +++ b/container/mail-robot/start @@ -16,6 +16,7 @@ names = Set.new %w[ SCHED_HTTPS_PORT ES_PORT MAILDIR + ACCOUNT_ISSUER lab ] -- 2.23.0

1 0

[PATCH compass-ci 1/3] add install ruby-git
by Luan Shengde 22 Mar '21

22 Mar '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/mail-robot/Dockerfile b/container/mail-robot/Dockerfile index e4b3e7c..41056a5 100644 --- a/container/mail-robot/Dockerfile +++ b/container/mail-robot/Dockerfile @@ -10,7 +10,7 @@ ENV DEBIAN_FRONTEND noninteractive COPY conf/sources.list* /etc/apt/ RUN apt-get update && \ - apt-get install -y git uuid-runtime ruby-listen ruby-json ruby-mail ruby-rest-client + apt-get install -y git uuid-runtime ruby-listen ruby-json ruby-mail ruby-rest-client ruby-git RUN gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ -- 2.23.0

1 0

[PATCH v4 compass-ci] container/web-backend: add API /get_table_error
by Lu Weitao 22 Mar '21

22 Mar '21

[Example] curl -X GET 'localhost:10002/get_job_error?group_id=wcl_ansible-openeuler-03-10' return: { "filter":{"group_id":"wcl_ansible-openeuler-03-10"}, "attributes":["job_id","error_id","error_message","link_to_result"], "objects":[ { "job_id": "crystal.1354921", "error_id": "ansible_test.error.Unable-to-start-service-httpd-Job-for-httpdservice-xxx.fail", "error_message": "{\"changed\": false, \"msg\": \"Unable to start service httpd: Job for httpd.service failed because a timeout was exceeded.}", "link_to_result": "http://172.17.0.1:11300/result/ansible_test/2021-03-10/vm-2p16g--wcl1/opene…", ate/crystal.1354921" }, { "job_id": "crystal.1354929", ... }, ... ] } Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend/web-backend | 19 +++++++++ src/lib/web_backend.rb | 67 +++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 33a3fae..91cebb5 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -146,3 +146,22 @@ end get '/get_repo_statistics' do new_refs_statistics(params) end + +# GET /get_job_error?suite=virttest&tbox_group=vm-2p8g +# Response +# { +# "filter": {"group_id":"wcl_ansible-openeuler-03-10"}, +# "attributes": ["job_id", "error_id", "error_message", "link_to_result"], +# "objects": [ +# { +# "job_id": "crystal.1354921", +# "error_id": "ansible_test.error.Unable-to-start-service-httpd-Job-for-httpdservice-failed.fail", +# "error_message": "{\"changed\": false, \"msg\": \"Unable to start service httpd: Job for httpd.service failed because a timeout was exceeded.}", +# "link_to_result": "http://172.17.0.1:11300/result/ansible_test/2021-03-10/vm-2p16g--wcl1/opene…" +# }, +# ... +# ] +# } +get '/get_job_error' do + get_job_error(params) +end diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 89c0f3c..08bffcb 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -12,6 +12,7 @@ require "#{CCI_SRC}/lib/compare.rb" require "#{CCI_SRC}/lib/constants.rb" require "#{CCI_SRC}/lib/es_query.rb" require "#{CCI_SRC}/lib/matrix2.rb" +require "#{CCI_SRC}/lib/params_group.rb" require "#{CCI_SRC}/lib/compare_data_format.rb" UPSTREAM_REPOS_PATH = ENV['UPSTREAM_REPOS_PATH'] || '/c/upstream-repos' @@ -548,3 +549,69 @@ def new_refs_statistics(params) end [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] end + + +# ------------------------------------------------------------------------------------------- +# job error table like: +# job_id error_id error_message link to result +# ------------------------------------------------------------------------------------- +# crystal.630608 "stderr.xxx" "messag:xxxx" https://$host:$port/$result_root +# ... +# ------------------------------------------------------------------------------------------- + +def get_job_error(params) + begin + body = job_error_body(params) + rescue StandardError => e + warn e.message + return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'get error table error'] + end + + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] +end + +def job_error_body(params) + error_objects = get_error_objects(params) + { + filter: params, + attributes: ['job_id', 'error_id', 'error_message', 'link_to_result'], + objects: error_objects, + }.to_json +end + +def get_error_objects(filter_items) + error_objs = [] + + job_list = get_job_list(filter_items) + job_list.each do |job| + error_obj = get_error_from_job(job) + error_objs << error_obj unless error_obj.empty? + end + + error_objs +end + +def get_job_list(items) + es = ESQuery.new(ES_HOST, ES_PORT) + query_results = es.multi_field_query(items) + + extract_jobs_list(query_results['hits']['hits']) +end + +# get all error_id from one job +def get_error_from_job(job) + job_error_obj = {} + job['stats'].each do |metric, value| + next unless metric.end_with?('.message') + + result_host = ENV['SRV_HTTP_RESULT_HOST'] || SRV_HTTP_RESULT_HOST + result_port = ENV['SRV_HTTP_RESULT_PORT'] || SRV_HTTP_RESULT_PORT + error_id = metric.sub('.message', '.fail') + job_error_obj['id'] = job['id'] + job_error_obj['error_id'] = error_id + job_error_obj['error_message'] = value + job_error_obj['link_to_result'] = value + end + + job_error_obj +end -- 2.23.0

1 0

[PATCH v2 compass-ci] providers/qemu/kvm.sh: added QEMU multiple network card functionality
by Wang Chenglong 22 Mar '21

22 Mar '21

[How] Use the 'nr_nic' to set the number of network cards in yaml. nr_nic: 2 root@vm-2p8g ~# ifconfig enp0s1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 172.18.42.60 netmask 255.255.0.0 broadcast 172.18.255.255 inet6 fe80::843:24ff:fef2:ec17 prefixlen 64 scopeid 0x20<link> ether 0a:43:24:f2:ec:17 txqueuelen 1000 (Ethernet) RX packets 155492 bytes 1172898602 (1.0 GiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 75 bytes 11207 (10.9 KiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 enp0s2: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 172.18.88.228 netmask 255.255.0.0 broadcast 172.18.255.255 inet6 fe80::820:70ff:fe34:d27b prefixlen 64 scopeid 0x20<link> ether 0a:20:70:34:d2:7b txqueuelen 1000 (Ethernet) RX packets 168 bytes 9079 (8.8 KiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 199208 bytes 32959935 (31.4 MiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- providers/qemu/kvm.sh | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index dcab81a..a1c0046 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -119,9 +119,34 @@ add_disk() done } +set_mac() +{ + job_id=$(awk -F'/' '/job_initrd_tmpfs/{print $(NF-1)}' $ipxe_script) + + if [ $(command -v es-find) ]; then + nr_nic=$(es-find id=$job_id | awk '/nr_nic/{print $2}' | tr -d ,) + else + echo "command not found: es-find. set nr_nic=1" + sleep 1 + fi + + mac_arr[1]=$mac + nr_nic=${nr_nic:-1} + + [ "$nr_nic" -ge "2" ] || return + for i in $(seq 2 $nr_nic) + do + mac=$(echo $hostname$i | md5sum | sed 's/^$..$$..$$..$$..$$..$.*$/0a-\1-\2-\3-\4-\5/') + mac_arr[$i]=$mac + done +} + set_nic() { - nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" + for i in $(seq 1 $nr_nic) + do + nic[$i]="-nic tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac_arr[$i]}" + done } set_device() @@ -190,7 +215,7 @@ individual_option() -machine virt-4.0,accel=kvm,gic-version=3 -cpu Kunpeng-920 -bios $bios - -nic $nic + ${nic[@]} ) ;; qemu-kvm) @@ -198,17 +223,17 @@ individual_option() -machine virt-4.0,accel=kvm,gic-version=3 -cpu Kunpeng-920 -bios $bios - -nic $nic + ${nic[@]} ) [ "$(arch)" == "x86_64" ] && arch_option=( -bios $bios - -nic $nic + ${nic[@]} ) ;; qemu-system-x86_64) arch_option=( -bios $bios - -nic $nic + ${nic[@]} ) ;; qemu-system-riscv64) @@ -234,6 +259,7 @@ set_options() { set_bios set_helper + set_mac set_nic set_device set_netdev -- 2.23.0

1 0