- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci 3/8] container/mail-robot: apply-account-component.rb
by Luan Shengde 04 Nov '20

04 Nov '20

ParseApplyAccountEmail parse commit url and pub_key check out commit url check commit url exists check base url in upstream-repos check commit available gitee.com clone the repo and check the commit exists non-gitee.com check the feedback of curl command to check the commit url available check out pub_key extract the attachment of the pub_key ApplyAccount apply account with apply_account_info check account SendMail send uuid email build email message send email send error email build error email message send error email Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- .../mail-robot/apply-account-component.rb | 232 ++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100755 container/mail-robot/apply-account-component.rb diff --git a/container/mail-robot/apply-account-component.rb b/container/mail-robot/apply-account-component.rb new file mode 100755 index 0000000..98272e7 --- /dev/null +++ b/container/mail-robot/apply-account-component.rb @@ -0,0 +1,232 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'mail' + +# check whether there is a commit url and ssh pub_key in the email +# apply account on junper server +class ParseApplyAccountEmail + def initialize(mail_content) + @mail_content = mail_content + @my_email = mail_content.from[0] + @my_name = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') + end + + def parse_commit_url_pub_key + commit_url = check_out_commit_url + pub_key = check_out_pub_key + + return commit_url, pub_key + end + + def send_error_email(error_message) + send_error_email_info = { + 'my_email' => @my_email, + 'my_name' => @my_name, + 'my_info' => '', + 'error_message' => error_message + } + send_err_mail = SendMail.new(send_error_email_info) + send_err_mail.send_error_email + end + + def extract_commit_url + mail_content_body = @mail_content.part[0].part[0].body.decoded || @mail_content.part[0].body.decoded + mail_content_line = mail_content_body.gsub(/\n/, '') + no_commit_url unless mail_content_line.match?(%r{my oss commit:\s*https?://[^/]*/[^/]*/[^/]*/commit/[\w\d]{40}}) + + mail_content_body.match(%r{https?://[^/]*/[^/]*/[^/]*/commit/[\w\d]{40}})[0] + end + + def no_commit_url + error_message = 'No matched commit url found' + send_error_email(error_message) + + raise error_message + end + + def check_out_commit_url + url = extract_commit_url + base_url = url.gsub(%r{/commit/[\w\d]{40}$}, '') + + check_base_url_in_upstream_repos('/c/upstream-repos', base_url) + check_commit_available(url, base_url) + + return url + end + + def check_base_url_in_upstream_repos(upstream_dir, base_url) + Dir.chdir(upstream_dir) + match_out = %x(grep -rn #{base_url}) + + return unless match_out.empty? + + error_message = 'The url is not in upstream-repo list' + send_error_email(error_message) + + raise error_message + end + + def check_commit_available(url, base_url) + hub_name = url.split('/')[2] + + # it requires authentication when execute curl to get the commit infomation + # clone the repo and then validate the commit for the email address + if hub_name.eql? 'gitee.com' + check_gitee_commit(url, base_url) + else + check_non_gitee_commit(url) + end + end + + def check_non_gitee_commit(url) + url_fdback = %x(curl #{url}) + email_index = url_fdback.index @my_email + + return unless email_index.nil? + + error_message = 'Your commit url is not a valid commit url.' + send_error_email(error_message) + + raise error_message + end + + def check_gitee_commit(url, base_url) + repo_dir = url.split('/')[-3] + repo_url = [base_url, 'git'].join('.') + commit_id = url.split('/')[-1] + + Dir.chdir ENV['ROBOT_TMP_REPOS_DIR'] + %x(/usr/bin/git clone #{repo_url} #{repo_dir}) + email_index = %x(/usr/bin/git -C #{repo_dir} show #{commit_id}).index @my_email + + FileUtils.rm_rf repo_dir + + check_gitee_commit_exist(email_index) + end + + def check_gitee_commit_exist(email_index) + return unless email_index.nil? + + error_message = 'Your commit url is not a valid commit url.' + send_error_email(error_message) + + raise error_message + end + + def check_out_pub_key + pub_key = @mail_content.part[1].body.decoded if @mail_content.part[1].filename == 'id_rsa.pub' + check_pub_key(pub_key) + + return pub_key + end + + def check_pub_key(pub_key) + return unless pub_key.nil? + + error_message = 'No pub_key found' + send_error_email(error_message) + + raise error_message + end +end + +# apply jumper account for user +class ApplyAccount + def initialize(apply_account_info) + @apply_account_info = apply_account_info + end + + def apply_jumper_account + account_info_str = %x(curl -XGET "#{ENV['JUMPER_HOST']}:#{ENV['JUMPER_PORT']}/assign_account" \ + -d '#{(a)apply_account_info.to_json}') + account_info = JSON.parse account_info_str + + check_account_info(account_info) + + return account_info + end + + def check_account_info(account_info) + return unless account_info['account'].nil? + + error_message = 'No more available jumper account.' + send_error_email(error_message) + + raise error_message + end +end + +# build mail data and send mail +class SendMail + def initialize(send_mail_info) + @send_mail_host = %x(/sbin/ip route |awk '/default/ {print $3}').chomp + @my_email = send_mail_info['my_email'] + @my_name = send_mail_info['my_name'] + @error_message = send_mail_info['error_message'] + @my_info = send_mail_info['my_info'] + end + + def send_uuid_email + email_msg = <<~EMAIL_MESSAGE + To: #{@my_email} + Subject: [compass-ci] Account Ready + + Dear #{@my_name}, + + Thank you for joining us. + You can use the following info to submit jobs: + + 1) setup default config + run the following command to add the below setup to default config file + mkdir -p ~/.config/compass-ci/defaults/ + cat >> ~/.config/compass-ci/defaults/\\${USER}.yaml <<-EOF + my_uuid: #{@my_info['my_uuid']} + my_email: #{@my_info['my_email']} + my_name: #{@my_info['my_name']} + EOF + + 2) download lkp-tests and dependencies + run the following command to install the lkp-test and effect the configuration + git clone https://gitee.com/wu_fengguang/lkp-tests.git + cd lkp-tests + make install + source ${HOME}/.bashrc && source ${HOME}/.bash_profile + + 3) submit job + reference: https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/tutorial.md + find how to write job yaml + submit jobs/netperf.yaml + + regards + compass-ci + EMAIL_MESSAGE + + send_mail(email_msg) + end + + def send_error_email + email_msg = <<~EMAIL_MESSAGE + To: #{@my_email} + Subject: apply account failed + + Dear #{@my_name} + + Your application for account failed with following errors + + #{@error_message} + + regards + compass-ci + EMAIL_MESSAGE + + send_mail(email_msg) + end + + def send_mail(email_msg) + %x(curl -XPOST "#{@send_mail_host}:#{ENV['SEND_MAIL_PORT']}/send_mail_text" -d "#{email_msg}") + end +end -- 2.23.0

2 8

[PATCH v2 compass-ci] container/sub-fluentd: fix heartbeat error
by Wu Zhende 04 Nov '20

04 Nov '20

[Error] fluent.warn: Unknown heartbeat response received from 172.17.0.1:24224. It may service out heartbeat_type default is tcp, delete the "heartbeat_type udp" to use tcp Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/sub-fluentd/docker-fluentd.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/container/sub-fluentd/docker-fluentd.conf b/container/sub-fluentd/docker-fluentd.conf index c674fb1..09fa461 100644 --- a/container/sub-fluentd/docker-fluentd.conf +++ b/container/sub-fluentd/docker-fluentd.conf @@ -56,7 +56,6 @@ @type forward flush_interval 0 send_timeout 60 - heartbeat_type udp heartbeat_interval 1 recover_wait 10 hard_timeout 60 -- 2.23.0

2 1

[PATCH v9 compass-ci 2/2] kernel_version.md: explain key "kernel_version"
by Xu Xijian 04 Nov '20

04 Nov '20

[why] Explain the meaning of new key "kernel_version" for scheduler, including what's the mapping between its typical values and actual files in the disk and how to set it. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- doc/job/kernel_version.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 doc/job/kernel_version.md diff --git a/doc/job/kernel_version.md b/doc/job/kernel_version.md new file mode 100644 index 0000000..f6a5f4d --- /dev/null +++ b/doc/job/kernel_version.md @@ -0,0 +1,39 @@ +# kernel_version + +Meaning: +- Every os has its kernel, however an os can start with different kernels according to different need. +- kernel_version is a key for users to specify a kernel version. +- If kernel_version is not given by users, it will use the default one. +- Here are the kernel versions we support for different os: + openeuler/aarch64/1.0 + - 4.19.90-2001(default) + + openeuler/aarch64/20.03 + - 4.19.90-2003(default) + - 4.19.90-mysql + - 4.19.90-nginx + + debian/aarch64/sid + - 5.4.0-4-arm64(default) + - 5.8.0-1-arm64 + + centos/aarch64/7.6.1810 + - 4.14.0-115.el7a.0.1(default) + + centos/aarch64/7.8.2003 + - 4.18.0-147.8.1.el7(default) + + centos/aarch64/8.1.1911 + - 4.18.0-147.el8(default) + +Related files: +- In initramfs boot process, every kernel version is related with a vmlinuz, module and headers. +- Files like below under $boot_dir, an example $boot_dir can be "/srv/os/openeuler/aarch64/20.03/boot". +├── headers-4.19.90-2003.cgz +├── headers.cgz -> headers-4.19.90-2003.cgz +├── modules-4.19.90-2003.cgz +├── modules.cgz -> modules-4.19.90-2003.cgz +├── vmlinuz-4.19.90-2003 + +Usage example: +- submit iperf.yaml testbox=vm-2p8g--$USER os=openeuler os_arch=aarch64 os_version=20.03 runtime=20 kernel_version=4.19.90-2003 -- 2.23.0

4 4

[PATCH compass-ci] container/master-fluentd: get logging-es info from yaml
by Wu Zhende 04 Nov '20

04 Nov '20

[Why] "logging-es" and "master-fluentd" may be on different services and may be performed over the network. In this scenario, the logging-es information needs to be obtained from the configuretion file. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/master-fluentd/docker-fluentd.conf | 4 ++-- container/master-fluentd/start | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/container/master-fluentd/docker-fluentd.conf b/container/master-fluentd/docker-fluentd.conf index 4660988..a57d4cd 100644 --- a/container/master-fluentd/docker-fluentd.conf +++ b/container/master-fluentd/docker-fluentd.conf @@ -40,8 +40,8 @@ <store> @type elasticsearch - host 172.17.0.1 - port 9202 + host "#{ENV['LOGGING_ES_HOST']}" + port "#{ENV['LOGGING_ES_PORT']}" suppress_type_name true flush_interval 1s index_name ${tag} diff --git a/container/master-fluentd/start b/container/master-fluentd/start index b3e3dbb..a05881c 100755 --- a/container/master-fluentd/start +++ b/container/master-fluentd/start @@ -3,9 +3,20 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig.rb' -docker_rm "master-fluentd" +names = Set.new %w[ + LOGGING_ES_HOST + LOGGING_ES_PORT +] + +defaults = relevant_defaults(names) +defaults['LOGGING_ES_HOST'] ||= '172.17.0.1' +defaults['LOGGING_ES_PORT'] ||= '9202' +env = docker_env(defaults) + +docker_rm 'master-fluentd' cmd = %w[ docker run @@ -13,6 +24,7 @@ cmd = %w[ --name master-fluentd -v /etc/localtime:/etc/localtime:ro -d +] + env + %w[ -u 1090:1090 -p 24224:24224/tcp -p 24224:24224/udp -- 2.23.0

2 1

[PATCH v2 lab-z9] hosts: fix taishan200-2280-2s48p-256g--a102 to taishan200-2280-2s48p-512g--a102
by Zhang Yu 04 Nov '20

04 Nov '20

[why] The memory of taishan200-2280-2s48p-256g--a102 is 512g, so need to fix 256g to 512g. Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- ...200-2280-2s48p-256g--a102 => taishan200-2280-2s48p-512g--a102} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename hosts/{taishan200-2280-2s48p-256g--a102 => taishan200-2280-2s48p-512g--a102} (100%) diff --git a/hosts/taishan200-2280-2s48p-256g--a102 b/hosts/taishan200-2280-2s48p-512g--a102 similarity index 100% rename from hosts/taishan200-2280-2s48p-256g--a102 rename to hosts/taishan200-2280-2s48p-512g--a102 -- 2.23.0

2 2

[PATCH lkp-tests 1/2] lib/job.rb: no need puts the a warning
by Li Ping 04 Nov '20

04 Nov '20

[why] we can get hostname from the repo lab-z9 or get hostname from lkp-tests/hosts --- lib/job.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/job.rb b/lib/job.rb index 5c2c9517..86cf3fa1 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -340,10 +340,8 @@ class Job lab_hosts_file = get_lab_hosts_file if lab_hosts_file hosts_file = lab_hosts_file - end - - if File.file?(hosts_file) - hosts_file + elsif File.file?(hosts_file) + hosts_file else puts("hosts_file not exist: #{hosts_file}, maybe need check testbox field") nil -- 2.23.0

1 0

[PATCH lkp-tests] libmicro: add PKGBUILD for libmicro
by Bai Jing 04 Nov '20

04 Nov '20

Signed-off-by: Bai Jing <799286817(a)qq.com> --- pkg/libmicro/PKGBUILD | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 pkg/libmicro/PKGBUILD diff --git a/pkg/libmicro/PKGBUILD b/pkg/libmicro/PKGBUILD new file mode 100644 index 00000000..e421c402 --- /dev/null +++ b/pkg/libmicro/PKGBUILD @@ -0,0 +1,20 @@ +pkgname=libMicro +pkgver=0.4 +pkgrel=0 +pkgdesc="A portable set of microbenchmarks focused on the syscall and system library interfaces." +arch=('i386' 'x86_64') +url="https://github.com/rzezeski/libMicro" +license=('CDDL') +source=("https://codeload.github.com/redhat-performance/libMicro/zip/0.4.0-rh") +md5sums=('SKIP') + + +build() { + cd "$srcdir/$pkgname-$pkgver.$pkgrel-rh" + make +} + +package() { + mkdir -p "$pkgdir/lkp/benchmarks/libMicro" + cp -r "$srcdir/$pkgname-$pkgver.$pkgrel-rh/"* "$pkgdir/lkp/benchmarks/libMicro" +} -- 2.23.0

2 2

[PATCH compass-ci] container/sub-fluentd: fix heartbeat error
by Wu Zhende 04 Nov '20

04 Nov '20

[Error] fluent.warn: Unknown heartbeat response received from 172.17.0.1:24224. It may service out Can't use udp to send heartbeat package. Change to tcp. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/sub-fluentd/docker-fluentd.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/container/sub-fluentd/docker-fluentd.conf b/container/sub-fluentd/docker-fluentd.conf index c674fb1..09fa461 100644 --- a/container/sub-fluentd/docker-fluentd.conf +++ b/container/sub-fluentd/docker-fluentd.conf @@ -56,7 +56,6 @@ @type forward flush_interval 0 send_timeout 60 - heartbeat_type udp heartbeat_interval 1 recover_wait 10 hard_timeout 60 -- 2.23.0

3 2

[PATCH v5 lkp-tests] tests/build-pkg: modify save location for *.cgz file
by Liu Shaofei 04 Nov '20

04 Nov '20

[why] the path of *.cgz file generated by build-pkg script cannot be obtained during test. So the path of cgz file need to be changed. [how] below is example for openeuler: latest.cgz -> f71820d6055373c9b9381615e28a28f307bb2df4.cgz f71820d6055373c9b9381615e28a28f307bb2df4.cgz v1.8.1.cgz -> 92c89f904a14d4bfc25abcd2c3d5c2da35a589d6.cgz 92c89f904a14d4bfc25abcd2c3d5c2da35a589d6.cgz v1.5.1.cgz -> df0584e46db9e7b412626deab10b923504745b2c.cgz df0584e46db9e7b412626deab10b923504745b2c.cgz Signed-off-by: Liu Shaofei <liushaofei5(a)huawei.com> --- tests/build-pkg | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/build-pkg b/tests/build-pkg index b10b1497..7c494a15 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -2,10 +2,12 @@ # - os # - os_arch # - os_version +# - os_mount # - pkgbuild_repo # - upstream_repo # - upstream_url # - upstream_commit +# - upstream_tag ## makepkg is a script that automates the building of packages; it can download and validate source files, ## check dependencies, configure build-time settings, compile the sources, install into a temporary root, @@ -21,15 +23,19 @@ check_vars() [ -n "$os_version" ] || die "os_version is empty" [ -n "$pkgbuild_repo" ] || die "pkgbuild_repo is empty" [ -n "$upstream_commit" ] || die "upstream_commit is empty" + [ -n "$os_mount" ] || die "os_mount is empty" } mount_dest() { + [[ "$os_mount" = "cifs" ]] && os_mount="nfs" + pack_to=${os_mount}/${os}/${os_arch}/${os_version} + PKG_MNT=/initrd/build-pkg mkdir -p "$PKG_MNT" [ -n "$LKP_SERVER" ] && { - mount $LKP_SERVER:$PKG_MNT $PKG_MNT || die "Failed to run mount" + mount -t cifs -o guest,vers=1.0,noacl,nouser_xattr //$LKP_SERVER$PKG_MNT $PKG_MNT || die "Failed to run mount" } } @@ -75,12 +81,22 @@ build_source_pkg() sed -i "s|^source=.*|${upstream_source}|g" PKGBUILD } - cgz_name="$PKG_MNT/${pkgname}/${upstream_commit}.cgz" + cgz_name="$PKG_MNT/${pack_to}/${pkgname}/${upstream_commit}.cgz" PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ - $LKP_SRC/sbin/makepkg -A --check --config $LKP_SRC/etc/makepkg.conf 2>&1 + $LKP_SRC/sbin/makepkg -A --check --skippgpcheck --config $LKP_SRC/etc/makepkg.conf 2>&1 +} + +create_softlink() +{ + [ -n "$upstream_tag" ] || return 0 + [ -e "$PKG_MNT/${pack_to}/${pkgname}/${upstream_tag}.cgz" ] && return 0 + + ln -sf "$(basename $(realpath ${cgz_name}))" "$PKG_MNT/${pack_to}/${pkgname}/${upstream_tag}.cgz" || return 0 + echo "create soft link: $PKG_MNT/${pack_to}/${pkgname}/${upstream_tag}.cgz -> $(basename $(realpath ${cgz_name}))" } check_vars mount_dest request_pkg build_source_pkg +create_softlink -- 2.23.0

4 5

[PATCH v3 compass-ci] sparrow/3-code: add ENV LOGGING_ES_HOST and LOGGING_ES_PORT
by Liu Yinsi 04 Nov '20

04 Nov '20

need LOGGING_ES_HOST and LOGGING_ES_PORT to start logging-es service. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 64f34f8..f4e4fa9 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -13,6 +13,8 @@ server_ip=$(ip route get 1.2.3.4 | awk '{print $7; exit}') : ${MASTER_FLUENTD_PORT:=24224} : ${DOCKER_REGISTRY_HOST:=$server_ip} : ${DOCKER_REGISTRY_PORT:=5001} +: ${LOGGING_ES_HOST:=$server_ip} +: ${LOGGING_ES_PORT:=9202} mkdir -p /etc/compass-ci/defaults cat > /etc/compass-ci/defaults/$server_name.yaml <<EOF @@ -28,9 +30,11 @@ MASTER_FLUENTD_HOST: $MASTER_FLUENTD_HOST MASTER_FLUENTD_PORT: $MASTER_FLUENTD_PORT DOCKER_REGISTRY_HOST: $DOCKER_REGISTRY_HOST DOCKER_REGISTRY_PORT: $DOCKER_REGISTRY_PORT +LOGGING_ES_HOST: $LOGGING_ES_HOST +LOGGING_ES_PORT: $LOGGING_ES_PORT EOF -cat > /etc/profile.d/compass.sh <<'EOF' +cat > /etc/profile.d/compass.sh <<EOF export LKP_SRC=/c/lkp-tests export CCI_SRC=/c/compass-ci export REPO_SRC=/c/upstream-repos -- 2.23.0

1 0