- Compass-ci - mailweb.openeuler.org

[PATCH lkp-tests] tests/build-pkg: unified indent
by Lin Jiaxin 02 Dec '20

02 Dec '20

Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- tests/build-pkg | 124 ++++++++++++++++++++++++------------------------ 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/tests/build-pkg b/tests/build-pkg index bdaef035..5e7e62d4 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -21,60 +21,60 @@ check_vars() { - [ -n "$os" ] || die "os is empty" - [ -n "$os_arch" ] || die "os_arch is empty" - [ -n "$os_version" ] || die "os_version is empty" - [ -n "$pkgbuild_repo" ] || die "pkgbuild_repo is empty" - [ -n "$upstream_commit" ] || die "upstream_commit is empty" - [ -n "$os_mount" ] || die "os_mount is empty" + [ -n "$os" ] || die "os is empty" + [ -n "$os_arch" ] || die "os_arch is empty" + [ -n "$os_version" ] || die "os_version is empty" + [ -n "$pkgbuild_repo" ] || die "pkgbuild_repo is empty" + [ -n "$upstream_commit" ] || die "upstream_commit is empty" + [ -n "$os_mount" ] || die "os_mount is empty" } mount_dest() { - # the same image is mounted to cifs and nfs, the generated cgz files - # are stored in the nfs directory. - [[ "$os_mount" = "cifs" ]] && os_mount="nfs" - pack_to=${os_mount}/${os}/${os_arch}/${os_version} + # the same image is mounted to cifs and nfs, the generated cgz files + # are stored in the nfs directory. + [[ "$os_mount" = "cifs" ]] && os_mount="nfs" + pack_to=${os_mount}/${os}/${os_arch}/${os_version} - PKG_MNT=/initrd/build-pkg - mkdir -p "$PKG_MNT" + PKG_MNT=/initrd/build-pkg + mkdir -p "$PKG_MNT" - [ -n "$LKP_SERVER" ] && { - mount -t cifs -o guest,vers=1.0,noacl,nouser_xattr,port=446 //$LKP_SERVER$PKG_MNT $PKG_MNT || die "Failed to run mount" - } + [ -n "$LKP_SERVER" ] && { + mount -t cifs -o guest,vers=1.0,noacl,nouser_xattr,port=446 //$LKP_SERVER$PKG_MNT $PKG_MNT || die "Failed to run mount" + } } get_pkgfile() { - curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' \ - -d '{"git_repo": "'$1'", "git_command": ["git-show", "HEAD:'$2'"]}' -o "${2##*/}" + curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' \ + -d '{"git_repo": "'$1'", "git_command": ["git-show", "HEAD:'$2'"]}' -o "${2##*/}" } request_pkg() { - local pkgrepo="${pkgbuild_repo%%//*}" - local pkgpath="./" - [[ "$pkgrepo" =~ ^archlinux/(packages|community) ]] && pkgpath="${pkgpath}${pkgbuild_repo#*//}/" - - filelist=$(curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${pkgrepo}'", "git_command": ["git-ls-files", "'${pkgpath}'"]}') - - for pkgfile in ${filelist[*]} - do - get_pkgfile "${pkgrepo}" "$pkgfile" - done - - # download PKGBUILD-$tag - benchmark="${upstream_repo##*/}" - benchmark="${benchmark%.git}" - [ -n "$PKGBUILD_TAG" ] && { - get_pkgfile "pkg/$benchmark" "PKGBUILD-$PKGBUILD_TAG" - [ -f "PKGBUILD-$PKGBUILD_TAG" ] || die "PKGBUILD-$PKGBUILD_TAG not exist" - [ -s "PKGBUILD-$PKGBUILD_TAG" ] || die "PKGBUILD-$PKGBUILD_TAG is empty" - } - - [ -f PKGBUILD ] || die "PKGBUILD not exist" - [ -s PKGBUILD ] || die "PKGBUILD is empty" + local pkgrepo="${pkgbuild_repo%%//*}" + local pkgpath="./" + [[ "$pkgrepo" =~ ^archlinux/(packages|community) ]] && pkgpath="${pkgpath}${pkgbuild_repo#*//}/" + + filelist=$(curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' \ + -d '{"git_repo": "'${pkgrepo}'", "git_command": ["git-ls-files", "'${pkgpath}'"]}') + + for pkgfile in ${filelist[*]} + do + get_pkgfile "${pkgrepo}" "$pkgfile" + done + + # download PKGBUILD-$tag + benchmark="${upstream_repo##*/}" + benchmark="${benchmark%.git}" + [ -n "$PKGBUILD_TAG" ] && { + get_pkgfile "pkg/$benchmark" "PKGBUILD-$PKGBUILD_TAG" + [ -f "PKGBUILD-$PKGBUILD_TAG" ] || die "PKGBUILD-$PKGBUILD_TAG not exist" + [ -s "PKGBUILD-$PKGBUILD_TAG" ] || die "PKGBUILD-$PKGBUILD_TAG is empty" + } + + [ -f PKGBUILD ] || die "PKGBUILD not exist" + [ -s PKGBUILD ] || die "PKGBUILD is empty" } create_softlink() @@ -95,32 +95,32 @@ create_softlink() replace_source() { - sed -i "s|^source=[^)]*|$1|g" "$2" + sed -i "s|^source=[^)]*|$1|g" "$2" } build_source_pkg() { - local repo_dir="" - source PKGBUILD - [ -n "$upstream_repo" ] && [ -n "$upstream_url" ] && { - - if [[ "$source" = *::* ]]; then - repo_dir="${source%%::*}::" - else - repo_dir="$(basename ${upstream_url})" - repo_dir="${repo_dir%.git*}::" - fi - - upstream_source="source=(\"${repo_dir}git://${LKP_SERVER}/${upstream_repo}#commit=${upstream_commit}\"" - replace_source "${upstream_source}" PKGBUILD - [ -n "$PKGBUILD_TAG" ] && replace_source "${upstream_source}" "PKGBUILD-$PKGBUILD_TAG" - } - - cgz_name="$PKG_MNT/${pack_to}/${pkgname}/${upstream_commit}.cgz" - PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ - $LKP_SRC/sbin/makepkg -A --check --skippgpcheck --config $LKP_SRC/etc/makepkg.conf 2>&1 - - [ "$?" == 0 ] && create_softlink + local repo_dir="" + source PKGBUILD + [ -n "$upstream_repo" ] && [ -n "$upstream_url" ] && { + + if [[ "$source" = *::* ]]; then + repo_dir="${source%%::*}::" + else + repo_dir="$(basename ${upstream_url})" + repo_dir="${repo_dir%.git*}::" + fi + + upstream_source="source=(\"${repo_dir}git://${LKP_SERVER}/${upstream_repo}#commit=${upstream_commit}\"" + replace_source "${upstream_source}" PKGBUILD + [ -n "$PKGBUILD_TAG" ] && replace_source "${upstream_source}" "PKGBUILD-$PKGBUILD_TAG" + } + + cgz_name="$PKG_MNT/${pack_to}/${pkgname}/${upstream_commit}.cgz" + PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ + $LKP_SRC/sbin/makepkg -A --check --skippgpcheck --config $LKP_SRC/etc/makepkg.conf 2>&1 + + [ "$?" == 0 ] && create_softlink } check_vars -- 2.23.0

1 0

[PATCH v4 compass-ci] lib: add error_messages.rb
by Lin Jiaxin 02 Dec '20

02 Dec '20

provide error messages for git bisect Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- lib/error_messages.rb | 73 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 lib/error_messages.rb diff --git a/lib/error_messages.rb b/lib/error_messages.rb new file mode 100644 index 0000000..7bf3aa8 --- /dev/null +++ b/lib/error_messages.rb @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' + +# usage: +# foo = ErrorMessages.new(build-pkg) +# build-pkg is a file that includes errors generated by make +# error_messages_hash = foo.obtain_error_messages +# error_messages_hash.each { |k,v| puts k; puts; v.each {|m| puts m } } +# output: +# src/arg.c:117:24: warning: comparison between signed and unsigned integer expressions [-Wsign-compare] +# +# src/arg.c:117:24: warning: comparison between signed and unsigned integer expressions [-Wsign-compare] +# for (nbarg = 0; nbarg < ARGM_NBARGS && ((mask >> (nbarg * ARGT_BITS)) & ARGT_MASK); nbarg++); +# ^ +class ErrorMessages + def initialize(file) + @file = file + @seqno = '' + @in_stderr = false + @error_line = '' + @error_details = '' + @error_message = {} + @error_messages = Hash.new { |h, k| h[k] = Set.new } + end + + def obtain_error_messages + File.open((a)file).each_line do |line| + next if extract_error_message(line) + next unless @in_stderr + next unless @error_message['error_line'] =~ /error|warning/i + + @error_messages[@error_message['error_line']] << @error_message['error_line'] + @error_message['error_details'] + end + return @error_messages + end + + private + + def extract_error_message(line) + if line =~ /^ / + @error_details += line + else + update_error_message(line) + case line + when /^## ______________([0-9.]+):stderr$/ + @in_stderr = true + @seqno = $1 + return true + + when /^## ______________#{@seqno}:enderr$/ + @in_stderr = false + @seqno = '' + add_error_message + end + end + end + + def update_error_message(line) + @error_message['error_line'] = @error_line + @error_message['error_details'] = @error_details + @error_line = line + @error_details = '' + end + + def add_error_message + return unless @error_message['error_line'] =~ /error|warning/i + + @error_messages[@error_message['error_line']] << @error_message['error_line'] + @error_message['error_details'] + end +end -- 2.23.0

1 0

[PATCH v2 compass-ci] scheduler: fix the values diplayed in job.yaml
by Bai Jing 02 Dec '20

02 Dec '20

[original values] initrds_uri: 'initrd http://172.168.131.113:8800/initrd/osimage/openeuler/aarch64/20.03/20201022… initrd http://172.168.131.113:8800/initrd/deps/nfs/debian/aarch64/sid/run-ipconfig… initrd http://172.17.0.1:8000/os/openeuler/aarch64/20.03/boot/modules-4.19.90-2003… initrd http://172.17.0.1:8000/os/openeuler/aarch64/20.03/boot/headers-4.19.90-2003… initrd http://172.168.131.113:8800/initrd/deps/initramfs/openeuler/aarch64/20.03/m… initrd http://172.168.131.113:8800/initrd/deps/initramfs/openeuler/aarch64/20.03/s… initrd http://172.168.131.113:8800/initrd/pkg/initramfs/openeuler/aarch64/20.03/mp… initrd http://172.168.131.113:8800/initrd/pkg/initramfs/openeuler/aarch64/20.03/sp… initrd http://172.168.131.113:8800/initrd/lkp/baijing/lkp-aarch64.cgz initrd http://172.168.131.113:3000/job_initrd_tmpfs/crystal.146511/job.cgz ' [now values] initrds_uri: - initrd http://172.168.131.113:8800/initrd/osimage/openeuler/aarch64/20.03/20201022… - initrd http://172.168.131.113:8800/initrd/deps/nfs/debian/aarch64/sid/run-ipconfig… - initrd http://172.17.0.1:8000/os/openeuler/aarch64/20.03/boot/modules-4.19.90-2003… - initrd http://172.17.0.1:8000/os/openeuler/aarch64/20.03/boot/headers-4.19.90-2003… - initrd http://172.168.131.113:8800/initrd/deps/initramfs/openeuler/aarch64/20.03/m… - initrd http://172.168.131.113:8800/initrd/deps/initramfs/openeuler/aarch64/20.03/s… - initrd http://172.168.131.113:8800/initrd/pkg/initramfs/openeuler/aarch64/20.03/mp… - initrd http://172.168.131.113:8800/initrd/pkg/initramfs/openeuler/aarch64/20.03/sp… - initrd http://172.168.131.113:8800/initrd/lkp/baijing/lkp-aarch64.cgz - initrd http://172.168.131.113:3087/job_initrd_tmpfs/crystal.147702/job.cgz Signed-off-by: Bai Jing <799286817(a)qq.com> --- src/scheduler/jobfile_operate.cr | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/scheduler/jobfile_operate.cr b/src/scheduler/jobfile_operate.cr index 98bf011..7ec3ea1 100644 --- a/src/scheduler/jobfile_operate.cr +++ b/src/scheduler/jobfile_operate.cr @@ -76,8 +76,24 @@ module Jobfile::Operate script_lines = JSON.parse(script_lines) end + def self.handle_job_field(job_content, fields) + fields.each do |field| + case field + when "initrds_uri" + values = job_content[field].to_s.split("\n").map(&.strip).reject!(&.empty?) + values = values.map { |value| JSON::Any.new(value) } + job_content[field] = JSON::Any.new(values) + end + end + + return job_content + end + def self.create_job_cpio(job_content : JSON::Any, base_dir : String) job_content = job_content.as_h + + need_to_handle = ["initrds_uri"] + tmp_job_content = handle_job_field(job_content.clone, need_to_handle) # put job2sh in an array if job_content.has_key?("job2sh") @@ -100,8 +116,8 @@ module Jobfile::Operate # - this will create a <'#! jobs/pixz.yaml':> in the yaml file # - but the orange is <#! jobs/pixz.yaml> in the user job.yaml # tested : no effect to job.sh - File.open(temp_yaml, "w") do |file| - YAML.dump(job_content, file) + File.open(temp_yaml, "a") do |file| + YAML.dump(tmp_job_content, file) end # generate unbroken job shell content -- 2.23.0

1 0

[PATCH compass-ci] scheduler: fix the values diplayed in job.yaml
by Bai Jing 02 Dec '20

02 Dec '20

[original values] initrds_uri: 'initrd http://172.168.131.113:8800/initrd/osimage/openeuler/aarch64/20.03/20201022… initrd http://172.168.131.113:8800/initrd/deps/nfs/debian/aarch64/sid/run-ipconfig… initrd http://172.17.0.1:8000/os/openeuler/aarch64/20.03/boot/modules-4.19.90-2003… initrd http://172.17.0.1:8000/os/openeuler/aarch64/20.03/boot/headers-4.19.90-2003… initrd http://172.168.131.113:8800/initrd/deps/initramfs/openeuler/aarch64/20.03/m… initrd http://172.168.131.113:8800/initrd/deps/initramfs/openeuler/aarch64/20.03/s… initrd http://172.168.131.113:8800/initrd/pkg/initramfs/openeuler/aarch64/20.03/mp… initrd http://172.168.131.113:8800/initrd/pkg/initramfs/openeuler/aarch64/20.03/sp… initrd http://172.168.131.113:8800/initrd/lkp/baijing/lkp-aarch64.cgz initrd http://172.168.131.113:3000/job_initrd_tmpfs/crystal.146511/job.cgz ' [now values] initrds_uri: - initrd http://172.168.131.113:8800/initrd/osimage/openeuler/aarch64/20.03/20201022… - initrd http://172.168.131.113:8800/initrd/deps/nfs/debian/aarch64/sid/run-ipconfig… - initrd http://172.17.0.1:8000/os/openeuler/aarch64/20.03/boot/modules-4.19.90-2003… - initrd http://172.17.0.1:8000/os/openeuler/aarch64/20.03/boot/headers-4.19.90-2003… - initrd http://172.168.131.113:8800/initrd/deps/initramfs/openeuler/aarch64/20.03/m… - initrd http://172.168.131.113:8800/initrd/deps/initramfs/openeuler/aarch64/20.03/s… - initrd http://172.168.131.113:8800/initrd/pkg/initramfs/openeuler/aarch64/20.03/mp… - initrd http://172.168.131.113:8800/initrd/pkg/initramfs/openeuler/aarch64/20.03/sp… - initrd http://172.168.131.113:8800/initrd/lkp/baijing/lkp-aarch64.cgz - initrd http://172.168.131.113:3087/job_initrd_tmpfs/crystal.147702/job.cgz Signed-off-by: Bai Jing <799286817(a)qq.com> --- src/scheduler/jobfile_operate.cr | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/scheduler/jobfile_operate.cr b/src/scheduler/jobfile_operate.cr index 98bf011..10e93fb 100644 --- a/src/scheduler/jobfile_operate.cr +++ b/src/scheduler/jobfile_operate.cr @@ -76,8 +76,24 @@ module Jobfile::Operate script_lines = JSON.parse(script_lines) end + def self.handle_job_field(job_content, fields) + fields.each do |field| + case field + when "initrds_uri" + values = job_content["initrds_uri"].to_s.split("\n").map(&.strip).reject!(&.empty?) + values = values.map { |value| JSON::Any.new(value) } + job_content["initrds_uri"] = JSON::Any.new(values) + end + end + + return job_content + end + def self.create_job_cpio(job_content : JSON::Any, base_dir : String) job_content = job_content.as_h + + need_to_handle = ["initrds_uri"] + tmp_job_content = handle_job_field(job_content.clone, need_to_handle) # put job2sh in an array if job_content.has_key?("job2sh") @@ -100,8 +116,8 @@ module Jobfile::Operate # - this will create a <'#! jobs/pixz.yaml':> in the yaml file # - but the orange is <#! jobs/pixz.yaml> in the user job.yaml # tested : no effect to job.sh - File.open(temp_yaml, "w") do |file| - YAML.dump(job_content, file) + File.open(temp_yaml, "a") do |file| + YAML.dump(tmp_job_content, file) end # generate unbroken job shell content -- 2.23.0

3 3

[PATCH v2 compass-ci 3/3] send-internet-mail: update start
by Luan Shengde 02 Dec '20

02 Dec '20

attach MAILDIR to the container store mail files that have been sent - MAILDIR/sent Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/start | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/container/send-internet-mail/start b/container/send-internet-mail/start index 483f01a..d6e4e25 100755 --- a/container/send-internet-mail/start +++ b/container/send-internet-mail/start @@ -12,25 +12,28 @@ names = Set.new %w[ ROBOT_EMAIL_ADDRESS ROBOT_EMAIL_PASSWORD SEND_MAIL_PORT + MAILDIR ] defaults = relevant_defaults(names) defaults['CCI_SRC'] = '/c/compass-ci' +defaults['MAILDIR'] ||= '/srv/cci/Maildir/.compass-ci' +defaults['HOST_SERVER'] = ENV['HOSTNAME'] env = docker_env(defaults) -cmd = %W[ +cmd = %w[ docker run --restart=always --name=send-internet-mail -d ] + env + %W[ -p #{defaults['SEND_MAIL_PORT']}:#{defaults['SEND_MAIL_PORT']} + -v #{defaults['MAILDIR']}:#{defaults['MAILDIR']}:rw -v #{ENV['CCI_SRC']}:#{defaults['CCI_SRC']} -v /etc/localtime:/etc/localtime:ro -w #{defaults['CCI_SRC']}/container/send-internet-mail - --log-driver json-file send-internet-mail ] -- 2.23.0

3 4

[PATCH v2 compass-ci] 本地安装compass-ci客户端.md: adjust markdown document format
by Xu Xijian 02 Dec '20

02 Dec '20

1. In markdown documents, text for bash should be contained in block as below: ```bash content ``` 2. Add blank line between title and normal text, or it will not line feed. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- ...205compass-ci\345\256\242\346\210\267\347\253\257.md" | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git "a/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" "b/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" index 43a6bf5..b986626 100644 --- "a/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" +++ "b/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" @@ -9,20 +9,25 @@ Compass-CI 将 [lkp-tests](https://gitee.com/wu_fengguang/lkp-tests) 作为客 ## Getting started 1. setup default config + run the following command to add the below setup to default config file +```SHELL mkdir -p ~/.config/compass-ci/defaults/ cat >> ~/.config/compass-ci/defaults/${USER}.yaml <<-EOF my_uuid: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxx my_email: xxx my_name: xx EOF - +``` 2. download lkp-tests and dependencies + run the following command to install the lkp-test and effect the configuration +```SHELL git clone https://gitee.com/wu_fengguang/lkp-tests.git cd lkp-tests make install source ~/.bashrc && source ~/.bash_profile - +``` 3. submit job + Now try [submitting a job to compass-ci](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manua… -- 2.23.0

1 0

[PATCH v2 compass-ci 1/3] send-internet-mail: add function: store email
by Luan Shengde 02 Dec '20

02 Dec '20

add function: store_email store email to MAILDIR/sent file name: ${email_addr}_${time_now} Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/send-internet-mail.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) mode change 100644 => 100755 container/send-internet-mail/send-internet-mail.rb diff --git a/container/send-internet-mail/send-internet-mail.rb b/container/send-internet-mail/send-internet-mail.rb old mode 100644 new mode 100755 index 3b1d159..851904c --- a/container/send-internet-mail/send-internet-mail.rb +++ b/container/send-internet-mail/send-internet-mail.rb @@ -19,3 +19,13 @@ smtp = { } Mail.defaults { delivery_method :smtp, smtp } + +def store_email(mail) + time_now = Time.new.strftime('%Y%m%d%H%M%S') + file_name = [mail.to[0], time_now].join('_') + file_full_name = File.join(ENV['MAILDIR'], 'sent', file_name) + File.open(file_full_name, 'w') do |f| + f.puts mail + end + FileUtils.chown_R(1144, 1110, file_full_name) +end -- 2.23.0

2 2

[PATCH v3 compass-ci] container/result-webdav: adjust the value of second_dir
by Cui Yili 02 Dec '20

02 Dec '20

[why] 1. when the uri starts with "//", etc., the previous regular expression did not get the second level directory correctly. 2. when the uploaded data package does not meet the format of "xxxx_yyyymmdd.cgz", uploading is also allowed, but no symbolic link. [how] 1. modify the variable second_dir regular expression. 2. make conditional judgments on link_name. Signed-off-by: cuiyili <2268260388(a)qq.com> --- container/result-webdav/nginx.conf | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/container/result-webdav/nginx.conf b/container/result-webdav/nginx.conf index c880522..1a4dddc 100644 --- a/container/result-webdav/nginx.conf +++ b/container/result-webdav/nginx.conf @@ -54,17 +54,21 @@ http { path = string.match(uri,"%g*/") upload_file = string.match(uri, ".+/([^/]*%.%w+)$") - second_dir = string.match(uri, "/.+/(.*)/.+") + second_dir = string.match(uri, "/*%w+/(%w+)/.+") - if (second_dir == "pkg") + if (second_dir == "pkg" or second_dir == "build-pkg") then - link_name = "latest.cgz" + link_name = "latest" else - link_name = string.match(upload_file, "(.*)_%d+%.cgz")..".cgz" + link_name = string.match(upload_file, "(.*)_%d+%.cgz") + end + + if (link_name ~= '') + then + link_name = link_name..".cgz" + root_patch = "/srv/"..path + io.popen("cd "..root_patch.."&& ln -sf "..upload_file.." "..link_name) end - - root_patch = "/srv/"..path - io.popen("cd "..root_patch.."&& ln -sf "..upload_file.." "..link_name) } } -- 2.23.0

1 0

[PATCH v2 lkp-tests] jobs/borrow-1h.yaml: set "pub_key" field as level-1 parameter
by Zhang Yale 02 Dec '20

02 Dec '20

[Why] Now the "pub_key" field is not placed under the "sshd" field in yaml, use "pub_key" to determine whether to password-free login. Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- jobs/borrow-1h.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jobs/borrow-1h.yaml b/jobs/borrow-1h.yaml index be22b94a..f2fedd20 100644 --- a/jobs/borrow-1h.yaml +++ b/jobs/borrow-1h.yaml @@ -1,13 +1,13 @@ suite: borrow testcase: borrow +pub_key: <%= + begin + File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp + rescue + nil + end + %> sshd: - pub_key: <%= - begin - File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp - rescue - nil - end - %> # sleep at the bottom sleep: 1h -- 2.23.0

1 0

[PATCH lkp-tests] jobs/borrow-1h.yaml: set "pub_key" field as level-1 parameter
by Zhang Yale 02 Dec '20

02 Dec '20

[Why] Now, the "pub_key" field is not placed under the "sshd" field in yaml, use "pub_key" to determine whether to password-free login. Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- jobs/borrow-1h.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/borrow-1h.yaml b/jobs/borrow-1h.yaml index be22b94a..38306441 100644 --- a/jobs/borrow-1h.yaml +++ b/jobs/borrow-1h.yaml @@ -1,13 +1,13 @@ suite: borrow testcase: borrow -sshd: - pub_key: <%= +pub_key: <%= begin File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp rescue nil end %> +sshd: # sleep at the bottom sleep: 1h -- 2.23.0

1 1

[PATCH compass-ci] stats_worker.cr: add blank line after next
by Xu Xijian 02 Dec '20

02 Dec '20

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/extract-stats/stats_worker.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index bcf6e52..4a6ce0f 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -102,6 +102,7 @@ class StatsWorker STDERR.puts e.message next end + next if is_exists new_error_ids << error_id -- 2.23.0

2 2

[PATCH v2 compass-ci] container/defconfig.rb: fix rubocop warning
by Liu Yinsi 02 Dec '20

02 Dec '20

defconfig.rb:23:12: Style/BlockDelimiters: Avoid using {...} for multi-line blocks. hash.map { |k, v| Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/defconfig.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/container/defconfig.rb b/container/defconfig.rb index dd52fe3..c0768ca 100755 --- a/container/defconfig.rb +++ b/container/defconfig.rb @@ -20,9 +20,7 @@ end def set_local_env hash = cci_defaults - hash.map { |k, v| - system "export #{k}=#{v}" - } + hash.map { |k, v| system "export #{k}=#{v}" } end def docker_env(hash) -- 2.23.0

1 0

[PATCH v1 compass-ci 1/2] sched: no need set kernel if os mount is container
by Cao Xueliang 02 Dec '20

02 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/lib/job.cr | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 44a6ecf..7758e78 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -146,16 +146,22 @@ class Job set_result_root() set_result_service() set_os_mount() + set_kernel() set_depends_initrd() - set_kernel_version() set_initrds_uri() - set_kernel_uri() - set_kernel_params() set_lkp_server() set_sshr_info() set_queue() end + private def set_kernel + return if os_mount == "container" + + set_kernel_version() + set_kernel_uri() + set_kernel_params() + end + private def append_init_field INIT_FIELD.each do |k, v| k = k.to_s -- 2.23.0

1 0

[PATCH compass-ci] sched: result root add "." and replace "/" to "-"
by Cao Xueliang 02 Dec '20

02 Dec '20

Before: archlinuxaur-zzookeeper-HEAD After: archlinux-aur-z-zookeeper-HEAD Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/scheduler/pp_params.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scheduler/pp_params.cr b/src/scheduler/pp_params.cr index 28ec6ef..717f25b 100644 --- a/src/scheduler/pp_params.cr +++ b/src/scheduler/pp_params.cr @@ -51,8 +51,8 @@ class Job private def format_string(original_str) temp = [] of String original_str.each_char do |char| - if "#{char}" =~ /-|\w/ - temp << "#{char}" + if "#{char}" =~ /-|\.|\/|\w/ + temp << "#{char}".gusb("\/", "-") end end -- 2.23.0

2 2

[PATCH compass-ci] lib/job.cr: set "pub_key" field as level-1 parameter
by Zhang Yale 02 Dec '20

02 Dec '20

[why] The "pub_key" field in yaml will not be set under sshd, It should be set as level-1 parameter for password-free login, and "sshd" can be placed below the "on_fail" field or elsewhere. Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- src/lib/job.cr | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index a4402ab..44a6ecf 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -184,7 +184,7 @@ class Job end private def set_sshr_info - return unless self["sshd"]? + return unless @hash["pub_key"]? self["sshr_port"] = ENV["SSHR_PORT"] self["sshr_port_base"] = ENV["SSHR_PORT_BASE"] @@ -196,8 +196,7 @@ class Job end private def set_my_ssh_pubkey - sshd = @hash["sshd"].as_h? || Hash(String, JSON::Any).new() - pub_key = sshd["pub_key"]?.to_s + pub_key = @hash["pub_key"]?.to_s update_account_my_pub_key(pub_key) @hash["my_ssh_pubkey"] = @account_info["my_ssh_pubkey"] -- 2.23.0

1 0

[PATCH v4 compass-ci 3/3] container/logging-es: set available memory size
by Liu Yinsi 02 Dec '20

02 Dec '20

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/logging-es/Dockerfile | 6 ++++-- container/logging-es/build | 19 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/container/logging-es/Dockerfile b/container/logging-es/Dockerfile index 6c01c43..a0acf94 100644 --- a/container/logging-es/Dockerfile +++ b/container/logging-es/Dockerfile @@ -4,6 +4,8 @@ ARG BASE_IMAGE FROM $BASE_IMAGE +ARG MEMORY + # docker image borrowed from hub.docker.com/r/gagara/elasticsearch-oss-arm64 MAINTAINER Wu Zhende <wuzhende666(a)163.com> @@ -14,8 +16,8 @@ RUN sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/elastic sed -i '$a cluster.initial_master_nodes: ["node-1"]' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a indices.memory.index_buffer_size: 20%' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a thread_pool.write.queue_size: 2000' /usr/share/elasticsearch/config/elasticsearch.yml && \ - sed -i "s/-Xms1g/-Xms30g/g" /usr/share/elasticsearch/config/jvm.options && \ - sed -i "s/-Xmx1g/-Xmx30g/g" /usr/share/elasticsearch/config/jvm.options + sed -i "s/-Xms1g/-Xms${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options && \ + sed -i "s/-Xmx1g/-Xmx${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options RUN mkdir /usr/share/elasticsearch/tmp && \ chown -R 1090:1090 /usr/share/elasticsearch diff --git a/container/logging-es/build b/container/logging-es/build index 451041b..6465ace 100755 --- a/container/logging-es/build +++ b/container/logging-es/build @@ -1,14 +1,17 @@ -#!/bin/bash +#!/usr/bin/env ruby # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true -declare -A BASE_IMAGE_DICT +require_relative '../defconfig.rb' -BASE_IMAGE_DICT=( - ["aarch64"]="gagara/elasticsearch-oss-arm64:7.6.2" - ["x86_64"]="elasticsearch:7.6.2" -) +BASE_IMAGE_DICT = { + 'aarch64' => 'gagara/elasticsearch-oss-arm64:7.6.2', + 'x86_64' => 'elasticsearch:7.6.2' +}.freeze -BASE_IMAGE=${BASE_IMAGE_DICT[$(arch)]} +BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] -docker build -t logging-es:7.6.2 --build-arg BASE_IMAGE="$BASE_IMAGE" . +available_memory = get_available_memory + +system "docker build -t logging-es:7.6.2 --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg MEMORY=#{available_memory} ." -- 2.23.0

1 0

[PATCH v4 compass-ci 2/3] container/es: set available memory size
by Liu Yinsi 02 Dec '20

02 Dec '20

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/es/Dockerfile | 6 ++++-- container/es/build | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/container/es/Dockerfile b/container/es/Dockerfile index e626dae..ed02490 100644 --- a/container/es/Dockerfile +++ b/container/es/Dockerfile @@ -3,6 +3,8 @@ FROM alpine:3.11 +ARG MEMORY + RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories RUN apk add --no-cache elasticsearch curl @@ -27,8 +29,8 @@ RUN rm -rf /etc/init.d/elasticsearch \ RUN sed -i 's:#path.data\: /path/to/data:path.data\: /srv/es:' /usr/share/es/config/elasticsearch.yml; RUN sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/es/config/elasticsearch.yml; -RUN sed -i "s/-Xms256m/-Xms20g/g" /usr/share/es/config/jvm.options -RUN sed -i "s/-Xmx256m/-Xmx20g/g" /usr/share/es/config/jvm.options +RUN sed -i "s/-Xms256m/-Xms${MEMORY}m/g" /usr/share/es/config/jvm.options +RUN sed -i "s/-Xmx256m/-Xmx${MEMORY}m/g" /usr/share/es/config/jvm.options WORKDIR /usr/share/es diff --git a/container/es/build b/container/es/build index d5607e0..17a225c 100755 --- a/container/es/build +++ b/container/es/build @@ -1,6 +1,10 @@ -#!/bin/sh +#!/usr/bin/env ruby # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true -docker build -t es643b:alpine311 . --network=host +require_relative '../defconfig.rb' +available_memory = get_available_memory + +system "docker build -t es643b:alpine311 --build-arg MEMORY=#{available_memory} --network=host ." -- 2.23.0

1 0

[PATCH v4 compass-ci 1/3] container/defconfig.rb: limit container memory usage
by Liu Yinsi 02 Dec '20

02 Dec '20

use Math.sqrt(memory) to get container occupied memory according to system memory size, instead of setting fixed size, more reasonable. In addition, '-Xms' not support floating number, so use to_i. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/defconfig.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/container/defconfig.rb b/container/defconfig.rb index 05c56db..dd52fe3 100755 --- a/container/defconfig.rb +++ b/container/defconfig.rb @@ -35,3 +35,11 @@ def docker_rm(container) system "docker stop #{container} && docker rm -f #{container}" end + +def get_available_memory + memory = File.readlines('/proc/meminfo')[0].chomp.split[1].to_f / 1048576 + + # set container available memory size, minimum size is 1024m, maximum size is 30720m, + # take the middle value according to the system memory size. + [1024, 30720, Math.sqrt(memory) * 1024].sort[1].to_i +end -- 2.23.0

1 0

[PATCH v3 compass-ci 3/3] container/logging-es: set available memory size
by Liu Yinsi 02 Dec '20

02 Dec '20

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/logging-es/Dockerfile | 6 ++++-- container/logging-es/build | 19 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/container/logging-es/Dockerfile b/container/logging-es/Dockerfile index 6c01c43..a0acf94 100644 --- a/container/logging-es/Dockerfile +++ b/container/logging-es/Dockerfile @@ -4,6 +4,8 @@ ARG BASE_IMAGE FROM $BASE_IMAGE +ARG MEMORY + # docker image borrowed from hub.docker.com/r/gagara/elasticsearch-oss-arm64 MAINTAINER Wu Zhende <wuzhende666(a)163.com> @@ -14,8 +16,8 @@ RUN sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/elastic sed -i '$a cluster.initial_master_nodes: ["node-1"]' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a indices.memory.index_buffer_size: 20%' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a thread_pool.write.queue_size: 2000' /usr/share/elasticsearch/config/elasticsearch.yml && \ - sed -i "s/-Xms1g/-Xms30g/g" /usr/share/elasticsearch/config/jvm.options && \ - sed -i "s/-Xmx1g/-Xmx30g/g" /usr/share/elasticsearch/config/jvm.options + sed -i "s/-Xms1g/-Xms${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options && \ + sed -i "s/-Xmx1g/-Xmx${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options RUN mkdir /usr/share/elasticsearch/tmp && \ chown -R 1090:1090 /usr/share/elasticsearch diff --git a/container/logging-es/build b/container/logging-es/build index 451041b..6465ace 100755 --- a/container/logging-es/build +++ b/container/logging-es/build @@ -1,14 +1,17 @@ -#!/bin/bash +#!/usr/bin/env ruby # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true -declare -A BASE_IMAGE_DICT +require_relative '../defconfig.rb' -BASE_IMAGE_DICT=( - ["aarch64"]="gagara/elasticsearch-oss-arm64:7.6.2" - ["x86_64"]="elasticsearch:7.6.2" -) +BASE_IMAGE_DICT = { + 'aarch64' => 'gagara/elasticsearch-oss-arm64:7.6.2', + 'x86_64' => 'elasticsearch:7.6.2' +}.freeze -BASE_IMAGE=${BASE_IMAGE_DICT[$(arch)]} +BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] -docker build -t logging-es:7.6.2 --build-arg BASE_IMAGE="$BASE_IMAGE" . +available_memory = get_available_memory + +system "docker build -t logging-es:7.6.2 --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg MEMORY=#{available_memory} ." -- 2.23.0

1 0

[PATCH v3 compass-ci 2/3] container/es: set available memory size
by Liu Yinsi 02 Dec '20

02 Dec '20

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/es/Dockerfile | 6 ++++-- container/es/build | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/container/es/Dockerfile b/container/es/Dockerfile index e626dae..ed02490 100644 --- a/container/es/Dockerfile +++ b/container/es/Dockerfile @@ -3,6 +3,8 @@ FROM alpine:3.11 +ARG MEMORY + RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories RUN apk add --no-cache elasticsearch curl @@ -27,8 +29,8 @@ RUN rm -rf /etc/init.d/elasticsearch \ RUN sed -i 's:#path.data\: /path/to/data:path.data\: /srv/es:' /usr/share/es/config/elasticsearch.yml; RUN sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/es/config/elasticsearch.yml; -RUN sed -i "s/-Xms256m/-Xms20g/g" /usr/share/es/config/jvm.options -RUN sed -i "s/-Xmx256m/-Xmx20g/g" /usr/share/es/config/jvm.options +RUN sed -i "s/-Xms256m/-Xms${MEMORY}m/g" /usr/share/es/config/jvm.options +RUN sed -i "s/-Xmx256m/-Xmx${MEMORY}m/g" /usr/share/es/config/jvm.options WORKDIR /usr/share/es diff --git a/container/es/build b/container/es/build index d5607e0..17a225c 100755 --- a/container/es/build +++ b/container/es/build @@ -1,6 +1,10 @@ -#!/bin/sh +#!/usr/bin/env ruby # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true -docker build -t es643b:alpine311 . --network=host +require_relative '../defconfig.rb' +available_memory = get_available_memory + +system "docker build -t es643b:alpine311 --build-arg MEMORY=#{available_memory} --network=host ." -- 2.23.0

1 0

[PATCH v3 compass-ci 1/3] container/defconfig.rb: limit container memory usage
by Liu Yinsi 02 Dec '20

02 Dec '20

use Math.sqrt(memory) to get container occupied memory according to system memory size, instead of setting fixed size, more reasonable. In addition, '-Xms' not support floating number, so use to_i. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/defconfig.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/container/defconfig.rb b/container/defconfig.rb index 05c56db..e3ca89c 100755 --- a/container/defconfig.rb +++ b/container/defconfig.rb @@ -35,3 +35,11 @@ def docker_rm(container) system "docker stop #{container} && docker rm -f #{container}" end + +def get_available_memory + memory = File.readlines('/proc/meminfo')[0].chomp.split[1].to_f/1048576 + + # set container available memory size, minimum size is 1024m, maximum size is 30720m, + # take the middle value according to the system memory size. + [1024, 30720, Math.sqrt(memory)*1024].sort[1].to_i +end -- 2.23.0

1 0

[PATCH compass-ci] lib/job.cr: set "pub_key" field as level-1 parameter for password-free login
by Zhang Yale 02 Dec '20

02 Dec '20

[why] The "pub_key" field in yaml is used as the field to determine whether to password-free login. Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- src/lib/job.cr | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index a4402ab..44a6ecf 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -184,7 +184,7 @@ class Job end private def set_sshr_info - return unless self["sshd"]? + return unless @hash["pub_key"]? self["sshr_port"] = ENV["SSHR_PORT"] self["sshr_port_base"] = ENV["SSHR_PORT_BASE"] @@ -196,8 +196,7 @@ class Job end private def set_my_ssh_pubkey - sshd = @hash["sshd"].as_h? || Hash(String, JSON::Any).new() - pub_key = sshd["pub_key"]?.to_s + pub_key = @hash["pub_key"]?.to_s update_account_my_pub_key(pub_key) @hash["my_ssh_pubkey"] = @account_info["my_ssh_pubkey"] -- 2.23.0

2 2

[PATCH compass-ci] define_files.md: text format adapt to markdown
by Xu Xijian 02 Dec '20

02 Dec '20

In markdown documents, text for bash should be contained in block as belows: ```bash content ``` Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- doc/job/define_files.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/job/define_files.md b/doc/job/define_files.md index 2c3457d..601d498 100644 --- a/doc/job/define_files.md +++ b/doc/job/define_files.md @@ -1,6 +1,7 @@ # define_files Meaning: +```bash - We use lkp-tests as client for compass-ci. There are many testcases in lkp-tests. most testcases contain the following files: distro/depends/$testcase distro/depends/$testcase-dev @@ -10,7 +11,10 @@ Meaning: jobs/$testcase.yaml - When user add new testcase files to lkp-tests or change existing testcase files in lkp-tests, these change files that related with test program would be added to define_files field. - The define_files field do not need to be specified by user in the $testcase.yaml. It can be generated automatically when user submit with the option '-a'. User can confirm the define_files field in the job.yaml under result_root. +``` Usage example: +```bash - % submit -a $testcase.yaml testbox=vm-2p8g--$USER os=openeuler os_arch=aarch64 os_version=20.03 - % cat /srv/result/$testcase/vm-2p8g--$USER/$date/$job_id/job.yaml +``` -- 2.23.0

2 3

[PATCH compass-ci] providers/docker: delete the noexec flag of tmpfs
by Xiao Shenwei 02 Dec '20

02 Dec '20

error messgae: ./configure permission denied [why] tmpfs default option (rw,nosuid,noexec,nodev) can't execute stuff from it cause of the noexec flag refer-to: https://github.com/docker/compose/issues/3425 Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- providers/docker/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/docker/run.sh b/providers/docker/run.sh index 9edad62..d532a56 100755 --- a/providers/docker/run.sh +++ b/providers/docker/run.sh @@ -23,7 +23,7 @@ cmd=( docker run --rm -m $memory - --mount type=tmpfs,destination=/tmp + --tmpfs /tmp:rw,exec,nosuid,nodev -e CCI_SRC=/c/compass-ci -v ${load_path}/lkp:/lkp -v ${DIR}/bin:/root/bin:ro -- 2.23.0

2 3

[PATCH v2 compass-ci 2/3] send-internet-mail/mail-post: call to store email
by Luan Shengde 02 Dec '20

02 Dec '20

call function store_email to store email to directory sent Signed-off-by: Luan Shengde <shdluan(a)163.com> --- lib/mail-post.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/mail-post.rb b/lib/mail-post.rb index 0b1f94a..cac7a76 100755 --- a/lib/mail-post.rb +++ b/lib/mail-post.rb @@ -51,5 +51,11 @@ def send_mail(mail_info) to mail_info['to'] body mail_info['body'] end + mail.deliver! + + return unless ENV['SEND_MAIL_PORT'].to_s == '49000' + return unless ENV['HOST_SERVER'] == 'z9' + + store_email(mail) end -- 2.23.0

1 0

[PATCH v2 compass-ci] providers/docker: delete the noexec flag of tmpfs
by Xiao Shenwei 02 Dec '20

02 Dec '20

error messgae: PKGBUILD: ./configure permission denied [why] tmpfs default option (rw,nosuid,noexec,nodev) can't be executed directly cause of the noexec flag refer-to: https://github.com/docker/compose/issues/3425 Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- providers/docker/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/docker/run.sh b/providers/docker/run.sh index 9edad62..d532a56 100755 --- a/providers/docker/run.sh +++ b/providers/docker/run.sh @@ -23,7 +23,7 @@ cmd=( docker run --rm -m $memory - --mount type=tmpfs,destination=/tmp + --tmpfs /tmp:rw,exec,nosuid,nodev -e CCI_SRC=/c/compass-ci -v ${load_path}/lkp:/lkp -v ${DIR}/bin:/root/bin:ro -- 2.23.0

1 0

[PATCH v3 compass-ci 1/2] providers/docker: delete the noexec flag of tmpfs
by Xiao Shenwei 02 Dec '20

02 Dec '20

error messgae: ./configure permission denied [why] tmpfs default option (rw,nosuid,noexec,nodev) can't execute stuff from it cause of the noexec flag refer-to: https://github.com/docker/compose/issues/3425 Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- providers/docker/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/docker/run.sh b/providers/docker/run.sh index 9edad62..d532a56 100755 --- a/providers/docker/run.sh +++ b/providers/docker/run.sh @@ -23,7 +23,7 @@ cmd=( docker run --rm -m $memory - --mount type=tmpfs,destination=/tmp + --tmpfs /tmp:rw,exec,nosuid,nodev -e CCI_SRC=/c/compass-ci -v ${load_path}/lkp:/lkp -v ${DIR}/bin:/root/bin:ro -- 2.23.0

1 1

[PATCH v3 compass-ci 2/2] container/netdata-slave: monitoring HW server
by Xiao Shenwei 02 Dec '20

02 Dec '20

[monitored object] file-server : master crystal.ci : slave z9 : slave ibmc-mgnt : slave backup-server: slave jumper-server: slave startup by multi-qemu or multi-docker machine [function] slave node will collect host machine metrics, send to master slave node does not provider data storage, alarm, web services. [usage] config/stream.conf destination = master-HOST[:PORT] api key = uuid refer-to: https://github.com/netdata/netdata/tree/master/streaming Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- container/netdata-slave/Dockerfile | 10 ++++++ container/netdata-slave/build | 5 +++ container/netdata-slave/config/netdata.conf | 5 +++ container/netdata-slave/config/stream.conf | 10 ++++++ container/netdata-slave/run | 34 +++++++++++++++++++++ 5 files changed, 64 insertions(+) create mode 100755 container/netdata-slave/Dockerfile create mode 100755 container/netdata-slave/build create mode 100644 container/netdata-slave/config/netdata.conf create mode 100644 container/netdata-slave/config/stream.conf create mode 100755 container/netdata-slave/run diff --git a/container/netdata-slave/Dockerfile b/container/netdata-slave/Dockerfile new file mode 100755 index 0000000..5c2c02f --- /dev/null +++ b/container/netdata-slave/Dockerfile @@ -0,0 +1,10 @@ +#!/bin/bash +# Origin: https://hub.docker.com/r/netdata/netdata +# SPDX-License-Identifier: GPL-3.0+ + +FROM netdata/netdata + +MAINTAINER Xiao Shenwei <xiaoshenwei96(a)163.com> + +RUN usermod -u 1090 netdata && \ + groupmod -g 1090 netdata diff --git a/container/netdata-slave/build b/container/netdata-slave/build new file mode 100755 index 0000000..1a84550 --- /dev/null +++ b/container/netdata-slave/build @@ -0,0 +1,5 @@ +#!/bin/bash +# Origin: https://hub.docker.com/r/netdata/netdata +# SPDX-License-Identifier: GPL-3.0+ + +docker build -t netdata/netdata-slave . diff --git a/container/netdata-slave/config/netdata.conf b/container/netdata-slave/config/netdata.conf new file mode 100644 index 0000000..456fe02 --- /dev/null +++ b/container/netdata-slave/config/netdata.conf @@ -0,0 +1,5 @@ +[global] + memory mode = none + +[health] + enabled = no diff --git a/container/netdata-slave/config/stream.conf b/container/netdata-slave/config/stream.conf new file mode 100644 index 0000000..ac91a7d --- /dev/null +++ b/container/netdata-slave/config/stream.conf @@ -0,0 +1,10 @@ +[stream] + enabled = yes + destination = + api key = + timeout seconds = 60 + default port = 19999 + send charts matching = * + buffer size bytes = 1048576 + reconnect delay seconds = 5 + initial clock resync iterations = 60 diff --git a/container/netdata-slave/run b/container/netdata-slave/run new file mode 100755 index 0000000..e790298 --- /dev/null +++ b/container/netdata-slave/run @@ -0,0 +1,34 @@ +#!/bin/bash +# Origin: https://hub.docker.com/r/netdata/netdata +# SPDX-License-Identifier: GPL-3.0+ + +. $CCI_SRC/container/defconfig.sh + +docker_rm netdata-slave + +DIR=$(dirname $(realpath $0)) + +cmd=( + docker run -d + --restart=unless-stopped + --hostname=$HOSTNAME + --name netdata-slave + -p 19999:19999 + -v $DIR/config/netdata.conf:/etc/netdata/netdata.conf + -v $DIR/config/stream.conf:/etc/netdata/stream.conf + -v /etc/passwd:/host/etc/passwd:ro + -v /etc/group:/host/etc/group:ro + -v /proc:/host/proc:ro + -v /sys:/host/sys:ro + -v /etc/localtime:/etc/localtime:ro + -v /etc/os-release:/host//etc/os-release:ro + -v /var/run/docker.sock:/var/run/docker.sock:ro + -v /srv/cache/netdata_cache:/var/cache/netdata + -v /srv/cache/netdata_lib:/var/lib/netdata + --log-driver json-file + --log-opt max-size=1g + --cap-add SYS_PTRACE + --security-opt apparmor=unconfined + netdata/netdata-slave +) +"${cmd[@]}" -- 2.23.0

1 0

[PATCH compass-ci] sched: no need set kernel if os mount is container
by Cao Xueliang 02 Dec '20

02 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/lib/job.cr | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index a4402ab..5798ae3 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -146,16 +146,22 @@ class Job set_result_root() set_result_service() set_os_mount() + set_kernel() set_depends_initrd() - set_kernel_version() set_initrds_uri() - set_kernel_uri() - set_kernel_params() set_lkp_server() set_sshr_info() set_queue() end + private def set_kernel + return if os_mount == "container" + + set_kernel_version() + set_kernel_uri() + set_kernel_params() + end + private def append_init_field INIT_FIELD.each do |k, v| k = k.to_s -- 2.23.0

2 1

[PATCH v2 compass-ci] doc/job: add arch-nr_node-nr_cpu-memory.md
by Zhang Yu 01 Dec '20

01 Dec '20

Add a document for introducing the arch, nr_node, nr_cpu, memory fields Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- doc/job/arch-nr_node-nr_cpu-memory.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/job/arch-nr_node-nr_cpu-memory.md diff --git a/doc/job/arch-nr_node-nr_cpu-memory.md b/doc/job/arch-nr_node-nr_cpu-memory.md new file mode 100644 index 0000000..ba12368 --- /dev/null +++ b/doc/job/arch-nr_node-nr_cpu-memory.md @@ -0,0 +1,18 @@ +# arch, nr_node, nr_cpu, memory + +Meaning: +- Each test machine has some basic information, such as arch, nr_node, nr_cpu, memory + these fields mean: + arch: the architecture of the machine. + nr_node: number of NUMA nodes. + nr_cpu: number of logical CPUs. + memory: the storage space of the machine. + +- These fields are from the file which in $LAB_SRC/hosts. +- Here is an example of these fields in the host file: + arch: aarch64 + nr_node: 4 + nr_cpu: 96 + memory: 256G + ... + -- 2.23.0

2 1

[PATCH v3 compass-ci] lib: add error_messages.rb
by Lin Jiaxin 01 Dec '20

01 Dec '20

provide error messages for git bisect Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- lib/error_messages.rb | 63 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 lib/error_messages.rb diff --git a/lib/error_messages.rb b/lib/error_messages.rb new file mode 100644 index 0000000..f59c73a --- /dev/null +++ b/lib/error_messages.rb @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' + +# obtain error messages hash +class ErrorMessages + def initialize(file) + @file = file + @seqno = '' + @in_stderr = false + @error_line = '' + @error_details = '' + @error_message = {} + @error_messages = Hash.new { |h, k| h[k] = Set.new } + end + + def obtain_error_messages + File.open((a)file).each_line do |line| + next if extract_error_message(line) + next unless @in_stderr + next unless @error_message['error_line'].downcase =~ /error|warning/ + + @error_messages[@error_message['error_line']] << @error_message['error_line'] + @error_message['error_details'] + end + return @error_messages + end + + private + + def extract_error_message(line) + if line =~ /^ / + @error_details += line + else + update_error_message(line) + case line + when /^## ______________([0-9.]+):stderr$/ + @in_stderr = true + @seqno = $1 + return true + + when /^## ______________#{@seqno}:enderr$/ + @in_stderr = false + @seqno = '' + add_error_message + end + end + end + + def update_error_message(line) + @error_message['error_line'] = @error_line + @error_message['error_details'] = @error_details + @error_line = line + @error_details = '' + end + + def add_error_message + return unless @error_message['error_line'].downcase =~ /error|warning/ + + @error_messages[@error_message['error_line']] << @error_message['error_line'] + @error_message['error_details'] + end +end -- 2.23.0

2 1

[PATCH v3 compass-ci] 本地安装compass-ci客户端.md: text format adapt to markdown
by Xu Xijian 01 Dec '20

01 Dec '20

In markdown documents, text for bash should be contained in block as below: ```bash content ``` Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- ...50\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" | 2 ++ 1 file changed, 2 insertions(+) diff --git "a/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" "b/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" index 43a6bf5..9b89d27 100644 --- "a/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" +++ "b/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" @@ -8,6 +8,7 @@ Compass-CI 将 [lkp-tests](https://gitee.com/wu_fengguang/lkp-tests) 作为客 ## Getting started +```bash 1. setup default config run the following command to add the below setup to default config file mkdir -p ~/.config/compass-ci/defaults/ @@ -23,6 +24,7 @@ Compass-CI 将 [lkp-tests](https://gitee.com/wu_fengguang/lkp-tests) 作为客 cd lkp-tests make install source ~/.bashrc && source ~/.bash_profile +``` 3. submit job Now try [submitting a job to compass-ci](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manua… -- 2.23.0

1 0

[PATCH v2 compass-ci] 本地安装compass-ci客户端.md: text format adapt to markdown
by Xu Xijian 01 Dec '20

01 Dec '20

In markdown documents, text for bash should be contained in block as below: ```bash content ``` Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- ...50\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" | 2 ++ 1 file changed, 2 insertions(+) diff --git "a/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" "b/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" index 43a6bf5..46fde13 100644 --- "a/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" +++ "b/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" @@ -8,6 +8,7 @@ Compass-CI 将 [lkp-tests](https://gitee.com/wu_fengguang/lkp-tests) 作为客 ## Getting started +```bash 1. setup default config run the following command to add the below setup to default config file mkdir -p ~/.config/compass-ci/defaults/ @@ -26,3 +27,4 @@ Compass-CI 将 [lkp-tests](https://gitee.com/wu_fengguang/lkp-tests) 作为客 3. submit job Now try [submitting a job to compass-ci](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manua… +``` -- 2.23.0

1 1

[PATCH compass-ci] 本地安装compass-ci客: text format adapt to markdown
by Xu Xijian 01 Dec '20

01 Dec '20

In markdown documents, text for bash should be contained in block as below: ```bash content ``` Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- ...50\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" | 2 ++ 1 file changed, 2 insertions(+) diff --git "a/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" "b/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" index 43a6bf5..46fde13 100644 --- "a/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" +++ "b/doc/manual/\346\234\254\345\234\260\345\256\211\350\243\205compass-ci\345\256\242\346\210\267\347\253\257.md" @@ -8,6 +8,7 @@ Compass-CI 将 [lkp-tests](https://gitee.com/wu_fengguang/lkp-tests) 作为客 ## Getting started +```bash 1. setup default config run the following command to add the below setup to default config file mkdir -p ~/.config/compass-ci/defaults/ @@ -26,3 +27,4 @@ Compass-CI 将 [lkp-tests](https://gitee.com/wu_fengguang/lkp-tests) 作为客 3. submit job Now try [submitting a job to compass-ci](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manua… +``` -- 2.23.0

3 5

[PATCH lkp-tests] lib/job.rb: fix submit matrix args by cmd args
by Wei Jihui 01 Dec '20

01 Dec '20

[why] submit "os os_arch os_version"="1 2 3" this submit will split this arg to: os, os_arch, os_version=1, 2, 3 [how] change space to '|', like: submit "os|os_arch|os_version"="1|2|3" ... Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/job.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/job.rb b/lib/job.rb index f9e244a9..2869dcb3 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -288,13 +288,13 @@ class Job def multi_args(hash) jobs_array = [] hash.each { |key, value| - next unless key =~ /^\w.*\s+.*\w$/ + next unless key =~ /^\w.*\|.*\w$/ - next unless value.class == Array + key_array = key.split('|') + [value].flatten.each do |v| + next unless v =~ /^\w.*\|.*\w$/ - key_array = key.split - value.each do |v| - v_array = v.split + v_array = v.split('|') next unless key_array.size == v_array.size hash_job = hash.clone -- 2.23.0

1 0

[PATCH v2 compass-ci 4/4] container: add locate_files in assistant container
by Cao Xueliang 01 Dec '20

01 Dec '20

An interface is provided to support obtaining the real path of the /srv/initrd/deps and pkg program. Example: Input: curl -H 'Content-Type: Application/json' -XPOST '172.17.0.1:8101/locate_files' -d '{"deps_files": ["/srv/initrd/deps/nfs/openeuler/aarch64/20.03/perf/perf.cgz"], "pkg_files": ["/srv/initrd/pkg/nfs/openeuler/aarch64/20.03/plzip/latest.cgz", "/srv/initrd/pkg/nfs/openeuler/aarch64/20.03/stream/latest.cgz"]}' Output: {"deps_files":["/srv/initrd/deps/nfs/openeuler/aarch64/20.03/perf/perf_20201104.cgz"], "pkg_files":["/srv/initrd/pkg/nfs/openeuler/aarch64/20.03/plzip/1-4.cgz", "/srv/initrd/pkg/nfs/openeuler/aarch64/20.03/stream/1-1.cgz"]} Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assistant/routes.rb | 31 +++++++++++++++++++++++ container/assistant/views/locate_files.rb | 19 ++++++++++++++ 2 files changed, 50 insertions(+) create mode 100755 container/assistant/routes.rb create mode 100644 container/assistant/views/locate_files.rb diff --git a/container/assistant/routes.rb b/container/assistant/routes.rb new file mode 100755 index 0000000..54d3670 --- /dev/null +++ b/container/assistant/routes.rb @@ -0,0 +1,31 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'sinatra' +require 'json' +require 'open3' + +require_relative './views/locate_files' + +set :bind, '0.0.0.0' +set :port, 8101 + +post '/locate_files' do + request.body.rewind + + begin + data = JSON.parse request.body.read + rescue JSON::ParserError + return [400, 'parse json params error'] + end + + begin + result = locate_files(data) + rescue StandardError => e + return [400, e.backtrace.inspect] + end + + [200, result.to_json] +end diff --git a/container/assistant/views/locate_files.rb b/container/assistant/views/locate_files.rb new file mode 100644 index 0000000..9699128 --- /dev/null +++ b/container/assistant/views/locate_files.rb @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +def locate_files(data) + result = {} + + data.each do |key, value| + temp = [] + value.each do |val| + val = val.strip + temp << File.realpath(val) if File.exist?(val) + end + + result.merge!({ "#{key}": temp }) + end + + return result +end -- 2.23.0

1 0

[PATCH v2 compass-ci 3/4] container: start script for assistant container
by Cao Xueliang 01 Dec '20

01 Dec '20

This is a assistant container, it provides some APIs. For now, it provides the locate_files function to give the real path for programs in /srv/initrd/deps and pkg. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assistant/start | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 container/assistant/start diff --git a/container/assistant/start b/container/assistant/start new file mode 100755 index 0000000..5f7822e --- /dev/null +++ b/container/assistant/start @@ -0,0 +1,21 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. $CCI_SRC/container/defconfig.sh + +docker_rm assistant + +cmd=( + docker run + --restart=always + --name assistant + -u nobody + -d + -p 8101:8101 + -v /srv/initrd:/srv/initrd + -v /etc/localtime:/etc/localtime:ro + debian:assistant +) + +"${cmd[@]}" -- 2.23.0

1 0

[PATCH v2 compass-ci 2/4] container: build script for assistant container
by Cao Xueliang 01 Dec '20

01 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assistant/build | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 container/assistant/build diff --git a/container/assistant/build b/container/assistant/build new file mode 100755 index 0000000..7778c9f --- /dev/null +++ b/container/assistant/build @@ -0,0 +1,5 @@ +#!/bin/sh +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +docker build -t debian:assistant . -- 2.23.0

1 0

[PATCH v2 compass-ci 1/4] container: assistant container Dockerfile
by Cao Xueliang 01 Dec '20

01 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assistant/Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 container/assistant/Dockerfile diff --git a/container/assistant/Dockerfile b/container/assistant/Dockerfile new file mode 100644 index 0000000..f1ad7a2 --- /dev/null +++ b/container/assistant/Dockerfile @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM debian + +MAINTAINER caoxl78320(a)163.com + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && \ + apt-get install -y ruby-json ruby-sinatra curl + +COPY routes.rb /usr/local/bin/ +COPY views /usr/local/bin/views/ + +CMD ["/usr/local/bin/routes.rb"] -- 2.23.0

1 0

[PATCH compass-ci 3/3] scheduler/scheduler.cr: add start_time in before_all
by Wu Zhende 01 Dec '20

01 Dec '20

Record the interface access time for calculating the interface invoking time Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/scheduler/scheduler.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr index 98b8e2e..9f5e670 100644 --- a/src/scheduler/scheduler.cr +++ b/src/scheduler/scheduler.cr @@ -31,8 +31,10 @@ module Scheduler VERSION = "0.2.0" add_context_storage_type(Sched) + add_context_storage_type(Time::Span) before_all do |env| + env.set "start_time", Time.monotonic env.set "sched", Sched.new(env) env.response.headers["Connection"] = "close" end -- 2.23.0

1 0

[PATCH compass-ci 2/3] lib/sched.cr: initialize JSONLogger
by Wu Zhende 01 Dec '20

01 Dec '20

initialize JSONLogger as @log, so can use @log in each interface in Shced. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 26ddd92..a9f893f 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -5,6 +5,7 @@ require "kemal" require "yaml" require "./job" +require "./json_logger" require "./block_helper" require "./taskqueue_api" require "./remote_git_client" @@ -32,6 +33,8 @@ class Sched @block_helper = BlockHelper.new @rgc = RemoteGitClient.new @env = env + @log = JSONLogger.new + @log.set_env(env) end def normalize_mac(mac : String) -- 2.23.0

1 0

[PATCH compass-ci 1/3] lib/json_logger: support env
by Wu Zhende 01 Dec '20

01 Dec '20

API can set env, after set env will add some env info to log. Add: status_code/method/resource/elapsed Like this: {"level_num":1,"level":"INFO","time":"2020-12-01 20:03:19+08:00","message":"test","status_code":200,"method":"POST","resource":"/submit_job","elapsed":"133.68µs"} Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/json_logger.cr | 65 ++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/src/lib/json_logger.cr b/src/lib/json_logger.cr index 371c042..2b5ef6a 100755 --- a/src/lib/json_logger.cr +++ b/src/lib/json_logger.cr @@ -6,27 +6,56 @@ require "json" require "any_merge" class JSONLogger < Logger - FORMATTER = Logger::Formatter.new do | severity, datetime, progname, msg, io| - level_num = severity.to_i32 - logger_hash = JSON.parse(%({"level_num": #{level_num}, - "level": "#{severity}", - "time": "#{datetime}" - })).as_h - - logger_hash.any_merge!({"progname" => progname}) unless progname.empty? - logger_hash.merge!(JSON.parse(%({"caller": #{caller}})).as_h) if level_num >= 2 - - begin - message = JSON.parse(msg).as_h - rescue - message = {"message" => msg} + def initialize(logdev = STDOUT, formatter = my_formatter) + @env = nil + @env_info = Hash(String, String | Int32).new + super(logdev, formatter: formatter) + end + + def my_formatter + Logger::Formatter.new do | severity, datetime, progname, msg, io| + get_env_info(@env.as(HTTP::Server::Context)) if @env + level_num = severity.to_i32 + logger_hash = JSON.parse(%({"level_num": #{level_num}, + "level": "#{severity}", + "time": "#{datetime}" + })).as_h + + logger_hash.any_merge!({"progname" => progname}) unless progname.empty? + logger_hash.merge!(JSON.parse(%({"caller": #{caller}})).as_h) if level_num >= 2 + + begin + message = JSON.parse(msg).as_h + rescue + message = {"message" => msg} + end + logger_hash.any_merge!(message) + logger_hash.any_merge!(@env_info) + + io << logger_hash.to_json end - logger_hash.any_merge!(message) + end - io << logger_hash.to_json + private def get_env_info(env : HTTP::Server::Context) + @env_info["status_code"] = env.response.status_code + @env_info["method"] = env.request.method + @env_info["resource"] = env.request.resource + + elapsed = get_elapsed(env) + @env_info["elapsed"] = elapsed.to_s if elapsed end - def initialize(logdev = STDOUT, formatter = FORMATTER) - super(logdev, formatter: formatter) + private def get_elapsed(env : HTTP::Server::Context) + start_time = env.get?("start_time") + return unless start_time + + elapsed = (Time.monotonic - start_time.as(Time::Span)).total_milliseconds + return "#{elapsed.round(2)}ms" if elapsed >= 1 + + "#{(elapsed * 1000).round(2)}µs" + end + + def set_env(env : HTTP::Server::Context) + @env = env end end -- 2.23.0

1 0

[PATCH v1 compass-ci 4/4] container: add locate_files in assistant container
by Cao Xueliang 01 Dec '20

01 Dec '20

An interface is provided to support obtaining the real path of the /srv/initrd/deps and pkg program. Example: Input: curl -H 'Content-Type: Application/json' -XPOST '172.17.0.1:8101/locate_files' -d '{"deps_files": ["/srv/initrd/deps/nfs/openeuler/aarch64/20.03/perf/perf.cgz"], "pkg_files": ["/srv/initrd/pkg/nfs/openeuler/aarch64/20.03/plzip/latest.cgz", "/srv/initrd/pkg/nfs/openeuler/aarch64/20.03/stream/latest.cgz"]}' Output: {"deps_files":["/srv/initrd/deps/nfs/openeuler/aarch64/20.03/perf/perf_20201104.cgz"], "pkg_files":["/srv/initrd/pkg/nfs/openeuler/aarch64/20.03/plzip/1-4.cgz", "/srv/initrd/pkg/nfs/openeuler/aarch64/20.03/stream/1-1.cgz"]} Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assistant/routes.rb | 31 +++++++++++++++++++++++ container/assistant/views/locate_files.rb | 21 +++++++++++++++ 2 files changed, 52 insertions(+) create mode 100755 container/assistant/routes.rb create mode 100644 container/assistant/views/locate_files.rb diff --git a/container/assistant/routes.rb b/container/assistant/routes.rb new file mode 100755 index 0000000..54d3670 --- /dev/null +++ b/container/assistant/routes.rb @@ -0,0 +1,31 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'sinatra' +require 'json' +require 'open3' + +require_relative './views/locate_files' + +set :bind, '0.0.0.0' +set :port, 8101 + +post '/locate_files' do + request.body.rewind + + begin + data = JSON.parse request.body.read + rescue JSON::ParserError + return [400, 'parse json params error'] + end + + begin + result = locate_files(data) + rescue StandardError => e + return [400, e.backtrace.inspect] + end + + [200, result.to_json] +end diff --git a/container/assistant/views/locate_files.rb b/container/assistant/views/locate_files.rb new file mode 100644 index 0000000..5769177 --- /dev/null +++ b/container/assistant/views/locate_files.rb @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +def locate_files(data) + result = {} + + data.each do |key, value| + temp = [] + value.each do |val| + val = val.strip + if File.exist?(val) + temp << File.realpath(val) + end + end + + result.merge!({ "#{key}": temp }) + end + + return result +end -- 2.23.0

2 2

[PATCH v1 compass-ci 3/4] container: start script for assistant container
by Cao Xueliang 01 Dec '20

01 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assistant/start | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 container/assistant/start diff --git a/container/assistant/start b/container/assistant/start new file mode 100755 index 0000000..5f7822e --- /dev/null +++ b/container/assistant/start @@ -0,0 +1,21 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. $CCI_SRC/container/defconfig.sh + +docker_rm assistant + +cmd=( + docker run + --restart=always + --name assistant + -u nobody + -d + -p 8101:8101 + -v /srv/initrd:/srv/initrd + -v /etc/localtime:/etc/localtime:ro + debian:assistant +) + +"${cmd[@]}" -- 2.23.0

2 2

[PATCH v1 compass-ci 1/4] container: assistant container Dockerfile
by Cao Xueliang 01 Dec '20

01 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assistant/Dockerfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 container/assistant/Dockerfile diff --git a/container/assistant/Dockerfile b/container/assistant/Dockerfile new file mode 100644 index 0000000..0287747 --- /dev/null +++ b/container/assistant/Dockerfile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM debian + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && \ + apt-get install -y ruby-json ruby-sinatra curl + +COPY routes.rb /usr/local/bin/ +COPY views /usr/local/bin/views/ + +CMD ["/usr/local/bin/routes.rb"] -- 2.23.0

2 2

[PATCH v2 compass-ci 4/4] container/defconfig: fix rubocop warning
by Liu Yinsi 01 Dec '20

01 Dec '20

defconfig.rb:23:12: C: [Corrected] Style/BlockDelimiters: Avoid using {...} for multi-line blocks. hash.map { |k, v| Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/defconfig.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container/defconfig.rb b/container/defconfig.rb index 026a183..320fba4 100755 --- a/container/defconfig.rb +++ b/container/defconfig.rb @@ -20,9 +20,9 @@ end def set_local_env hash = cci_defaults - hash.map { |k, v| + hash.map do |k, v| system "export #{k}=#{v}" - } + end end def docker_env(hash) -- 2.23.0

2 2

[PATCH v2 lkp-tests 2/2] sbin: load PKGBUILD.src content
by Liu Shaofei 01 Dec '20

01 Dec '20

The contents of PKGBUILD.src is the following example: source=( "http://172.168.131.113:8800/initrd/build-tar/ansible/ansible-2.9.4.tar.gz" ) Signed-off-by: Liu Shaofei <liushaofei5(a)huawei.com> --- sbin/makepkg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sbin/makepkg b/sbin/makepkg index 6f8b95af..498b4872 100755 --- a/sbin/makepkg +++ b/sbin/makepkg @@ -895,6 +895,7 @@ update_pkgver() { sed --follow-symlinks -i "s:^pkgrel=[^ ]*:pkgrel=1:" "$BUILDFILE" source_safe "$BUILDFILE" [[ -n $PKGBUILD_TAG ]] && source_safe "$BUILDFILE-$PKGBUILD_TAG" + [[ -f PKGBUILD.src ]] && source_safe "PKGBUILD.src" local fullver=$(get_full_version) msg "$(gettext "Updated version: %s")" "$pkgbase $fullver" else @@ -3699,6 +3700,7 @@ else fi source_buildfile "$BUILDFILE" [[ -n $PKGBUILD_TAG ]] && source_buildfile "$BUILDFILE-$PKGBUILD_TAG" + [[ -f PKGBUILD.src ]] && source_buildfile PKGBUILD.src fi # set defaults if they weren't specified in buildfile -- 2.23.0

1 0

[PATCH v2 lkp-tests 1/2] tests: replace *.tar.* address with local address
by Liu Shaofei 01 Dec '20

01 Dec '20

When the address of source field is the format of "*.tar.*", we need to replace it with local address. The following example: source=("https://github.com/azukiapp/${pkgname}/archive/v${pkgver}.tar.gz") => source=("http://${LKP_SERVER}:8800/initrd/build-tar/${pkgname}/v${pkgver}.tar.gz") Signed-off-by: Liu Shaofei <liushaofei5(a)huawei.com> --- tests/build-pkg | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/build-pkg b/tests/build-pkg index bdaef035..617dc0b7 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -98,6 +98,26 @@ replace_source() sed -i "s|^source=[^)]*|$1|g" "$2" } +# replace *.tar.* address from source field in PKGBUILD +replace_tar_source() +{ + echo "source=(" > PKGBUILD.src + for url in ${source[@]} + do + echo "$url" | egrep '.+\.tar\..+|.+\.tar' && { + local_url="http://${LKP_SERVER}:8800/initrd/build-tar/${pkgname}/${url##*/}" + + # check whether local server exists *.tar.* file + wget -q --spider "${local_url}" + [ "$?" == 0 ] || continue + + url="\"${local_url}\"" + } + echo "$url" >> PKGBUILD.src + done + echo ")" >> PKGBUILD.src +} + build_source_pkg() { local repo_dir="" @@ -116,6 +136,8 @@ build_source_pkg() [ -n "$PKGBUILD_TAG" ] && replace_source "${upstream_source}" "PKGBUILD-$PKGBUILD_TAG" } + replace_tar_source + cgz_name="$PKG_MNT/${pack_to}/${pkgname}/${upstream_commit}.cgz" PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ $LKP_SRC/sbin/makepkg -A --check --skippgpcheck --config $LKP_SRC/etc/makepkg.conf 2>&1 -- 2.23.0

1 0

[PATCH lab-z9] lab-z9: add cluster from a103 to a107
by Wei Jihui 01 Dec '20

01 Dec '20

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- cluster/cs-s1-a105-c4 | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 cluster/cs-s1-a105-c4 diff --git a/cluster/cs-s1-a105-c4 b/cluster/cs-s1-a105-c4 new file mode 100644 index 0000000..fa29b74 --- /dev/null +++ b/cluster/cs-s1-a105-c4 @@ -0,0 +1,22 @@ +switch: Switch-P10 +ip0: 2 +nodes: + taishan200-2280-2s48p-512g--a105: + roles: [ server ] + macs: [ "44:67:47:d7:76:c4" ] + + taishan200-2280-2s48p-256g--a103: + roles: [ client1 ] + macs: [ "44:67:47:c9:d8:b8" ] + + taishan200-2280-2s48p-256g--a104: + roles: [ client2 ] + macs: [ "44:67:47:d7:6d:f0" ] + + taishan200-2280-2s48p-256g--a106: + roles: [ client3 ] + macs: [ "44:67:47:86:1c:30" ] + + taishan200-2280-2s48p-256g--a107: + roles: [ client4 ] + macs: [ "84:46:fe:73:a2:71" ] -- 2.23.0

1 0

[PATCH v5 compass-ci] doc/manual: add how-to-log-in-the-machine.md
by Zhang Yu 01 Dec '20

01 Dec '20

Add a document for introducing how to log in the machine Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- doc/manual/how-to-log-in-the-machine.md | 86 +++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 doc/manual/how-to-log-in-the-machine.md diff --git a/doc/manual/how-to-log-in-the-machine.md b/doc/manual/how-to-log-in-the-machine.md new file mode 100644 index 0000000..fefd718 --- /dev/null +++ b/doc/manual/how-to-log-in-the-machine.md @@ -0,0 +1,86 @@ +这篇文档将告诉你如何登陆测试环境 + +# 1. 前提条件 +请先学习: +* [apply-account.md](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc…, 配置个人邮箱 +* [如何申请测试机.md](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manua…, 并在本地生成RSA公私钥对 +* [submit命令详解.md](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/ma…, submit 命令详解 + +# 2. 操作方法 +## 2.1 在job.yaml里加上sshd和sleep字段, 以host-info.yaml任务为例： + +```shell + suite: host-info + category: functional + pub_key: <%= + begin + File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp + rescue + nil + end + %> + sshd: + # sleep at the bottom + sleep: 1h + + host-info: +``` + +## 2.2 以下有2种方式可以登录到测试机： +### 第一种：使用submit -m -c的方式： + 这种方式提交的任务会在指定的位置sleep并直接登陆到测试机中，适用于登陆测试环境后手动调试 + 命令：submit -m -c host-info.yaml + 任务提交完成后，当测试执行到sshd后会自动登陆到测试机器上： + 效果如下： + +```shell + hi6325@account-vm ~% submit -m -c atomic.yaml + submit atomic.yaml, got job_id=crystal.146528 + query=>{"job_id":["crystal.146528"]} + connect to ws://localhost:11310/filter + {"job_id": "crystal.146528", "result_root": "/srv/result/atomic/2020-12-01/vm-2p8g/openeuler-20.03-aarch64/1-1000/crystal.146528", "job_state": "set result root" + {"job_id": "crystal.146528", "job_state": "boot"} + {"job_id": "crystal.146528", "job_state": "download"} + "time":"2020-12-01 10:12:33","mac":"0a-2d-7b-d9-f8-b1","ip":"172.18.252.12","job_id":"crystal.146528","state":"running","testbox":"vm-2p8g.zhyl-453231"} + {"job_state":"running","job_id":"crystal.146528"} + {"job_id": "crystal.146528", "state": "set ssh port", "ssh_port": "51750", "tbox_name": "vm-2p8g.zhyl-453231"} + + root@vm-2p8g ~# +``` + +### 第二种：根据邮件信息使用ssh方式登录测试机： + 这种是使用submit方式提交的任务完成后，系统自动发送一封邮件提醒您可以在指定时间内登陆到测试环境 + 命令： submit host-info.yaml + 任务执行完成后，系统发送邮件内容如下： + +```shell + Subject: [NOTIFY Compass-ci] vm-2p8g-294828 ready to use + + Dear $my_username: + Thanks for your participation in software ecosystem! + According to your application, vm-2p8g-294828 has been provisioned. + The datails are as follows: + + Login: + ssh root(a)api.compass-ci.openeuler.org -p $port + Due time: + $deadline + HW: + nr_cpu: $nr_cpu + memory: $memory + testbox: $testbox + OS: + $os $os_version $os_arch + Regards + Compass-Ci +``` + + 可通过ssh方式登陆到测试环境： + 命令： ssh root(a)api.compass-ci.openeuler.org -p $port + 效果如下： + +```shell + hi6325@account-vm ~% ssh root(a)api.compass-ci.openeuler.org -p 51400 + + root@vm-2p8g ~# +``` -- 2.23.0

1 0