- Compass-ci - mailweb.openeuler.org

[PATCH v4 lkp-tests 1/2] tests/cci-makepkg: add download PKGBUILD patch function
by Lin Jiaxin 17 Nov '20

17 Nov '20

[Why] Support more version's PKGBUILD. [How] Download the required file through remote-git from /srv/git/pkg/ usage: job yaml add new field: PKGBUILD_TAG About $PKGBUILD_TAG, it is from PKGBUILD file's $pkgver-$pkgrel. example: PKGBUILD_TAG: 1.0-20 You should create a git repo that name same as $benchmark in /srv/git/pkg/ before you add PKGBUILD_TAG to yaml. If not, cci-makepkg will report an error. And everything same as before if no PKGBUILD_TAG. Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- tests/cci-makepkg | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/cci-makepkg b/tests/cci-makepkg index a105f189..a62b5be0 100755 --- a/tests/cci-makepkg +++ b/tests/cci-makepkg @@ -39,12 +39,29 @@ cd $LKP_SRC/pkg/$benchmark || die "pkg is empty" mount -t cifs -o guest,vers=1.0,noacl,nouser_xattr //$LKP_SERVER$PKG_MNT $PKG_MNT || die "Failed to run mount" } +download_pkgfile() +{ + curl -sS -H 'Content-Type: Application/json' -XPOST "$LKP_SERVER"':8100/git_command' \ + -d '{"git_repo": "'$1'", "git_command": ["git-show", "HEAD:'$2'"]}' -o "$2" +} + +get_pkgfile() +{ + local pkg_tag_file="PKGBUILD-$PKGBUILD_TAG" + + download_pkgfile "pkg/$benchmark" "$pkg_tag_file" + + [ -f "$pkg_tag_file" ] || die "$pkg_tag_file not exist" + [ -s "$pkg_tag_file" ] || die "$pkg_tag_file is empty" +} + get_pkg_info() { var=$1 pkg_dir=${2:-.} ( . $pkg_dir/PKGBUILD + [ -n "$PKGBUILD_TAG" ] && . $pkg_dir/PKGBUILD-$PKGBUILD_TAG eval echo '$'$var ) } @@ -71,6 +88,8 @@ update_shared_pkg() echo "update shared pkg link ${benchmark}.cgz -> ${bm_name}/${cgz_name}" } +[ -n "$PKGBUILD_TAG" ] && get_pkgfile + distro_install_depends lkp-dev [ $os = "centos" ] && -- 2.23.0

2 1

[PATCH v3 compass-ci 2/4] assign-account: add new option for ssh key
by Luan Shengde 17 Nov '20

17 Nov '20

add -r|--gen-sshkey option for answerback-email [why] enable one click login testbox from jumper server [how] add -r|--rsa-key option to enable generate private/public key on jumper server register the public key to es Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/assign-account/answerback-email.rb | 77 ++++++++++---------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index 686a327..39091c7 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -7,12 +7,23 @@ # samba mount # ssh logshn (huawei, ) (install pubkey / send password) +# 1) external user: email apply => mail robot +# 2) ./answerback-email: huawei user: cli apply +# => params +# => REST api (params different) +# => robot email notify + +# user_type: +# apply_from: mail/cli +# allow_login: + require 'json' require 'mail' require 'set' require 'optparse' require_relative '../defconfig' require_relative '../../lib/es_client' +require_relative 'send-account-email' names = Set.new %w[ JUMPER_HOST @@ -31,17 +42,18 @@ SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 49000 my_info = { 'my_email' => nil, 'my_name' => nil, + 'my_commit_url' => nil, 'my_uuid' => %x(uuidgen).chomp, - 'my_ssh_pubkey' => nil + 'my_ssh_pubkey' => [], + 'gen_sshkey' => false } -def init_info(email_file, my_info) - mail_content = Mail.read(email_file) +def init_info(mail_content, my_info) my_info['my_email'] = mail_content.from[0] my_info['my_name'] = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') - my_info['my_ssh_pubkey'] = if mail_content.part[1].filename == 'id_rsa.pub' - mail_content.part[1].body.decoded - end + my_info['my_ssh_pubkey'] << if mail_content.part[1].filename == 'id_rsa.pub' + mail_content.part[1].body.decoded + end end options = OptionParser.new do |opts| @@ -55,17 +67,23 @@ options = OptionParser.new do |opts| opts.on('-e email_address', '--email email_address', 'appoint email address') do |email_address| my_info['my_email'] = email_address - # when apply account with email address, will get no user name + # when apply account with email address, will get no user name and commit url my_info['my_name'] = '' + my_info['my_commit_url'] = '' end opts.on('-s pub_key_file', '--ssh-pubkey pub_key_file', \ 'ssh pub_key file, enable password-less login') do |pub_key_file| - my_info['my_ssh_pubkey'] = File.read(pub_key_file) + my_info['my_ssh_pubkey'] << File.read(pub_key_file).chomp end opts.on('-f email_file', '--raw-email email_file', 'email file') do |email_file| - init_info(email_file, my_info) + mail_content = Mail.read(email_file) + init_info(mail_content, my_info) + end + + opts.on('-g', '--gen-sshkey', 'generate jumper ras public/private key and return pubkey') do + my_info['gen_sshkey'] = true end opts.on_tail('-h', '--help', 'show this message') do @@ -76,32 +94,6 @@ end options.parse!(ARGV) -def build_message(email, account_info) - message = <<~EMAIL_MESSAGE - To: #{email} - Subject: [compass-ci] jumper account is ready - - Dear user: - - Thank you for joining us. - You can use the following command to login the jumper server: - - Login command: - ssh -p #{account_info['jumper_port']} #{account_info['my_login_name']}@#{account_info['jumper_host']} - - Account password: - #{account_info['my_password']} - - Suggest: - If you use the password to login, change it in time. - - regards - compass-ci - EMAIL_MESSAGE - - return message -end - def apply_account(my_info) account_info_str = %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d '#{my_info.to_json}') JSON.parse account_info_str @@ -114,12 +106,19 @@ def send_account(my_info) raise message if my_info['my_email'].nil? account_info = apply_account(my_info) - # for manually assign account, there will be no my_commit_url - # but the key my_commit_url is required for es - my_info['my_commit_url'] = '' my_info['my_login_name'] = account_info['my_login_name'] - my_info.delete 'my_ssh_pubkey' + + unless account_info['my_jumper_pubkey'].nil? + my_info['my_ssh_pubkey'] << account_info['my_jumper_pubkey'].chomp + end + + my_info.delete 'gen_sshkey' store_account_info(my_info) + + send_mail(my_info, account_info) +end + +def send_mail(my_info, account_info) message = build_message(my_info['my_email'], account_info) %x(curl -XPOST '#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") -- 2.23.0

2 12

[PATCH v8 lkp-tests 2/3] submit on_fail to the scheduler
by Hu Xuejiao 17 Nov '20

17 Nov '20

[why] When the task is failed(state), it need in the waiting state [how] When job2sh.rb defined sh_on_state, it will be invoked by the scheduler Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sbin/submit | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sbin/submit b/sbin/submit index bc90e2e0..a325e912 100755 --- a/sbin/submit +++ b/sbin/submit @@ -115,12 +115,14 @@ ARGV.each do |jobfile| sh_run_job = job.sh_run_job sh_extract_stats = job.sh_extract_stats sh_define_files = job.sh_define_files + sh_on_fail = job.sh_on_state(state='on_fail') sh_hash = { 'job2sh' => { 'run_job' => sh_run_job, 'extract_stats' => sh_extract_stats, - 'define_files' => sh_define_files + 'define_files' => sh_define_files, + 'on_fail' => sh_on_fail } } -- 2.23.0

1 0

[PATCH v8 lkp-tests 1/3] job2sh.rb: define on_state
by Hu Xuejiao 17 Nov '20

17 Nov '20

[why] If job.yaml is defined: on_fail: sleep(tests): 300(runtime) [how] By on_fail(sh_on_state), it will appear in job.sh: on_fail() { . $LKP_SRC/lib/job.sh . $LKP_SRC/lib/env.sh run_test $LKP_SRC/tests/wrapper sleep 300 } Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- lib/job2sh.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/job2sh.rb b/lib/job2sh.rb index 9c8b578d..9d0a7bb2 100755 --- a/lib/job2sh.rb +++ b/lib/job2sh.rb @@ -363,14 +363,14 @@ class Job2sh < Job out_line "}\n\n" end - def sh_on_state(job = nil) + def sh_on_state(state = nil, job = nil) @script_lines = [] @programs = available_programs(:workload_elements) job ||= (@jobx || @job).clone - if job.include?('on_fail') + if state == "on_fail" && job.include?('on_fail') @cur_func = :run_job -- 2.23.0

1 0

[PATCH lkp-tests] depends/sysbench-mysql: add depends
by Zhang Yu 17 Nov '20

17 Nov '20

Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- distro/depends/sysbench-mysql | 1 + 1 file changed, 1 insertion(+) diff --git a/distro/depends/sysbench-mysql b/distro/depends/sysbench-mysql index a248fd07..b0d33ffe 100644 --- a/distro/depends/sysbench-mysql +++ b/distro/depends/sysbench-mysql @@ -1,3 +1,4 @@ libnuma1 xfsprogs ethtool +libmariadb3 -- 2.23.0

1 0

[PATCH v2 lkp-tests 2/4] test/build-pkg: fix a softlink will be established if the package fails
by Sun Yukui 17 Nov '20

17 Nov '20

After packaging fails, a softlink will be established, but this is wrong. If the packaging fails, the softlink should not be established. Signed-off-by: Sun Yukui <sun.yukui(a)foxmail.com> --- tests/build-pkg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/build-pkg b/tests/build-pkg index 6426f2da..8d5fc7d4 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -101,10 +101,11 @@ build_source_pkg() cgz_name="$PKG_MNT/${pack_to}/${pkgname}/${upstream_commit}.cgz" PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ $LKP_SRC/sbin/makepkg -A --check --skippgpcheck --config $LKP_SRC/etc/makepkg.conf 2>&1 + + [ "$?" == 0 ] || create_softlink } check_vars mount_dest request_pkg build_source_pkg -create_softlink -- 2.23.0

3 3

[PATCH v2 lkp-tests 2/3] iso2rootfs: add step: config something of rootfs
by Yu Chuan 17 Nov '20

17 Nov '20

[Why] The initial rootfs have the follow two problems: - no dns resolver; - selinux is opened; Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/iso2rootfs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/iso2rootfs b/tests/iso2rootfs index 2e7bcd47ef4f..87a51acaca5f 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -338,6 +338,27 @@ run_qcow2rootfs() ./run "${qcow2_path}" "${ROOTFS_DES_DIR}" } + +############ config rootfs ############ +config_dns_resolver() +{ + cat <<-EOF > "${ROOTFS_DES_DIR}/etc/resolv.conf" + nameserver 114.114.114.114 + nameserver 8.8.8.8 + EOF +} + +disable_selinux() +{ + sed -i 's/^SELINUX=.*/SELINUX=disabled/g' "${ROOTFS_DES_DIR}/etc/selinux/config" +} + +config_rootfs() +{ + config_dns_resolver + disable_selinux +} + ############ test rootfs ############ get_qemu_efi_fd() { @@ -497,6 +518,8 @@ main() config_qcow2rootfs run_qcow2rootfs + config_rootfs + test_rootfs [ -z ${test_yaml} ] || submit_test_job_yaml -- 2.23.0

3 5

[PATCH v2 lkp-tests 1/3] iso2rootfs: support not submit test job
by Yu Chuan 17 Nov '20

17 Nov '20

[Why] If user only want generate a rootfs without testing, he can not write submit related fields in the job.yaml. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 2 ++ tests/iso2rootfs | 22 ++++++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 644c64be06b4..d4f0ffc0e80c 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -11,6 +11,8 @@ iso2rootfs: rootfs_path: initrd_http_host: initrd_http_port: + + # submit test yaml related fields test_yaml: test_os: test_os_arch: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index e8a027b0dad2..2e7bcd47ef4f 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -53,15 +53,8 @@ check_yaml_vars() "rootfs_path" "initrd_http_host" "initrd_http_port" - "test_yaml" - "test_os" - "test_os_arch" - "test_os_version" - "test_os_mount" - "test_testbox" ) - local yaml_t for yaml_t in "${yaml_vars[@]}" do [ -n "$(eval echo "\$${yaml_t}")" ] || die "cannot fine value of var: ${yaml_t}." @@ -443,6 +436,19 @@ submit_test_job_yaml() [ -d "${LKP_SRC}" ] || die "cannot find value of var: LKP_SRC." + local test_yaml_vars=( + "test_os" + "test_os_arch" + "test_os_version" + "test_os_mount" + "test_testbox" + ) + + for yaml_t in "${test_yaml_vars[@]}" + do + [ -n "$(eval echo "\$${yaml_t}")" ] || die "cannot fine value of var: ${yaml_t}." + done + cd "${LKP_SRC}" && { chmod +x ./sbin/install-dependencies.sh ./sbin/install-dependencies.sh @@ -492,7 +498,7 @@ main() run_qcow2rootfs test_rootfs - submit_test_job_yaml + [ -z ${test_yaml} ] || submit_test_job_yaml post_works } -- 2.23.0

3 4

[PATCH lkp-tests] stats/libmicro: fix rubocop warn
by Lu Kaiyi 17 Nov '20

17 Nov '20

Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- stats/libmicro | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stats/libmicro b/stats/libmicro index 530d8e2e..b7e895bf 100755 --- a/stats/libmicro +++ b/stats/libmicro @@ -4,6 +4,7 @@ # close_tmp 1 1 0.61880 201 0 640 $stdin.each_line do |line| - next unless line =~ /^(\w+)\s+\S+\s+\S+\s+(\S+)/ - puts "#{$1}: #{$2}" + next unless line =~ /^(\w+)\s+\S+\s+\S+\s+(\S+)/ + + puts "#{$1}: #{$2}" end -- 2.23.0

1 0

[PATCH v2 lkp-tests] distro/archlinux: add archlinux distro file
by Wang Yong 17 Nov '20

17 Nov '20

add archlinux pack method Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- distro/archlinux | 104 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100755 distro/archlinux diff --git a/distro/archlinux b/distro/archlinux new file mode 100755 index 000000000..e6d7dce50 --- /dev/null +++ b/distro/archlinux @@ -0,0 +1,104 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +shopt -s nullglob + +. $LKP_SRC/lib/install.sh +. $LKP_SRC/distro/common +. $LKP_SRC/lib/debug.sh + +distro_install_depends() +{ + local script + local bm="$1" + local scripts=$(find $LKP_SRC/distro/depends/ -name "$bm" -o -name "${bm}.[0-9]") + + for script in $scripts + do + script=$(basename $script) + packages=$(get_dependency_packages $DISTRO $script) + + [ -z "$packages" ] && continue + + echo install packages for $script: $packages + + for pkg in $packages + do + pacman -Qqs ----noconfirm $pkg || pacman -Sy --noconfirm $pkg + done + done +} + +pack_benchmark() +{ + setup_proxy + + distro_install_depends lkp-dev + + # Process each benchmark + for BM_NAME in $benchmark + do + distro_install_depends $BM_NAME-dev || continue + echo $LKP_SRC/sbin/pack -d $DISTRO -f -c -s $PKG_MNT/$pack_to $BM_NAME + ( + $LKP_SRC/sbin/pack -d $DISTRO -f -c -s $PKG_MNT/$pack_to $BM_NAME + ) + done +} + +pacman_download() +{ + echo "pacman -Sw --noconfirm $*" + pacman -Sw --noconfirm "$@" +} + +pacman_pack() +{ + local target_dir=/opt/pkgs + local date=$(date +"%Y%m%d") + + local downloaded_pkgs=$(find /var/ -type f -name "*.pkg.tar.xz") + + [ "$downloaded_pkgs" ] || return + + mkdir -p $target_dir + + mv $downloaded_pkgs $target_dir + + find $target_dir | cpio --quiet -o -H newc --owner=root.root | gzip -n -9 >$pack_to/${BM_NAME}_$date.cgz + + ln -sf ${BM_NAME}_$date.cgz $pack_to/${BM_NAME}.cgz || return + chown .lkp $pack_to/${BM_NAME}_$date.cgz $pack_to/${BM_NAME}.cgz || return + echo package installed to $pack_to/${BM_NAME}.cgz + + ls $target_dir/*.pkg.tar.gz > $pack_to/.${BM_NAME}.packages + + rm -rf $target_dir +} + +cleanup_downloaded_pkgs() +{ + find /var/ -type f -name "*.pkg.tar.xz" -exec rm -f -- '{}' \; +} + +pack_benchmark_deps() +{ + setup_proxy + + for BM_NAME in $benchmark + do + + cleanup_downloaded_pkgs + + check_shared_package $BM_NAME + + packages=$(echo $(get_dependency_packages $DISTRO $BM_NAME)) + + if pacman_download $packages; then + pacman_pack + else + echo "failed to pack-deps $BM_NAME" >&2 + echo $? + fi + done +} -- 2.23.0

3 8