- Compass-ci - mailweb.openeuler.org

[PATCH lkp-tests] etc/latency: added
by Lu Weitao 26 Apr '21

26 Apr '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- etc/latency | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 etc/latency diff --git a/etc/latency b/etc/latency new file mode 100644 index 000000000..25dab098e --- /dev/null +++ b/etc/latency @@ -0,0 +1,13 @@ +dbench.max_latency +dd.kill_time +dd.startup_time +ebizzy.throughput.per_thread.stddev_percent +fileio.latency_95th_ms +fileio.latency_max_ms +iostat\..*\.(await|r_await|w_await|svctm|avgqu-sz)\.max +latency_stats\..* +oltp.latency_95th_ms +oltp.latency_max_ms +oltp.latency_min_ms +tbench.max_latency +vm-scalability.stddev -- 2.23.0

2 2

[PATCH v2 lkp-tests 2/2] lib/rpm.sh: add add_repo function
by Wang Yong 26 Apr '21

26 Apr '21

add function for rpmbuild-pkg and install-rpm Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- lib/rpm.sh | 20 ++++++++++++++++++++ tests/install-rpm | 22 +--------------------- tests/rpmbuild-pkg | 6 ++++++ 3 files changed, 27 insertions(+), 21 deletions(-) create mode 100755 lib/rpm.sh diff --git a/lib/rpm.sh b/lib/rpm.sh new file mode 100755 index 000000000..91f234da4 --- /dev/null +++ b/lib/rpm.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +add_repo() +{ + custom_repo_name=($custom_repo_name) + custom_repo_addr=($custom_repo_addr) + + for i in ${!custom_repo_name[@]} + do + cat <<EOF >> /etc/yum.repos.d/"${custom_repo_name[$i]}.repo" +[${custom_repo_name[$i]}] +name=${custom_repo_name[$i]} +baseurl=${custom_repo_addr[$i]} +enabled=1 +gpgcheck=0 +priority=100 + +EOF + done +} \ No newline at end of file diff --git a/tests/install-rpm b/tests/install-rpm index d69d2d4c1..7449b3bd1 100755 --- a/tests/install-rpm +++ b/tests/install-rpm @@ -9,6 +9,7 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh . $LKP_SRC/lib/log.sh +. $LKP_SRC/lib/rpm.sh : "${compat_os:=budding-openeuler}" : "${DISTRO:=openeuler}" @@ -19,29 +20,8 @@ [ -n "${custom_repo_name}" ] || die "custom_repo_name is empty" [ -n "${custom_repo_addr}" ] || die "custom_repo_addr is empty" -custom_repo_name=($custom_repo_name) -custom_repo_addr=($custom_repo_addr) rpm_name=($rpm_name) -add_repo() -{ - custom_repo_name=($custom_repo_name) - custom_repo_addr=($custom_repo_addr) - - for i in ${!custom_repo_name[@]} - do - cat <<EOF >> /etc/yum.repos.d/"${custom_repo_name[$i]}.repo" -[${custom_repo_name[$i]}] -name=${custom_repo_name[$i]} -baseurl=${custom_repo_addr[$i]} -enabled=1 -gpgcheck=0 -priority=100 - -EOF - done -} - install_rpm() { log_info "Starting install test: yum install -y ${rpm_name[@]}" diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index 312582d16..5c694b9d4 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -3,10 +3,15 @@ # - compat_os # - repo_name # - repo_addr +# - custom_repo_name +# - custom_repo_addr # Git repo jobs only use the first two parameters, SRPMs jobs use all +# user can add custom_repo_name and custom_repo_addr to setup +# local repo, value can be an array . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh +. $LKP_SRC/lib/rpm.sh : "${compat_os:=budding-openeuler}" @@ -133,6 +138,7 @@ update_repo_mq() -d "{\"upload_rpms\": ["${full_list}"]}" } +add_repo [ -n "$repo_name" ] && from_srpm || from_git build_rpm upload_rpm_pkg -- 2.23.0

1 0

[PATCH lkp-tests 1/2] tests/install-rpm: add suite install-rpm
by Wang Yong 26 Apr '21

26 Apr '21

add install-rpm to install/remove and ldd test for rpm packages Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- distro/depends/install-rpm | 1 + jobs/install-rpm.yaml | 4 ++ tests/install-rpm | 79 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 distro/depends/install-rpm create mode 100755 jobs/install-rpm.yaml create mode 100755 tests/install-rpm diff --git a/distro/depends/install-rpm b/distro/depends/install-rpm new file mode 100644 index 000000000..602479c9a --- /dev/null +++ b/distro/depends/install-rpm @@ -0,0 +1 @@ +dnf-plugins-core diff --git a/jobs/install-rpm.yaml b/jobs/install-rpm.yaml new file mode 100755 index 000000000..9afa9d992 --- /dev/null +++ b/jobs/install-rpm.yaml @@ -0,0 +1,4 @@ +suite: install-rpm +category: functional + +install-rpm: diff --git a/tests/install-rpm b/tests/install-rpm new file mode 100755 index 000000000..428ca38d6 --- /dev/null +++ b/tests/install-rpm @@ -0,0 +1,79 @@ +#!/bin/bash +# - compat_os +# - rpm_name +# - custom_repo_name +# - custom_repo_addr +# user can add custom_repo_name and custom_repo_addr to setup +# local repo, value can be an array + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/upload.sh +. $LKP_SRC/lib/log.sh + +: "${compat_os:=budding-openeuler}" +: "${DISTRO:=openeuler}" +: "${basearch:=$(arch)}" +. $LKP_SRC/distro/${DISTRO} + +[ -n "${rpm_name}" ] || die "rpm_name is empty" +[ -n "${custom_repo_name}" ] || die "custom_repo_name is empty" +[ -n "${custom_repo_addr}" ] || die "custom_repo_addr is empty" + +custom_repo_name=($custom_repo_name) +custom_repo_addr=($custom_repo_addr) +rpm_name=($rpm_name) + +add_repo() +{ + custom_repo_name=($custom_repo_name) + custom_repo_addr=($custom_repo_addr) + + for i in ${!custom_repo_name[@]} + do + cat <<EOF >> /etc/yum.repos.d/"${custom_repo_name[$i]}.repo" +[${custom_repo_name[$i]}] +name=${custom_repo_name[$i]} +baseurl=${custom_repo_addr[$i]} +enabled=1 +gpgcheck=0 +priority=100 + +EOF + done +} + +install_rpm() +{ + log_info "Starting install test: yum install -y ${rpm_name[@]}" + yum install -y ${rpm_name[@]} || die "install rpm failed" +} + +ldd_test() +{ + log_info "Starting ldd test" + for i in $(rpm -qa ${rpm_name[@]}) + do + ldd_list=$(rpm -ql $i | awk -F'/' '$3 ~ /bin|sbin|lib|lib64/') + [ -z "${ldd_list}" ] || { + for i in ${ldd_list[@]} + do + [[ -f "$i" && -x "$i" ]] && { + echo "ldd for $i" + ldd -d -r $i + } + done + } + done +} + +remove_rpm() +{ + log_info "Starting remove test: yum remove -y ${rpm_name[@]}" + yum remove -y ${rpm_name[@]} || die "uninstall rpm failed" +} + +distro_install_depends install-rpm +add_repo +install_rpm +ldd_test +remove_rpm -- 2.23.0

1 1

[PATCH v3 lkp-tests] lib/bootstrap.sh: fix the output error when ping not exist
by Wang Chenglong 26 Apr '21

26 Apr '21

[Why] If the ping command exists, then run test_ping, if not, then skip. [Error] [ 179.335889] Kernel tests: Boot OK! [ 180.180309] LKP: waiting for network... [ 182.991732] /lkp/lkp/src/lib/bootstrap.sh: line 90: ping: command not found [ 183.010815] /lkp/lkp/src/lib/bootstrap.sh: line 90: ping: command not found Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- lib/bootstrap.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 070dabf46..5b82c2a6b 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -88,6 +88,10 @@ net_devices_link() test_ping() { + has_cmd ping || { + echo "no command ping, skip ping test." + return 0 + } ping -c 1 -w 1 $LKP_SERVER > /dev/null } -- 2.23.0

2 2

[PATCH v4 compass-ci] sbin/download-rootfs: download osimage
by Liu Yinsi 26 Apr '21

26 Apr '21

the default os_mount=initramfs, so download osimage to run qemu kvm. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/download-rootfs | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/sbin/download-rootfs b/sbin/download-rootfs index 68a68039..cea416c0 100755 --- a/sbin/download-rootfs +++ b/sbin/download-rootfs @@ -19,22 +19,35 @@ os_version=${3:-20.03} load_cci_defaults rootfs_dir=/srv/os/${os}/${os_arch}/${os_version} +osimage_dir=/srv/initrd/osimage/${os}/${os_arch}/${os_version} -download_os_rootfs() -{ - mkdir -p $rootfs_dir - cd $rootfs_dir +mkdir -p $rootfs_dir $osimage_dir - download_url="http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" - wget $download_url +wget_download() +{ + local download_url=$1 + wget $download_url + + [ "$?" -ne 0 ] && { + log_error "failed to download os rootfs: $download_url" + exit 1 + } +} - [ "$?" -ne 0 ] && { - log_error "failed to download os rootfs: $download_url" - exit 1 - } +download_os_rootfs() +{ + cd $rootfs_dir + wget_download "http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" + + gzip -dc ${os_version}.cgz | cpio -id + rm -rf ${os_version}.cgz +} - gzip -dc ${os_version}.cgz | cpio -id - rm -rf ${os_version}.cgz +download_osimage() +{ + cd $osimage_dir + wget_download "http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/initrd/osimage/…" + wget_download "http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/initrd/deps/nfs…" } is_virtual() @@ -42,4 +55,4 @@ is_virtual() dmidecode -s system-product-name | grep -iq "virtual" } -is_virtual || download_os_rootfs +is_virtual || ( download_os_rootfs && download_osimage ) -- 2.23.0

3 4

[PATCH compass-ci] sparrow/3-code/dev-env: rename some variables
by Liu Yinsi 26 Apr '21

26 Apr '21

[why] prevent to use variables value from z9 in testbox. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index a478fffe..ba213e3a 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -41,16 +41,16 @@ server_name=$(hostname | cut -f1 -d.) : ${git_server:=$server_ip} : ${SCHED_HTTPS_HOST:=$server_ip} : ${SCHED_HTTPS_PORT:=20002} -: ${DOCKER_PROXY_HOST:=$server_ip} +: ${docker_proxy_host:=$server_ip} : ${DOCKER_PROXY_PORT:=3128} -: ${JUMPER_HOST:=$server_ip} +: ${jumper_host:=$server_ip} : ${JUMPER_PORT:=10000} -: ${MONITOR_HOST:=$server_ip} +: ${monitor_host:=$server_ip} : ${MONITOR_PORT:=20001} : ${FETCH_MAIL_DIR:=/srv/cci/Maildir} : ${FETCH_MAIL_BOX:=.compass-ci} : ${GIT_DAEMON_HOST:=$server_ip} -: ${GIT_MIRROR_HOST:=$server_ip} +: ${git_mirror_host:=$server_ip} : ${REDIS_PORT:=6379} : ${MAILDIR:=/srv/cci/Maildir/.compass-ci} : ${SENT_MAILDIR:=/srv/cci/Maildir/.sent} @@ -122,16 +122,16 @@ DOCKER_REGISTRY_PORT: $DOCKER_REGISTRY_PORT LOCAL_SEND_MAIL_PORT: $LOCAL_SEND_MAIL_PORT MAILDIR: /srv/cci/Maildir/.compass-ci LOCAL_ROBOT_EMAIL_ADDRESS: $LOCAL_ROBOT_EMAIL_ADDRESS -DOCKER_PROXY_HOST: $DOCKER_PROXY_HOST +DOCKER_PROXY_HOST: $docker_proxy_host DOCKER_PROXY_PORT: $DOCKER_PROXY_PORT -JUMPER_HOST: $JUMPER_HOST +JUMPER_HOST: $jumper_host JUMPER_PORT: $JUMPER_PORT -MONITOR_HOST: $MONITOR_HOST +MONITOR_HOST: $monitor_host MONITOR_PORT: $MONITOR_PORT FETCH_MAIL_DIR: $FETCH_MAIL_DIR FETCH_MAIL_BOX: $FETCH_MAIL_BOX GIT_DAEMON_HOST: $GIT_DAEMON_HOST -GIT_MIRROR_HOST: $GIT_MIRROR_HOST +GIT_MIRROR_HOST: $git_mirror_host REDIS_PORT: $REDIS_PORT MAILDIR: $MAILDIR SENT_MAILDIR: $SENT_MAILDIR -- 2.23.0

2 3

[PATCH compass-ci] sparrow: set default value $CCI_SRC
by Liu Yinsi 26 Apr '21

26 Apr '21

if $CCI_SRC is null, can't find check_public_network.sh script. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/0-package/common | 2 ++ sparrow/0-package/os/openEuler | 2 ++ sparrow/5-build/os-ready | 2 ++ 3 files changed, 6 insertions(+) diff --git a/sparrow/0-package/common b/sparrow/0-package/common index eb86b242..82c09ff5 100755 --- a/sparrow/0-package/common +++ b/sparrow/0-package/common @@ -2,6 +2,8 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +[[ $CCI_SRC ]] || CCI_SRC=/c/compass-ci + . $CCI_SRC/lib/check_public_network.sh public_network_ok && { diff --git a/sparrow/0-package/os/openEuler b/sparrow/0-package/os/openEuler index 8cdadd52..5829a123 100755 --- a/sparrow/0-package/os/openEuler +++ b/sparrow/0-package/os/openEuler @@ -2,6 +2,8 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +[[ $CCI_SRC ]] || CCI_SRC=/c/compass-ci + . $CCI_SRC/lib/check_public_network.sh pkgs=( diff --git a/sparrow/5-build/os-ready b/sparrow/5-build/os-ready index 5e1acc48..91323379 100755 --- a/sparrow/5-build/os-ready +++ b/sparrow/5-build/os-ready @@ -2,6 +2,8 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +[[ $CCI_SRC ]] || CCI_SRC=/c/compass-ci + . $CCI_SRC/lib/check_public_network.sh public_network_ok && $CCI_SRC/sbin/download-rootfs -- 2.23.0

2 1

[PATCH v2 lkp-tests] lib/bootstrap.sh: fix the output error when ping not exist
by Wang Chenglong 26 Apr '21

26 Apr '21

[Why] If the ping command exists, then run test_ping, if not, then skip. [Error] [ 179.335889] Kernel tests: Boot OK! [ 180.180309] LKP: waiting for network... [ 182.991732] /lkp/lkp/src/lib/bootstrap.sh: line 90: ping: command not found [ 183.010815] /lkp/lkp/src/lib/bootstrap.sh: line 90: ping: command not found Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- lib/bootstrap.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 070dabf46..c699b40e4 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -88,6 +88,10 @@ net_devices_link() test_ping() { + [ $(command -v ping) ] || { + echo "no command ping, skip ping test." + return 0 + } ping -c 1 -w 1 $LKP_SERVER > /dev/null } -- 2.23.0

2 2

[PATCH lkp-tests] lib/bootstrap.sh: fix the output error when ping not exist
by Wang Chenglong 26 Apr '21

26 Apr '21

[Why] If the ping command exists, then run test_ping, if not, then skip. [Error] [ 179.335889] Kernel tests: Boot OK! [ 180.180309] LKP: waiting for network... [ 182.991732] /lkp/lkp/src/lib/bootstrap.sh: line 90: ping: command not found [ 183.010815] /lkp/lkp/src/lib/bootstrap.sh: line 90: ping: command not found Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- lib/bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 070dabf46..c5b8dcd12 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -88,6 +88,7 @@ net_devices_link() test_ping() { + [ $(command -v ping) ] || return 0 ping -c 1 -w 1 $LKP_SERVER > /dev/null } -- 2.23.0

2 2

[PATCH lkp-tests 2/2] lib/rpm.sh: add add_repo function
by Wang Yong 25 Apr '21

25 Apr '21

add function for rpmbuild-pkg and install-rpm Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- lib/rpm.sh | 20 ++++++++++++++++++++ tests/install-rpm | 22 +--------------------- tests/rpmbuild-pkg | 6 ++++++ 3 files changed, 27 insertions(+), 21 deletions(-) create mode 100755 lib/rpm.sh diff --git a/lib/rpm.sh b/lib/rpm.sh new file mode 100755 index 000000000..91f234da4 --- /dev/null +++ b/lib/rpm.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +add_repo() +{ + custom_repo_name=($custom_repo_name) + custom_repo_addr=($custom_repo_addr) + + for i in ${!custom_repo_name[@]} + do + cat <<EOF >> /etc/yum.repos.d/"${custom_repo_name[$i]}.repo" +[${custom_repo_name[$i]}] +name=${custom_repo_name[$i]} +baseurl=${custom_repo_addr[$i]} +enabled=1 +gpgcheck=0 +priority=100 + +EOF + done +} \ No newline at end of file diff --git a/tests/install-rpm b/tests/install-rpm index 428ca38d6..6c2f5731a 100755 --- a/tests/install-rpm +++ b/tests/install-rpm @@ -9,6 +9,7 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh . $LKP_SRC/lib/log.sh +. $LKP_SRC/lib/rpm.sh : "${compat_os:=budding-openeuler}" : "${DISTRO:=openeuler}" @@ -19,29 +20,8 @@ [ -n "${custom_repo_name}" ] || die "custom_repo_name is empty" [ -n "${custom_repo_addr}" ] || die "custom_repo_addr is empty" -custom_repo_name=($custom_repo_name) -custom_repo_addr=($custom_repo_addr) rpm_name=($rpm_name) -add_repo() -{ - custom_repo_name=($custom_repo_name) - custom_repo_addr=($custom_repo_addr) - - for i in ${!custom_repo_name[@]} - do - cat <<EOF >> /etc/yum.repos.d/"${custom_repo_name[$i]}.repo" -[${custom_repo_name[$i]}] -name=${custom_repo_name[$i]} -baseurl=${custom_repo_addr[$i]} -enabled=1 -gpgcheck=0 -priority=100 - -EOF - done -} - install_rpm() { log_info "Starting install test: yum install -y ${rpm_name[@]}" diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index 312582d16..5c694b9d4 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -3,10 +3,15 @@ # - compat_os # - repo_name # - repo_addr +# - custom_repo_name +# - custom_repo_addr # Git repo jobs only use the first two parameters, SRPMs jobs use all +# user can add custom_repo_name and custom_repo_addr to setup +# local repo, value can be an array . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh +. $LKP_SRC/lib/rpm.sh : "${compat_os:=budding-openeuler}" @@ -133,6 +138,7 @@ update_repo_mq() -d "{\"upload_rpms\": ["${full_list}"]}" } +add_repo [ -n "$repo_name" ] && from_srpm || from_git build_rpm upload_rpm_pkg -- 2.23.0

1 0

[PATCH compass-ci] sbin/es-jobs-mapping.sh: add filed
by Lu Weitao 25 Apr '21

25 Apr '21

add nr_run(integer), author_email(keyword) Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- sbin/es-jobs-mapping.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sbin/es-jobs-mapping.sh b/sbin/es-jobs-mapping.sh index 67b1ed14..0c2fc62d 100755 --- a/sbin/es-jobs-mapping.sh +++ b/sbin/es-jobs-mapping.sh @@ -148,6 +148,12 @@ else "upstream_commit": { "type": "keyword" }, + "base_commit": { + "type": "keyword" + }, + "nr_run": { + "type": "integer" + }, "my_email": { "type": "keyword" }, -- 2.23.0

2 2

[PATCH compass-ci] container/submit: mount /c/lab-z9
by Li Yuanchao 25 Apr '21

25 Apr '21

So can use hostfiles in lab-z9/hosts Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/auto-submit/start | 2 ++ 1 file changed, 2 insertions(+) diff --git a/container/auto-submit/start b/container/auto-submit/start index ad4ba9d..55b40f2 100755 --- a/container/auto-submit/start +++ b/container/auto-submit/start @@ -27,8 +27,10 @@ cmd = %w[ -d -e CCI_SRC=#{DOCKER_CCI} -e LKP_SRC=#{DOCKER_LKP} + -e CCI_REPOS=/c -v #{ENV['CCI_SRC']}:#{DOCKER_CCI} -v #{ENV['LKP_SRC']}:#{DOCKER_LKP} + -v /c/lab-z9:/c/lab-z9 -v /etc/localtime:/etc/localtime:ro -v /etc/compass-ci/defaults:/etc/compass-ci/defaults:ro -v /etc/compass-ci/register:/etc/compass-ci/register:ro -- 2.23.0

1 0

[PATCH compass-ci 2/2] lib/mail_job_result: optimize mail body
by Lu Weitao 25 Apr '21

25 Apr '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/mail_job_result.rb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/mail_job_result.rb b/lib/mail_job_result.rb index 752b8dcf..389ad26a 100644 --- a/lib/mail_job_result.rb +++ b/lib/mail_job_result.rb @@ -37,9 +37,11 @@ class MailJobResult Hi, - html_part: Thanks for your participation in Kunpeng and software ecosystem! + Thanks for your participation in Kunpeng and software ecosystem! Your Job: #{@job_id} had finished. - Bellow are comparsion result of the base_commit: + Bellow are comparsion result of the base_commit and current_commit: + \tbase_commit: #{@job['base_commit']} + \tcurrent_commit: #{@job['upstream_commit']} #{context.to_s} \n\n#{signature}" BODY @@ -72,14 +74,15 @@ end def get_compare_result(job) min_samples = job['nr_num'].to_i base_commit = job['base_commit'] - commit_id = job['commit_id'] + commit_id = job['upstream_commit'] return nil unless base_commit && commit_id - condition_list = [{'base_commit' => base_commit}, {'commit_id' => commit_id}] + condition_list = [{'base_commit' => base_commit}, {'upstream_commit' => commit_id}] options = { :min_samples => min_samples, :no_print => true} matrices_list, suite_list = create_matrices_list(condition_list, options[:min_samples]) return nil if matrices_list.size < 2 - compare_matrixes(matrices_list, suite_list, options: options) + m_titles = [base_commit.slice(0, 16), commit_id.slice(0, 24)] + compare_matrixes(matrices_list, suite_list, nil, m_titles, options: options) end -- 2.23.0

1 0

[PATCH compass-ci 1/2] lib/compare_matrixes.rb: ahead of the success result
by Lu Weitao 25 Apr '21

25 Apr '21

[Why] user care more about successful result Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/compare_matrixes.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compare_matrixes.rb b/lib/compare_matrixes.rb index 67001265..fef5714a 100644 --- a/lib/compare_matrixes.rb +++ b/lib/compare_matrixes.rb @@ -812,7 +812,7 @@ def get_all_result_str(matrixes_values, suite_list, matrixes_titles, matrixes_nu failure_str = get_result_str(matrixes_values[false].sort, suite_list, matrixes_titles, false, theme) success_str = get_result_str(matrixes_values[true].sort, suite_list, matrixes_titles, true, theme) - failure_str + success_str + success_str + failure_str end def get_result_str(values, suite_list, matrixes_titles, success, theme) -- 2.23.0

1 0

[PATCH v2 lkp-tests 2/2] lkp/host: add taishan200-2280-2s48p-256g
by Li Yuanchao 25 Apr '21

25 Apr '21

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- hosts/taishan200-2280-2s48p-256g | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 hosts/taishan200-2280-2s48p-256g diff --git a/hosts/taishan200-2280-2s48p-256g b/hosts/taishan200-2280-2s48p-256g new file mode 100644 index 000000000..b326025d7 --- /dev/null +++ b/hosts/taishan200-2280-2s48p-256g @@ -0,0 +1,4 @@ +arch: aarch64 +nr_cpu: 96 +nr_node: 4 +memory: 256G -- 2.23.0

1 0

[PATCH v2 lkp-tests 1/2] get repo my-sql_with_boostkit_patch from git_server
by Li Yuanchao 25 Apr '21

25 Apr '21

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- pkg/mysql-8-0-20/PKGBUILD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/mysql-8-0-20/PKGBUILD b/pkg/mysql-8-0-20/PKGBUILD index 5be0d9cf9..66df6b33f 100644 --- a/pkg/mysql-8-0-20/PKGBUILD +++ b/pkg/mysql-8-0-20/PKGBUILD @@ -5,8 +5,8 @@ pkgrel=1 arch=('aarch64' 'x86_64') url="http://www.mysql.com" license=('GPLv2') -source=("https://gitee.com/li-yancheng/my-sql_with_boostkit_patch.git" - "https://gitee.com/li-yancheng/gcc.git" +source=("git://$GIT_SERVER/upstream/m/my-sql_with_boostkit_patch/my-sql_with_boostkit_patch.git" + "git://$GIT_SERVER/upstream/g/gcc/gcc.git" "my.cnf") md5sums=('SKIP' 'SKIP' '99c4965aa870c475000993f52a0dee66') -- 2.23.0

1 0

[PATCH lkp-tests] tests/self-test: add the execute permission
by Lin Jiaxin 25 Apr '21

25 Apr '21

Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- tests/self-test | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/self-test diff --git a/tests/self-test b/tests/self-test old mode 100644 new mode 100755 -- 2.23.0

1 0

[PATCH lkp-tests] get repo my-sql_with_boostkit_patch from git_server
by Li Yuanchao 25 Apr '21

25 Apr '21

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- pkg/mysql-8-0-20/PKGBUILD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/mysql-8-0-20/PKGBUILD b/pkg/mysql-8-0-20/PKGBUILD index 5be0d9cf9..66df6b33f 100644 --- a/pkg/mysql-8-0-20/PKGBUILD +++ b/pkg/mysql-8-0-20/PKGBUILD @@ -5,8 +5,8 @@ pkgrel=1 arch=('aarch64' 'x86_64') url="http://www.mysql.com" license=('GPLv2') -source=("https://gitee.com/li-yancheng/my-sql_with_boostkit_patch.git" - "https://gitee.com/li-yancheng/gcc.git" +source=("git://$GIT_SERVER/upstream/m/my-sql_with_boostkit_patch/my-sql_with_boostkit_patch.git" + "git://$GIT_SERVER/upstream/g/gcc/gcc.git" "my.cnf") md5sums=('SKIP' 'SKIP' '99c4965aa870c475000993f52a0dee66') -- 2.23.0

1 0

[PATCH compass-ci] delimiter: use delimiter job id as the group_id in every bisect process job
by Cao Xueliang 25 Apr '21

25 Apr '21

we use es-jobs to track bisect process job, for example: es-jobs group_id=z9.3117489 | grep job_stat kvcount.job_state=failed: 1 kvcount.job_state=finished: 2 raw.id.[job_state=failed]: ["z9.3117506"] raw.id.[job_state=finished]: ["z9.3117490","z9.3117512"] Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter/utils.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index c782ea2..5ecbeb5 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -136,12 +136,13 @@ module Utils job = JSON.parse job_yaml record_jobs(job['id'], job['upstream_commit']) - job['suite'] = 'bisect' + job['suite'] = 'bisect-process' job['my_name'] = ENV['my_name'] job['my_email'] = ENV['my_email'] job['my_token'] = ENV['secrets_my_token'] job['bad_job_id'] = job_id job['testbox'] = job['tbox_group'] + job['group_id'] = "#{ENV['id']}" job.delete('id') job.delete('queue') -- 2.23.0

1 0

[PATCH compass-ci] delimiter: use delimiter id as the group_id in every bisect process job
by Cao Xueliang 25 Apr '21

25 Apr '21

we use es-jobs to track bisect process job, for example: es-jobs group_id=z9.3117489 | grep job_stat kvcount.job_state=failed: 1 kvcount.job_state=finished: 2 raw.id.[job_state=failed]: ["z9.3117506"] raw.id.[job_state=finished]: ["z9.3117490","z9.3117512"] Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter/utils.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index c782ea2..5ecbeb5 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -136,12 +136,13 @@ module Utils job = JSON.parse job_yaml record_jobs(job['id'], job['upstream_commit']) - job['suite'] = 'bisect' + job['suite'] = 'bisect-process' job['my_name'] = ENV['my_name'] job['my_email'] = ENV['my_email'] job['my_token'] = ENV['secrets_my_token'] job['bad_job_id'] = job_id job['testbox'] = job['tbox_group'] + job['group_id'] = "#{ENV['id']}" job.delete('id') job.delete('queue') -- 2.23.0

1 1

[PATCH lkp-tests] jobs/mysql-8-0-20.yaml: delete set testbox
by Luan Shengde 25 Apr '21

25 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/mysql-8-0-20.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/jobs/mysql-8-0-20.yaml b/jobs/mysql-8-0-20.yaml index 070c851d5..dbf8340d7 100644 --- a/jobs/mysql-8-0-20.yaml +++ b/jobs/mysql-8-0-20.yaml @@ -1,8 +1,6 @@ suite: cci-makepkg category: functional -testbox: taishan200-2280-2s48p-256g--a18 - os: openeuler os_version: 20.03 os_mount: initramfs -- 2.23.0

1 0

[PATCH compass-ci] lib/mail_bisect_result: fix can't open git url
by Cao Xueliang 25 Apr '21

25 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/mail_bisect_result.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mail_bisect_result.rb b/lib/mail_bisect_result.rb index 5a71225..38828a3 100644 --- a/lib/mail_bisect_result.rb +++ b/lib/mail_bisect_result.rb @@ -55,7 +55,7 @@ class MailBisectResult Hi #{(a)git_commit.author_name}, Some error/warning(s) are found in - git url: #{@upstream_url}/#{@commit_id} + git url: #{@upstream_url}/commit/#{@commit_id} git commit: #{@commit_id} ("#{(a)git_commit.subject}") All error/warning(s) (new ones prefixed by >>): -- 2.23.0

1 0

[PATCH compass-ci] container/assign-account: delete useless varilables
by Luan Shengde 25 Apr '21

25 Apr '21

service assign-account doesnot use ES Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/assign-account/start | 2 -- 1 file changed, 2 deletions(-) diff --git a/container/assign-account/start b/container/assign-account/start index c8c55da0..42b34c29 100755 --- a/container/assign-account/start +++ b/container/assign-account/start @@ -13,8 +13,6 @@ cmd=( --restart=always --name=assign-account -d - -e ES_USER=$ES_USER - -e ES_PASSWORD=$ES_PASSWORD -v /etc:/etc:rw -v /srv/account-info:/opt/account_data/:rw -v /home:/home:rw -- 2.23.0

1 0

[PATCH lkp-tests] jobs/mysql-8-0-20.yaml: add default value for mysql server
by Luan Shengde 25 Apr '21

25 Apr '21

tests/sysbench-mysql need the default value in the following line: : "${mysql_host:=$server}" Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/mysql-8-0-20.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/mysql-8-0-20.yaml b/jobs/mysql-8-0-20.yaml index 8c30f963e..070c851d5 100644 --- a/jobs/mysql-8-0-20.yaml +++ b/jobs/mysql-8-0-20.yaml @@ -8,7 +8,7 @@ os_version: 20.03 os_mount: initramfs mysql_user: -mysql_host: +mysql_host: localhost mysql_port: mysql_db: mysql_password: -- 2.23.0

1 0

[PATCH v2 lkp-tests] pkg/mysql-8-0-20: enable make install gcc
by Luan Shengde 25 Apr '21

25 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- pkg/mysql-8-0-20/PKGBUILD | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/mysql-8-0-20/PKGBUILD b/pkg/mysql-8-0-20/PKGBUILD index 154f936cf..5be0d9cf9 100644 --- a/pkg/mysql-8-0-20/PKGBUILD +++ b/pkg/mysql-8-0-20/PKGBUILD @@ -12,7 +12,7 @@ md5sums=('SKIP' 'SKIP' '99c4965aa870c475000993f52a0dee66') build() { - # build_gcc + build_gcc build_mysql } @@ -48,7 +48,8 @@ build_gcc() --with-boot-ldflags='-Wl,-z,relro,-z,now' \ --with-multilib-list=lp64 make -j96 - make install + make install & + wait } } -- 2.23.0

1 0

[PATCH compass-ci] sbin/auto_submit: don't get base_commit for master
by Li Yuanchao 24 Apr '21

24 Apr '21

base_commit is to find where a branch start from master, it's not for master. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sbin/auto_submit b/sbin/auto_submit index dcf5b80..595053e 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -64,6 +64,8 @@ class AutoSubmit end def get_argvs_for_competition(real_argvs, mirror_dir, branch, commit_id) + return if branch == 'master' + argvs_for_base = Array.new(real_argvs) author_email = %x(git -C #{mirror_dir} log --format=%ae -1 #{commit_id}).chomp committer_email = %x(git -C #{mirror_dir} log --format=%ce -1 #{commit_id}).chomp -- 2.23.0

1 0

[PATCH lkp-tests] pkg/mysql-8-0-20: enable make install gcc
by Luan Shengde 24 Apr '21

24 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- pkg/mysql-8-0-20/PKGBUILD | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/mysql-8-0-20/PKGBUILD b/pkg/mysql-8-0-20/PKGBUILD index 154f936cf..134a50096 100644 --- a/pkg/mysql-8-0-20/PKGBUILD +++ b/pkg/mysql-8-0-20/PKGBUILD @@ -12,7 +12,7 @@ md5sums=('SKIP' 'SKIP' '99c4965aa870c475000993f52a0dee66') build() { - # build_gcc + build_gcc build_mysql } @@ -48,7 +48,8 @@ build_gcc() --with-boot-ldflags='-Wl,-z,relro,-z,now' \ --with-multilib-list=lp64 make -j96 - make install + make install & + wait } } -- 2.23.0

1 0

[PATCH compass-ci 3/3] container/extrac-stats/Dockerfile: add terminal-table
by Lu Weitao 24 Apr '21

24 Apr '21

[Why] the ruby-dev need terminal-table Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/extract-stats/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/extract-stats/Dockerfile b/container/extract-stats/Dockerfile index 08abc8c2..30bc865d 100644 --- a/container/extract-stats/Dockerfile +++ b/container/extract-stats/Dockerfile @@ -10,7 +10,7 @@ RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweiclou RUN umask 002 && \ gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ - gem install rest-client activesupport git json yaml elasticsearch + gem install rest-client activesupport git json yaml elasticsearch terminal-table ENV RUNTIME_DIR /c/cci/extract -- 2.23.0

1 0

[PATCH compass-ci 2/3] sbin/mail-job: mail compare result to author and committer
by Lu Weitao 24 Apr '21

24 Apr '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/mail_job_result.rb | 55 +++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/lib/mail_job_result.rb b/lib/mail_job_result.rb index e5884705..752b8dcf 100644 --- a/lib/mail_job_result.rb +++ b/lib/mail_job_result.rb @@ -4,6 +4,7 @@ require_relative 'mail_client.rb' require_relative 'es_query.rb' +require_relative 'compare.rb' require_relative 'constants.rb' require 'json' @@ -16,27 +17,44 @@ class MailJobResult end def send_mail - json = compose_mail.to_json - MailClient.new.send_mail(json) + data = compose_mail + return nil unless data + MailClient.new.send_mail_encode(data) end def compose_mail set_submitter_info - subject = "[Compass-CI] job: #{@job_id} result" + context = get_compare_result(@job) + return nil unless context + + subject = "[Compass-CI] job: #{@job_id} compare result" signature = "Regards\nCompass-CI\nhttps://gitee.com/openeuler/compass-ci" - body = "Hi, - Thanks for your participation in Kunpeng and software ecosystem! + + data = <<~BODY + To: #{@email_to} + Bcc: #{@email_cc} + Subject: #{subject} + + Hi, + + html_part: Thanks for your participation in Kunpeng and software ecosystem! Your Job: #{@job_id} had finished. - Please check job result: http://#{@result_host}:#{@result_port}#{@result_root}\n\n#{signature}" - { 'to' => @submitter_email, 'body' => body, 'subject' => subject } + Bellow are comparsion result of the base_commit: + #{context.to_s} + \n\n#{signature}" + BODY + + data end def set_submitter_info - job = query_job - exit unless job && job['email'] + sleep 10 + @job = query_job + exit unless @job && @job['author_email'] - @submitter_email = job['email'] - @result_root = job['result_root'] + @email_to = @job['author_email'] + @email_cc = @job['committer_email'] + @result_root = @job['result_root'] end def query_job @@ -50,3 +68,18 @@ class MailJobResult query_result['hits']['hits'][0]['_source'] end end + +def get_compare_result(job) + min_samples = job['nr_num'].to_i + base_commit = job['base_commit'] + commit_id = job['commit_id'] + return nil unless base_commit && commit_id + + condition_list = [{'base_commit' => base_commit}, {'commit_id' => commit_id}] + options = { :min_samples => min_samples, :no_print => true} + + matrices_list, suite_list = create_matrices_list(condition_list, options[:min_samples]) + return nil if matrices_list.size < 2 + + compare_matrixes(matrices_list, suite_list, options: options) +end -- 2.23.0

1 0

[PATCH compass-ci 1/3] sbin/compare: support min_samples mode
by Lu Weitao 24 Apr '21

24 Apr '21

[Example] sbin/compare id=crystal.1686878,crystal.1686879 id=crystal.1686880,crystal.1686881 --min_samples 3 Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/compare.rb | 22 +++++++++++++++++++--- lib/matrix2.rb | 4 +++- sbin/compare | 5 +++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/lib/compare.rb b/lib/compare.rb index 7386caea..a6fd9f98 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -20,7 +20,13 @@ require 'yaml' def compare_matrices_list(argv, common_conditions, options) condition_list = parse_argv(argv, common_conditions) - matrices_list, suite_list = create_matrices_list(condition_list) + matrices_list, suite_list = create_matrices_list(condition_list, options[:min_samples]) + if matrices_list.size < 2 + return nil if options[:no_print] + + warn "matrix less than min_samples" + end + compare_matrixes(matrices_list, suite_list, options: options) end @@ -35,13 +41,15 @@ def parse_argv(argv, common_conditions) conditions end -def create_matrices_list(conditions) +def create_matrices_list(conditions, min_samples) matrices_list = [] suite_list = [] es = ESQuery.new conditions.each do |condition| query_results = es.multi_field_query(condition, desc_keyword: 'start_time') - matrix, suites = combine_query_data(query_results) + matrix, suites = combine_query_data(query_results, min_samples) + next unless matrix + matrices_list << matrix suite_list.concat(suites) end @@ -49,6 +57,14 @@ def create_matrices_list(conditions) return matrices_list, suite_list end +def matrices_verification(matrices, options) + if matrices.size < 2 + return nil if options[:no_print] + + warn "matrix less than min_samples" + end +end + # ------------------------------------------------------------------------------------------- # compare_group # - one condition only diff --git a/lib/matrix2.rb b/lib/matrix2.rb index 12245a80..2fd63243 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -101,8 +101,10 @@ end # input: query results from es_query # return: matrix -def combine_query_data(query_data) +def combine_query_data(query_data, min_sample) job_list = extract_jobs_list(query_data['hits']['hits']) + return nil if job_list.size < min_sample + create_matrix(job_list) end diff --git a/sbin/compare b/sbin/compare index 2ee32688..bcb66ccf 100755 --- a/sbin/compare +++ b/sbin/compare @@ -22,6 +22,7 @@ is_group = false dimensions = nil colorful = nil options = {} +options[:min_samples] = 1 template = nil opt_parser = OptionParser.new do |opts| @@ -48,6 +49,10 @@ opt_parser = OptionParser.new do |opts| colorful = theme end + opts.on('--min_samples min_samples', 'give a number to litmit the min_samples of matrix which used to compare') do |min_samples| + options[:min_samples] = min_samples.to_i + end + opts.on('-t', '--template template', 'compare with user-defined template') do |t| template = t end -- 2.23.0

1 0

[PATCH lkp-tests] test: optimize mysql start command
by Luan Shengde 23 Apr '21

23 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/mysql-8-0-20 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mysql-8-0-20 b/tests/mysql-8-0-20 index 0c60ed9dd..990a053d7 100755 --- a/tests/mysql-8-0-20 +++ b/tests/mysql-8-0-20 @@ -54,8 +54,8 @@ create_database() start_mysql() { - "$mysqld" --defaults-file=/etc/my.cnf --port="$mysql_port" - sleep 20 + "$mysql_server" start & + wait } main() -- 2.23.0

1 0

[PATCH v3 compass-ci] sbin/download-rootfs: download osimage
by Liu Yinsi 23 Apr '21

23 Apr '21

the default os_mount=initramfs, so download osimage to run qemu kvm. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/download-rootfs | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/sbin/download-rootfs b/sbin/download-rootfs index 68a68039..cea416c0 100755 --- a/sbin/download-rootfs +++ b/sbin/download-rootfs @@ -19,22 +19,35 @@ os_version=${3:-20.03} load_cci_defaults rootfs_dir=/srv/os/${os}/${os_arch}/${os_version} +osimage_dir=/srv/initrd/osimage/${os}/${os_arch}/${os_version} -download_os_rootfs() -{ - mkdir -p $rootfs_dir - cd $rootfs_dir +mkdir -p $rootfs_dir $osimage_dir - download_url="http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" - wget $download_url +wget_download() +{ + local download_url=$1 + wget $download_url + + [ "$?" -ne 0 ] && { + log_error "failed to download os rootfs: $download_url" + exit 1 + } +} - [ "$?" -ne 0 ] && { - log_error "failed to download os rootfs: $download_url" - exit 1 - } +download_os_rootfs() +{ + cd $rootfs_dir + wget_download "http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" + + gzip -dc ${os_version}.cgz | cpio -id + rm -rf ${os_version}.cgz +} - gzip -dc ${os_version}.cgz | cpio -id - rm -rf ${os_version}.cgz +download_osimage() +{ + cd $osimage_dir + wget_download "http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/initrd/osimage/…" + wget_download "http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/initrd/deps/nfs…" } is_virtual() @@ -42,4 +55,4 @@ is_virtual() dmidecode -s system-product-name | grep -iq "virtual" } -is_virtual || download_os_rootfs +is_virtual || ( download_os_rootfs && download_osimage ) -- 2.23.0

1 0

[PATCH v3 compass-ci] container/scheduler-http: fix unfriendly error
by Liu Yinsi 23 Apr '21

23 Apr '21

before: /c/compass-ci/container/scheduler-https# ./start Traceback (most recent call last): ./start:14:in `<main>': CA certificate is missing. Ignore if the HTTPS scheduler is not required (RuntimeError) after: /c/compass-ci/container/scheduler-https# ./start CA certificate is missing. Ignore if the HTTPS scheduler is not required Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/scheduler-https/start | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/container/scheduler-https/start b/container/scheduler-https/start index 4ec84a28..5e79d2be 100755 --- a/container/scheduler-https/start +++ b/container/scheduler-https/start @@ -10,8 +10,10 @@ ca_path = '/etc/compass-ci/ca' crt_path = "#{ca_path}/server.crt" key_path = "#{ca_path}/server.key" -warn_msg = 'CA certificate is missing. Ignore if the HTTPS scheduler is not required' -raise warn_msg unless File.exist?(crt_path) && File.exist?(key_path) +unless File.exist?(crt_path) && File.exist?(key_path) + puts 'CA certificate is missing. Ignore if the HTTPS scheduler is not required' + exit +end names = Set.new %w[ SCHED_HOST -- 2.23.0

1 0

[PATCH v5 lkp-tests] tests/qcow2rootfs-x86_64: fix failed to build container using absolute path
by Wang Chenglong 23 Apr '21

23 Apr '21

[Error] root@2288hv5-2s44p-384g--b6 /c/lkp-tests/tests# /c/compass-ci/container/qcow2rootfs/build /c/compass-ci/container/qcow2rootfs/build: line 5: ../defconfig.sh: No such file or directory /c/compass-ci/container/qcow2rootfs/build: line 7: docker_skip_rebuild: command not found Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/qcow2rootfs-x86_64 | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/qcow2rootfs-x86_64 b/tests/qcow2rootfs-x86_64 index f698fb7ec..a783dc1a6 100755 --- a/tests/qcow2rootfs-x86_64 +++ b/tests/qcow2rootfs-x86_64 @@ -9,12 +9,17 @@ set_qcow2rootfs_tools() { export QCOW2_PATH="/c/compass-ci/container/qcow2rootfs" - git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci || return 1 + export CCI_SRC="/c/compass-ci" + git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci rm ${QCOW2_PATH}/root/etc/yum.repos.d/* wget -O ${QCOW2_PATH}/root/etc/yum.repos.d/CentOS-Base.repo https://repo.huaweicloud.com/repository/conf/CentOS-7-reg.repo - ${QCOW2_PATH}/build + cd ${QCOW2_PATH} + ./build || { + echo "build container failed~" + exit 1 + } sed -i 's#$EXTRACT_ROOT/bin/extract $QCOW2_FILE $tar_out_file#$EXTRACT_ROOT/bin/extract $EXTRACT_ROOT/qcow2-dir/$QCOW2_NAME $tar_out_file#' \ - ${QCOW2_PATH}/bin/mail + ${QCOW2_PATH}/bin/main } @@ -52,7 +57,13 @@ extract_rootfs() export ROOTFS_DIR="${qcow2_version}-$(date +%Y-%m-%d-%H-%M-%S)" export ROOTFS_PATH="${MOUNT_POINT}/openeuler/${qcow2_arch}/automatic-build/${ROOTFS_DIR}" mkdir -p $ROOTFS_PATH - ${QCOW2_PATH}/run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} + cd $CCI_SRC/container/dracut-initrd + ./build || { + echo "build container failed~" + exit 1 + } + cd ${QCOW2_PATH} + ./run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} } set_resolv_config() -- 2.23.0

1 0

[PATCH v4 lkp-tests] tests/qcow2rootfs-x86_64: fix failed to build container using absolute path
by Wang Chenglong 23 Apr '21

23 Apr '21

[Error] root@2288hv5-2s44p-384g--b6 /c/lkp-tests/tests# /c/compass-ci/container/qcow2rootfs/build /c/compass-ci/container/qcow2rootfs/build: line 5: ../defconfig.sh: No such file or directory /c/compass-ci/container/qcow2rootfs/build: line 7: docker_skip_rebuild: command not found Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/qcow2rootfs-x86_64 | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/qcow2rootfs-x86_64 b/tests/qcow2rootfs-x86_64 index f698fb7ec..792a80fb9 100755 --- a/tests/qcow2rootfs-x86_64 +++ b/tests/qcow2rootfs-x86_64 @@ -9,12 +9,17 @@ set_qcow2rootfs_tools() { export QCOW2_PATH="/c/compass-ci/container/qcow2rootfs" - git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci || return 1 + export CCI_SRC="/c/compass-ci" + git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci rm ${QCOW2_PATH}/root/etc/yum.repos.d/* wget -O ${QCOW2_PATH}/root/etc/yum.repos.d/CentOS-Base.repo https://repo.huaweicloud.com/repository/conf/CentOS-7-reg.repo - ${QCOW2_PATH}/build + cd ${QCOW2_PATH} + ./build || { + echo "build container failed~" + exit 1 + } sed -i 's#$EXTRACT_ROOT/bin/extract $QCOW2_FILE $tar_out_file#$EXTRACT_ROOT/bin/extract $EXTRACT_ROOT/qcow2-dir/$QCOW2_NAME $tar_out_file#' \ - ${QCOW2_PATH}/bin/mail + ${QCOW2_PATH}/bin/main } @@ -51,15 +56,20 @@ extract_rootfs() { export ROOTFS_DIR="${qcow2_version}-$(date +%Y-%m-%d-%H-%M-%S)" export ROOTFS_PATH="${MOUNT_POINT}/openeuler/${qcow2_arch}/automatic-build/${ROOTFS_DIR}" + cd $CCI_SRC/container/dracut-initrd + ./build || { + echo "build container failed~" + exit 1 + } mkdir -p $ROOTFS_PATH - ${QCOW2_PATH}/run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} + ${QCOW2_PATH}/run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} } set_resolv_config() { - cat >${ROOTFS_PATH}/etc/resolv.conf<<-EOF - nameserver 114.114.114.114 - EOF + cat >${ROOTFS_PATH}/etc/resolv.conf<<-EOF + nameserver 114.114.114.114 + EOF } set_link() @@ -71,6 +81,7 @@ set_link() echo $ROOTFS_PATH } + set_qcow2rootfs_tools check_url download_qcow2_img -- 2.23.0

1 1

[PATCH compass-ci] sbin: configuring ES authentication information
by Wu Zhende 23 Apr '21

23 Apr '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- sbin/es-accounts-mapping.sh | 8 ++++++-- sbin/es-jobs-mapping.sh | 10 +++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/sbin/es-accounts-mapping.sh b/sbin/es-accounts-mapping.sh index c4ac2fc..580b459 100755 --- a/sbin/es-accounts-mapping.sh +++ b/sbin/es-accounts-mapping.sh @@ -2,15 +2,19 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +. $CCI_SRC/container/defconfig.sh + +load_service_authentication + # check whether accounts index has created -status_code=$(curl -sSIL -w "%{http_code}\\n" -o /dev/null http://localhost:9200/accounts) +status_code=$(curl -sSIL -u "${ES_USER}:${ES_PASSWORD}" -w "%{http_code}\\n" -o /dev/null http://localhost:9200/accounts) if [ "$status_code" -eq 200 ] then echo "accounts index has been created, exit." else echo "begin create index." - curl -sSH 'Content-Type: Application/json' -XPUT 'http://localhost:9200/accounts' -d '{ + curl -sSH 'Content-Type: Application/json' -XPUT 'http://localhost:9200/accounts' -u "${ES_USER}:${ES_PASSWORD}" -d '{ "mappings": { "dynamic": false, "properties": { diff --git a/sbin/es-jobs-mapping.sh b/sbin/es-jobs-mapping.sh index 67b1ed1..15000d9 100755 --- a/sbin/es-jobs-mapping.sh +++ b/sbin/es-jobs-mapping.sh @@ -2,6 +2,10 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +. $CCI_SRC/container/defconfig.sh + +load_service_authentication + # Determine whether curl is installed. If not, install curl. if ! [ -x "$(command -v curl)" ] then @@ -28,14 +32,14 @@ then fi # Determine whether jobs index has created -status_code=$(curl -sSIL -w "%{http_code}\n" -o /dev/null http://localhost:9200/jobs) +status_code=$(curl -sSIL -u "${ES_USER}:${ES_PASSWORD}" -w "%{http_code}\n" -o /dev/null http://localhost:9200/jobs) if [ $status_code -eq 200 ] then echo "jobs index has create, exit." else echo "jobs index not exists, begin create index." - curl -sSH 'Content-Type: Application/json' -XPUT 'http://localhost:9200/jobs' -d '{ + curl -sSH 'Content-Type: Application/json' -XPUT 'http://localhost:9200/jobs' -u "${ES_USER}:${ES_PASSWORD}" -d '{ "mappings": { "dynamic": false, "dynamic_templates": [ @@ -185,7 +189,7 @@ else echo "create jobs index failed." else echo "set index.mapping.total_fields.limit: 10000" - curl -sS -XPUT 127.0.0.1:9200/jobs/_settings -H 'Content-Type: application/json' \ + curl -sS -XPUT 127.0.0.1:9200/jobs/_settings -u "${ES_USER}:${ES_PASSWORD}" -H 'Content-Type: application/json' \ -d '{"index.mapping.total_fields.limit": 10000}' fi fi -- 2.23.0

1 0

[PATCH lkp-tests 3/3] tests/qcow2rootfs-x86_64: fix failed to build container using absolute path
by Wang Chenglong 23 Apr '21

23 Apr '21

[Error] root@2288hv5-2s44p-384g--b6 /c/lkp-tests/tests# /c/compass-ci/container/qcow2rootfs/build /c/compass-ci/container/qcow2rootfs/build: line 5: ../defconfig.sh: No such file or directory /c/compass-ci/container/qcow2rootfs/build: line 7: docker_skip_rebuild: command not found Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/qcow2rootfs-x86_64 | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/qcow2rootfs-x86_64 b/tests/qcow2rootfs-x86_64 index f698fb7ec..6826e39ef 100755 --- a/tests/qcow2rootfs-x86_64 +++ b/tests/qcow2rootfs-x86_64 @@ -9,12 +9,17 @@ set_qcow2rootfs_tools() { export QCOW2_PATH="/c/compass-ci/container/qcow2rootfs" + export CCI_SRC="/c/compass-ci" git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci || return 1 rm ${QCOW2_PATH}/root/etc/yum.repos.d/* wget -O ${QCOW2_PATH}/root/etc/yum.repos.d/CentOS-Base.repo https://repo.huaweicloud.com/repository/conf/CentOS-7-reg.repo - ${QCOW2_PATH}/build + cd ${QCOW2_PATH} + ./build || { + echo "build container failed~" + exit 1 + } sed -i 's#$EXTRACT_ROOT/bin/extract $QCOW2_FILE $tar_out_file#$EXTRACT_ROOT/bin/extract $EXTRACT_ROOT/qcow2-dir/$QCOW2_NAME $tar_out_file#' \ - ${QCOW2_PATH}/bin/mail + ${QCOW2_PATH}/bin/main } @@ -51,15 +56,20 @@ extract_rootfs() { export ROOTFS_DIR="${qcow2_version}-$(date +%Y-%m-%d-%H-%M-%S)" export ROOTFS_PATH="${MOUNT_POINT}/openeuler/${qcow2_arch}/automatic-build/${ROOTFS_DIR}" + cd $CCI_SRC/container/dracut-initrd + ./build || { + echo "build container failed~" + exit 1 + } mkdir -p $ROOTFS_PATH - ${QCOW2_PATH}/run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} + ${QCOW2_PATH}/run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} || exit 1 } set_resolv_config() { - cat >${ROOTFS_PATH}/etc/resolv.conf<<-EOF - nameserver 114.114.114.114 - EOF + cat >${ROOTFS_PATH}/etc/resolv.conf<<-EOF + nameserver 114.114.114.114 + EOF } set_link() @@ -71,6 +81,7 @@ set_link() echo $ROOTFS_PATH } + set_qcow2rootfs_tools check_url download_qcow2_img -- 2.23.0

2 2

[PATCH lkp-tests 1/3] install-iso-pre: add depend pkg: wget
by Yu Chuan 23 Apr '21

23 Apr '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- distro/depends/install-iso-pre | 1 + 1 file changed, 1 insertion(+) diff --git a/distro/depends/install-iso-pre b/distro/depends/install-iso-pre index 4569e1e3cdc0..7a66195e44fd 100644 --- a/distro/depends/install-iso-pre +++ b/distro/depends/install-iso-pre @@ -1,3 +1,4 @@ lvm2 ruby git +wget -- 2.23.0

1 0

[PATCH v2 compass-ci] sbin/download-rootfs: download osimage
by Liu Yinsi 23 Apr '21

23 Apr '21

the default os_mount=initramfs, so download osimage to run qemu kvm. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/download-rootfs | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/sbin/download-rootfs b/sbin/download-rootfs index 68a68039..d936dbe6 100755 --- a/sbin/download-rootfs +++ b/sbin/download-rootfs @@ -19,22 +19,35 @@ os_version=${3:-20.03} load_cci_defaults rootfs_dir=/srv/os/${os}/${os_arch}/${os_version} +osimage_dir=/srv/initrd/osimage/${os}/${os_arch}/${os_version} -download_os_rootfs() -{ - mkdir -p $rootfs_dir - cd $rootfs_dir +mkdir -p $rootfs_dir $osimage_dir - download_url="http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" - wget $download_url +download_one() +{ + local download_url=$1 + wget $download_url + + [ "$?" -ne 0 ] && { + log_error "failed to download os rootfs: $download_url" + exit 1 + } +} - [ "$?" -ne 0 ] && { - log_error "failed to download os rootfs: $download_url" - exit 1 - } +download_os_rootfs() +{ + cd $rootfs_dir + download_one "http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" + + gzip -dc ${os_version}.cgz | cpio -id + rm -rf ${os_version}.cgz +} - gzip -dc ${os_version}.cgz | cpio -id - rm -rf ${os_version}.cgz +download_osimage() +{ + cd $osimage_dir + download_one "http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/initrd/osimage/…" + download_one "http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/initrd/deps/nfs…" } is_virtual() @@ -43,3 +56,4 @@ is_virtual() } is_virtual || download_os_rootfs +is_virtual || download_osimage -- 2.23.0

1 0

[PATCH v2 compass-ci] container/scheduler-http: fix unfriendly error
by Liu Yinsi 23 Apr '21

23 Apr '21

before: /c/compass-ci/container/scheduler-https# ./start Traceback (most recent call last): ./start:14:in `<main>': CA certificate is missing. Ignore if the HTTPS scheduler is not required (RuntimeError) after: /c/compass-ci/container/scheduler-https# ./start CA certificate is missing. Ignore if the HTTPS scheduler is not required Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/scheduler-https/start | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/container/scheduler-https/start b/container/scheduler-https/start index 4ec84a28..374300c0 100755 --- a/container/scheduler-https/start +++ b/container/scheduler-https/start @@ -11,7 +11,10 @@ crt_path = "#{ca_path}/server.crt" key_path = "#{ca_path}/server.key" warn_msg = 'CA certificate is missing. Ignore if the HTTPS scheduler is not required' -raise warn_msg unless File.exist?(crt_path) && File.exist?(key_path) +unless File.exist?(crt_path) && File.exist?(key_path) + puts warn_msg + exit +end names = Set.new %w[ SCHED_HOST -- 2.23.0

1 0

[PATCH compass-ci] container/es: support to build es images in x86_64 machine
by Liu Yinsi 23 Apr '21

23 Apr '21

choose base image according to different arch. arch: aarch64 => base image: elasticsearch:7.11.1@sha256:d52cda1e73d1b1915ba2d76ca1e426620c7b5d6942d9d2f432259503974ba786 arch: x86_64 => base image: elasticsearch:7.11.1 Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/es/Dockerfile | 3 ++- container/es/build | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/container/es/Dockerfile b/container/es/Dockerfile index f29e677e..bc320076 100644 --- a/container/es/Dockerfile +++ b/container/es/Dockerfile @@ -1,7 +1,8 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM elasticsearch:7.11.1@sha256:d52cda1e73d1b1915ba2d76ca1e426620c7b5d6942d9d2f432259503974ba786 +ARG BASE_IMAGE +FROM ${BASE_IMAGE} ARG MEMORY ARG USER diff --git a/container/es/build b/container/es/build index e0232270..73593f4b 100755 --- a/container/es/build +++ b/container/es/build @@ -14,6 +14,12 @@ names = Set.new %w[ defaults = relevant_service_authentication(names) docker_skip_rebuild "es:7.11.1" +BASE_IMAGE_DICT = { + 'aarch64' => 'elasticsearch:7.11.1@sha256:d52cda1e73d1b1915ba2d76ca1e426620c7b5d6942d9d2f432259503974ba786', + 'x86_64' => 'elasticsearch:7.11.1' }.freeze + +BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] + available_memory = get_available_memory -system "docker build -t es:7.11.1 --build-arg MEMORY=#{available_memory} --build-arg USER=#{defaults['ES_USER']} --build-arg PASSWORD=#{defaults['ES_PASSWORD']} --network=host ." +system "docker build -t es:7.11.1 --build-arg MEMORY=#{available_memory} --build-arg USER=#{defaults['ES_USER']} --build-arg PASSWORD=#{defaults['ES_PASSWORD']} --build-arg BASE_IMAGE=#{BASE_IMAGE} --network=host ." -- 2.23.0

1 0

[PATCH compass-ci] container/kibana: configuring ES authentication information
by Wu Zhende 23 Apr '21

23 Apr '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana/Dockerfile | 7 +++++-- container/kibana/build | 10 +++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/container/kibana/Dockerfile b/container/kibana/Dockerfile index 970eb5a..7085618 100644 --- a/container/kibana/Dockerfile +++ b/container/kibana/Dockerfile @@ -5,8 +5,11 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -# docker image borrowed from hub.docker.com/r/gagara/kibana-oss-arm64 +ARG USER +ARG PASSWORD MAINTAINER Wu Zhende <wuzhende666(a)163.com> -RUN sed -i 's/server.host: "0"/server.host: "0.0.0.0"/' config/kibana.yml +RUN sed -i 's/server.host: "0"/server.host: "0.0.0.0"/' config/kibana.yml && \ + sed -i "6i\elasticsearch.username: ${USER}" config/kibana.yml && \ + sed -i "6i\elasticsearch.password: ${PASSWORD}" config/kibana.yml diff --git a/container/kibana/build b/container/kibana/build index 9c75aeb..c7bcd40 100755 --- a/container/kibana/build +++ b/container/kibana/build @@ -3,8 +3,16 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig' +names = Set.new %w[ + ES_USER + ES_PASSWORD +] + +defaults = relevant_service_authentication(names) + docker_skip_rebuild "kibana:7.11.1" BASE_IMAGE_DICT = { @@ -13,4 +21,4 @@ BASE_IMAGE_DICT = { BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] -system "docker build -t kibana:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} ." +system "docker build -t kibana:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg USER=#{defaults['ES_USER']} --build-arg PASSWORD=#{defaults['ES_PASSWORD']} ." -- 2.23.0

1 0

[PATCH v2 compass-ci] container/etcd: adapt to the x86_64
by Cao Xueliang 23 Apr '21

23 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/etcd/Dockerfile | 9 +++++---- container/etcd/build | 11 ++++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/container/etcd/Dockerfile b/container/etcd/Dockerfile index f4746ab..1612dbd 100644 --- a/container/etcd/Dockerfile +++ b/container/etcd/Dockerfile @@ -6,12 +6,13 @@ FROM alpine:3.13 MAINTAINER Cao Xueliang <caoxl78320(a)163.com> ARG VERSION +ARG ARCH RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories && \ apk update && apk add ca-certificates && \ - wget https://repo.huaweicloud.com/etcd/${VERSION}/etcd-${VERSION}-linux-arm64.ta… && \ - tar zxvf etcd-${VERSION}-linux-arm64.tar.gz && \ - mv etcd-${VERSION}-linux-arm64/etcd* /bin/ && \ - rm -Rf etcd-${VERSION}-linux-arm64* /var/cache/apk/* + wget https://repo.huaweicloud.com/etcd/${VERSION}/etcd-${VERSION}-linux-${ARCH}.… && \ + tar zxvf etcd-${VERSION}-linux-${ARCH}.tar.gz && \ + mv etcd-${VERSION}-linux-${ARCH}/etcd* /bin/ && \ + rm -Rf etcd-${VERSION}-linux-${ARCH}* /var/cache/apk/* VOLUME /data diff --git a/container/etcd/build b/container/etcd/build index 37c2326..58def3d 100755 --- a/container/etcd/build +++ b/container/etcd/build @@ -4,7 +4,12 @@ . ../defconfig.sh -version=v3.4.15 -docker_skip_rebuild "etcd-${version}:alpine" +VERSION=v3.4.15 +docker_skip_rebuild "etcd-${VERSION}:alpine" -docker build -t etcd-${version}:alpine --build-arg VERSION=${version} . +[ "$(arch)" == "aarch64" ] && ARCH=arm64 +[ "$(arch)" == "x86_64" ] && ARCH=amd64 + +docker_skip_rebuild "etcd-${VERSION}:alpine" + +docker build -t etcd-${VERSION}:alpine --build-arg VERSION=${VERSION} --build-arg ARCH=${ARCH} . -- 2.23.0

1 0

[PATCH v2 lkp-tests] jobs: add keys for jobs
by Luan Shengde 23 Apr '21

23 Apr '21

add key for specify mysql socket Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/mysql-8-0-20.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/mysql-8-0-20.yaml b/jobs/mysql-8-0-20.yaml index 1926c5547..8c30f963e 100644 --- a/jobs/mysql-8-0-20.yaml +++ b/jobs/mysql-8-0-20.yaml @@ -32,3 +32,4 @@ sysbench-mysql: oltp_read_only: false report_interval: 1 events: 20000 + mysql_socket: /data/mysql/run/mysql.sock -- 2.23.0

1 0

[PATCH lkp-tests 5/6] tests: fix bugs
by Luan Shengde 23 Apr '21

23 Apr '21

del delete line: skip-grant-tables it will lead to access denied error. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/mysql-8-0-20 | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/mysql-8-0-20 b/tests/mysql-8-0-20 index ebc7c0e90..0c60ed9dd 100755 --- a/tests/mysql-8-0-20 +++ b/tests/mysql-8-0-20 @@ -40,7 +40,6 @@ configure_mysql() sleep 10 "$mysql" -u "$mysql_user" -e "flush privileges; use mysql; alter user 'root'@'localhost' identified by '$mysql_password'; flush privileges;" - sed -i '/^skip-grant-tables/d' /etc/my.cnf "$mysql_server" restart sleep 10 } -- 2.23.0

2 2

[PATCH lkp-tests 4/6] depends: add depends list file for installing mysql
by Luan Shengde 23 Apr '21

23 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- distro/depends/mysql-8-0-20-dev | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 distro/depends/mysql-8-0-20-dev diff --git a/distro/depends/mysql-8-0-20-dev b/distro/depends/mysql-8-0-20-dev new file mode 100644 index 000000000..23c6ff661 --- /dev/null +++ b/distro/depends/mysql-8-0-20-dev @@ -0,0 +1,16 @@ +libnuma1 +xfsprogs +ethtool +gcc +cmake +libssl-dev +libncurses5-dev +libtirpc-dev +rpcsvc-proto-devel +rpcgen +gmp +gmp-devel +libmpc +libmpc-devel +mpfr +mpfr-devel -- 2.23.0

1 1

[PATCH lkp-tests 3/6] tests: add initialize file for mysql
by Luan Shengde 23 Apr '21

23 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/mysql-8-0-20 | 73 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 tests/mysql-8-0-20 diff --git a/tests/mysql-8-0-20 b/tests/mysql-8-0-20 new file mode 100755 index 000000000..ebc7c0e90 --- /dev/null +++ b/tests/mysql-8-0-20 @@ -0,0 +1,73 @@ +#!/bin/sh +# - data_dev +# mysql_user +# mysql_host +# mysql_port +# mysql_db +# mysql_password +. $LKP_SRC/lib/common.sh + +: "${mysql_user:=root}" +: "${mysql_host:=127.0.0.1}" +: "${mysql_port:=3306}" +: "${mysql_db:=sysbench_test}" +: "${mysql_password:=mysql_password}" + +create_basedir() +{ + mkdir -p /data + mkfs.xfs -f /dev/"$data_dev" + mount /dev/"$data_dev" /data + + mkdir -p /data/mysql/{data,share,tmp,run,log} + + useradd mysql + touch /data/mysql/log/mysql.log + chown -R mysql:mysql /data + chown -R mysql:mysql /data/mysql/log/mysql.log +} + +configure_mysql() +{ + if [ ! -f $mysqld ] || [ ! -f $mysql ] || [ ! -f $mysql_server ]; then + echo "mysql is not installed!" && exit 1 + fi + + "$mysqld" --user=mysql --datadir=/data/mysql/data --basedir=/data/mysql --initialize --port="$mysql_port" + sleep 10 + + "$mysql_server" start + sleep 10 + + "$mysql" -u "$mysql_user" -e "flush privileges; use mysql; alter user 'root'@'localhost' identified by '$mysql_password'; flush privileges;" + sed -i '/^skip-grant-tables/d' /etc/my.cnf + "$mysql_server" restart + sleep 10 +} + +create_database() +{ + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "use mysql; update user set host ='%' where user='root'; flush privileges;" + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $mysql_db;" + "$mysql_server" stop + sleep 10 +} + +start_mysql() +{ + "$mysqld" --defaults-file=/etc/my.cnf --port="$mysql_port" + sleep 20 +} + +main() +{ + local mysql="/usr/local/mysql/bin/mysql" + local mysqld="/usr/local/mysql/bin/mysqld" + local mysql_server="/usr/local/mysql/support-files/mysql.server" + + create_basedir + configure_mysql + create_database + start_mysql +} +main -- 2.23.0

1 1

[PATCH lkp-tests 2/6] add pkgbuild file
by Luan Shengde 23 Apr '21

23 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- pkg/mysql-8-0-20/PKGBUILD | 79 +++++++++++++++++++++++++++++++++++++++ pkg/mysql-8-0-20/my.cnf | 69 ++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 pkg/mysql-8-0-20/PKGBUILD create mode 100644 pkg/mysql-8-0-20/my.cnf diff --git a/pkg/mysql-8-0-20/PKGBUILD b/pkg/mysql-8-0-20/PKGBUILD new file mode 100644 index 000000000..154f936cf --- /dev/null +++ b/pkg/mysql-8-0-20/PKGBUILD @@ -0,0 +1,79 @@ +pkgname=mysql-server-mysql +pkgver=8.0.20 +gccdir=gcc +pkgrel=1 +arch=('aarch64' 'x86_64') +url="http://www.mysql.com" +license=('GPLv2') +source=("https://gitee.com/li-yancheng/my-sql_with_boostkit_patch.git" + "https://gitee.com/li-yancheng/gcc.git" + "my.cnf") +md5sums=('SKIP' 'SKIP' '99c4965aa870c475000993f52a0dee66') + +build() +{ + # build_gcc + build_mysql +} + +build_gcc() +{ + mkdir -p $HOME/rpmbuild/SOURCES/ && { + cd $srcdir/gcc && cp -r * $HOME/rpmbuild/SOURCES/ + } + + cd $HOME/rpmbuild/SOURCES/ && { + rpmbuild -bp gcc.spec + } + + cd $HOME/rpmbuild/BUILD/gcc-9.3.0 && { + ./configure --prefix=/usr/ \ + --enable-shared \ + --enable-threads=posix \ + --enable-checking=release \ + --with-system-zlib \ + --enable-__cxa_atexit \ + --disable-libunwind-exceptions \ + --enable-gnu-unique-object \ + --enable-linker-build-id \ + --with-linker-hash-style=gnu \ + --enable-languages=c,c++,objc,obj-c++,fortran,lto \ + --enable-plugin \ + --enable-initfini-array \ + --disable-libgcj \ + --without-isl \ + --without-cloog \ + --enable-gnu-indirect-function \ + --with-stage1-ldflags='-Wl,-z,relro,-z,now' \ + --with-boot-ldflags='-Wl,-z,relro,-z,now' \ + --with-multilib-list=lp64 + make -j96 + make install + } +} + +build_mysql() +{ + unset CPPFLAGS + unset CFLAGS + unset CXXFLAGS + cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver + + CMAKE=cmake + grep -sqF "CentOS Linux release 7" /etc/centos-release && CMAKE=cmake3 + + mkdir -p build + cd build && { + $CMAKE .. -DBUILD_CONFIG=mysql_release -DCMAKE_INSTALL_PREFIX=/usr/local/mysql -DMYSQL_DATADIR=/data/mysql/data -DSYSCONFDIR=/etc -DWITH_BOOST=../boost/boost_1_70_0/ + make -j96 + } +} + +package() +{ + cp my.cnf /etc/ + + cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver/build && { + make install + } +} diff --git a/pkg/mysql-8-0-20/my.cnf b/pkg/mysql-8-0-20/my.cnf new file mode 100644 index 000000000..49e5bc3df --- /dev/null +++ b/pkg/mysql-8-0-20/my.cnf @@ -0,0 +1,69 @@ +[mysqld_safe] +log-error=/data/mysql/log/mysql.log +pid-file=/data/mysql/run/mysqld.pid +[mysqldump] +quick +[mysql] +[client] +# default-character-set=utf8mb4 +default-character-set=utf8 +socket=/data/mysql/run/mysql.sock + +[mysqld] +socket=/data/mysql/run/mysql.sock +tmpdir=/data/mysql/tmp +datadir=/data/mysql/data +basedir=/usr/local/mysql +default_authentication_plugin=mysql_native_password +user=mysql +memlock=on +bind-address=0.0.0.0 +skip-name-resolve +skip-grant-tables + +lc-messages=en_US +lc-messages-dir=/usr/local/mysql/share + +innodb_page_size=64k +max_connections=4000 +back_log=2048 +performance_schema=OFF +max_prepared_stmt_count=128000 + +#file +innodb_file_per_table +innodb_log_file_size=2048M +innodb_log_files_in_group=32 +innodb_open_files=10000 +table_open_cache_instances=64 + +#buffers +innodb_buffer_pool_size=128G +innodb_buffer_pool_instances=64 +innodb_log_buffer_size=2048M + +#tune +sync_binlog=0 +innodb_flush_log_at_trx_commit=0 +innodb_use_native_aio=1 +innodb_spin_wait_delay=160 +innodb_sync_spin_loops=20 +innodb_flush_method=O_DIRECT +innodb_io_capacity=30000 +innodb_io_capacity_max=40000 +innodb_lru_scan_depth=9000 +innodb_page_cleaners=1 +innodb_spin_wait_pause_multiplier=25 + +#perf special +innodb_flush_neighbors=0 +innodb_write_io_threads=1 +innodb_read_io_threads=1 +innodb_purge_threads=1 + +sql_mode=STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO,STRICT_ALL_TABLES + +#skop_log_bin +ssl=0 + +table_open_cache=15000 -- 2.23.0

1 1

[PATCH lkp-tests 1/6] jobs: add job file for installing/testing mysql
by Luan Shengde 23 Apr '21

23 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/mysql-8-0-20.yaml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 jobs/mysql-8-0-20.yaml diff --git a/jobs/mysql-8-0-20.yaml b/jobs/mysql-8-0-20.yaml new file mode 100644 index 000000000..1926c5547 --- /dev/null +++ b/jobs/mysql-8-0-20.yaml @@ -0,0 +1,34 @@ +suite: cci-makepkg +category: functional + +testbox: taishan200-2280-2s48p-256g--a18 + +os: openeuler +os_version: 20.03 +os_mount: initramfs + +mysql_user: +mysql_host: +mysql_port: +mysql_db: +mysql_password: +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +sysbench-mysql: + oltp_test_mode: complex + max_requests: 0 + mysql_table_engine: innodb + oltp_tables_count: 1000 + oltp_table_size: 100000 + nr_threads: 256 + runtime: 60 + rand_type: special + rand_spec_pct: 100 + oltp_read_only: false + report_interval: 1 + events: 20000 -- 2.23.0

1 1

[PATCH lkp-tests 6/6] tests: add specify mysql socket
by Luan Shengde 23 Apr '21

23 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/sysbench-mysql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/sysbench-mysql b/tests/sysbench-mysql index 0ade2dab9..d7b7fc148 100755 --- a/tests/sysbench-mysql +++ b/tests/sysbench-mysql @@ -14,6 +14,7 @@ # mysql_db # mysql_password # db_driver +# mysql_socket : "${mysql_user:=root}" : "${mysql_host:=$server}" @@ -30,6 +31,7 @@ : "${report_interval:=1}" : "${oltp_read_only:=off}" : "${events:=2000000000}" +: "${mysql_socket:=/data/mysql/run/mysql.sock}" args1=( --mysql-user=$mysql_user @@ -44,10 +46,12 @@ args1=( --rand-spec-pct=$rand_spec_pct --threads=$nr_threads --time=$runtime + --mysql-socket=$mysql_socket ) args2=( --mysql-user=$mysql_user + --mysql-socket=$mysql_socket --mysql-password=$mysql_password --mysql-host=$mysql_host --mysql-port=$mysql_port -- 2.23.0

1 0