April 2021 - Compass-ci - mailweb.openeuler.org

[PATCH v2 lkp-tests 2/2] lib/email.sh: add selftest_content()
by Liu Yinsi 22 Apr '21

22 Apr '21

This will send email to user when finish selftest. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- lib/email.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/email.sh b/lib/email.sh index 4f1055033..24ef7bb64 100755 --- a/lib/email.sh +++ b/lib/email.sh @@ -80,6 +80,22 @@ Compass-CI " } +selftest_content() +{ + email_content="To: $my_email +Subject: [NOTIFY Compass-CI] Self-test report + +Dear $my_name: + + Your self-test groud_id=selftest job result is: + + $job_state + +Regards +Compass-CI +" +} + send_email() { local subject=$1 -- 2.23.0

1 0

[PATCH compass-ci] contianer/register-accounts: add env ES_USER ES_PASSWORD
by Li Yuanchao 22 Apr '21

22 Apr '21

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/defconfig.rb | 1 + container/register-accounts/start | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/container/defconfig.rb b/container/defconfig.rb index f090351..0a8e4d5 100755 --- a/container/defconfig.rb +++ b/container/defconfig.rb @@ -11,6 +11,7 @@ def cci_defaults '/etc/compass-ci/service/*.yaml', '/etc/compass-ci/accounts/*.yaml', '/etc/compass-ci/register/*.yaml', + '/etc/compass-ci/*.yaml', "#{ENV['HOME']}/.config/compass-ci/defaults/*.yaml"]).each do |file| hash.update YAML.load_file(file) || {} end diff --git a/container/register-accounts/start b/container/register-accounts/start index c2acc75..ad70048 100755 --- a/container/register-accounts/start +++ b/container/register-accounts/start @@ -9,6 +9,8 @@ require_relative '../defconfig.rb' names = Set.new %w[ ES_HOST ES_PORT + ES_USER + ES_PASSWORD lab ] @@ -25,7 +27,7 @@ cmd = %w[ ] + env + %W[ -e CCI_SRC=#{DEFAULT_CCI} -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} - -v /etc/compass-ci/register:/etc/compass-ci/register:ro + -v /etc/compass-ci:/etc/compass-ci:ro -v /etc/localtime:/etc/localtime:ro -w #{DEFAULT_CCI}/container/register-accounts register-accounts -- 2.23.0

1 0

[PATCH lkp-tests 4/4] distro/depends: depends package file for compile installing mysql
by Luan Shengde 22 Apr '21

22 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- distro/depends/mysql-8-0-20-dev | 257 ++++++++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 distro/depends/mysql-8-0-20-dev diff --git a/distro/depends/mysql-8-0-20-dev b/distro/depends/mysql-8-0-20-dev new file mode 100644 index 000000000..2da6fc930 --- /dev/null +++ b/distro/depends/mysql-8-0-20-dev @@ -0,0 +1,257 @@ +make +bison +ncurses +ncurses-devel +libaio-devel +openssl +openssl-devel +gmp +gmp-devel +mpfr +mpfr-devel +libmpc +libmpc-devel +wget +tar +gcc +gcc-c++ +git +rpcgen +cmake +bison +bison-devel +m4 +libtirpc-devel +openldap-devel +protobuf +protobuf-devel +protobuf-lite-devel +dnf-plugins-core +rpm-build +asciidoc +autoconf +automake +bzip2-devel +curl-devel +desktop-file-utils +emacs +expat-devel +flex +gcc-gfortran +gdb +glib2-devel +gnutls-devel +gpm-devel +libX11-devel +libacl-devel +libdb-devel +libpsl-devel +libtool +lksctp-tools-devel +pcre-devel +perl-generators +policycoreutils +python2-devel +python2-setuptools +python3-devel +unixODBC-devel +util-linux-devel +xmlto +xz-devel +cracklib-devel +gpgme-devel +guile-devel +jsoncpp-devel +libarchive-devel +libidn2-devel +libmetalink-devel +libsecret-devel +libtool-devel +libuv-devel +perl-HTTP-Daemon +perl-MailTools +python2-google-apputils +python3-google-apputils +python3-sphinx +qt5-qtbase-devel +rhash-devel +texinfo +ModemManager-glib +SDL +adobe-mappings-cmap +adobe-mappings-cmap-lang +adobe-mappings-pdf +alsa-lib +avahi-glib +babeltrace +boost-regex +cdparanoia-libs +ctags +dbus-glib +dejavu-fonts +diffutils +docbook-dtds +docbook-style-xsl +emacs-common +enchant +fftw-libs-double +fontconfig-devel +freeglut +freetype-devel +gd +gdb-headless +geoclue2 +ghostscript +giflib +glew +google-droid-sans-fonts +gpm +graphviz +gstreamer1 +gstreamer1-plugins-base +hunspell +hunspell-en +hyphen +iso-codes +jbig2dec +libXau-devel +libXdamage-devel +libXext-devel +libXfixes-devel +libXv +libXxf86misc +libXxf86vm +libXxf86vm-devel +libattr-devel +libdrm +libdrm-devel +libffi-devel +libfontenc +libgfortran +libglvnd +libglvnd-devel +libgpg-error-devel +libidn +libijs +libnotify +libogg +libpaper +libpng-devel +librsvg2 +libtasn1-devel +libtheora +libvisual +libvorbis +libwebp +libxcb-devel +libxshmfence +libxslt +linuxconsoletools +lksctp-tools +mailcap +mcpp +mesa-demos +mesa-libEGL +mesa-libEGL-devel +mesa-libGL +mesa-libGL-devel +mesa-libGLU +mesa-libgbm +mesa-libglapi +nettle-devel +openjpeg2 +opus +orc +p11-kit-devel +perl-Encode-Locale +perl-Fedora-VSP +perl-HTTP-Date +perl-HTTP-Message +perl-IO-HTML +perl-IO-Socket-SSL +perl-LWP-MediaTypes +perl-Net-SSLeay +perl-Text-Unidecode +perl-TimeDate +perl-URI +python2 +python2-debug +python2-help +python2-six +python2-tkinter +python2-tools +python3-babel +python3-chardet +python3-idna +python3-jinja2 +python3-markupsafe +python3-pysocks +python3-pytz +python3-requests +python3-rpm-generators +python3-urllib3 +qt-settings +qt5-qtbase +qt5-qtbase-common +sgml-common +source-highlight +tk +unixODBC +urw-base35-bookman-fonts +urw-base35-c059-fonts +urw-base35-d050000l-fonts +urw-base35-fonts +urw-base35-fonts-common +urw-base35-gothic-fonts +urw-base35-nimbus-mono-ps-fonts +urw-base35-nimbus-roman-fonts +urw-base35-nimbus-sans-fonts +urw-base35-p052-fonts +urw-base35-standard-symbols-ps-fonts +urw-base35-z003-fonts +webkit2gtk3 +webkit2gtk3-jsc +woff2 +xcb-util +xorg-x11-font-utils +xorg-x11-fonts +xorg-x11-proto-devel +xorg-x11-server-utils +OpenEXR-libs +gc-devel +ilmbase +libwmf +perl-Class-Inspector +perl-Net-SMTP-SSL +perl-Unicode-EastAsianWidth +perl-libintl-perl +python-sphinx-locale +python2-dateutil +python2-gflags +python2-pytz +python3-docutils +python3-funcsigs +python3-gflags +python3-imagesize +python3-mock +python3-packaging +python3-pbr +python3-pygments +python3-snowballstemmer +python3-sphinx-theme-alabaster +python3-sphinx_rtd_theme +python3-sphinxcontrib-websupport +python3-sqlalchemy +python3-whoosh +qt5-qtbase-gui +qt5-rpm-macros +xcb-util-image +xcb-util-keysyms +xcb-util-renderutil +xcb-util-wm +fontawesome-fonts +lato-fonts +python2-pip +ImageMagick +perl-File-ShareDir +google-roboto-slab-fonts -- 2.23.0

2 2

[PATCH v2 lkp-tests] jobs: add job yaml file for building/testing mysql
by Luan Shengde 22 Apr '21

22 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/mysql-8-0-20.yaml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 jobs/mysql-8-0-20.yaml diff --git a/jobs/mysql-8-0-20.yaml b/jobs/mysql-8-0-20.yaml new file mode 100644 index 000000000..bf93ecdab --- /dev/null +++ b/jobs/mysql-8-0-20.yaml @@ -0,0 +1,34 @@ +suite: cci-makepkg +category: functional + +testbox: taishan200-2280-2s48p-256g--a18 + +os: openeuler +os_version: 20.03 +os_mount: initramfs + +mysql_user: root +mysql_host: localhost +mysql_port: 3306 +mysql_db: sysbench_test +mysql_password: mysql_password +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +sysbench-mysql: + oltp_test_mode: complex + max_requests: 0 + mysql_table_engine: innodb + oltp_tables_count: 1000 + oltp_table_size: 100000 + nr_threads: 256 + runtime: 60 + rand_type: special + rand_spec_pct: 100 + oltp_read_only: false + report_interval: 1 + events: 20000 -- 2.23.0

1 0

[PATCH lkp-tests] jobs/qcow2rootfs-x86_64.yaml: the yaml for qcow2rootfs
by Wang Chenglong 22 Apr '21

22 Apr '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- jobs/qcow2rootfs-x86_64.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 jobs/qcow2rootfs-x86_64.yaml diff --git a/jobs/qcow2rootfs-x86_64.yaml b/jobs/qcow2rootfs-x86_64.yaml new file mode 100644 index 000000000..1002283a3 --- /dev/null +++ b/jobs/qcow2rootfs-x86_64.yaml @@ -0,0 +1,27 @@ +suite: qcow2rootfs-x86_64 +category: benchmark +qcow2rootfs-x86_64: + + # The related fields about qcow2 image to be used to generate rootfs + # it should be the same as the official provided image name + qcow2_os: + qcow2_arch: + qcow2_version: + + # place the result rootfs related fields + # 1. result rootfs will be placed in the following localtion on the + # remote file server: + # - {remote_file_server}/{rootfs_path}/{iso_os}/{iso_arch}/ + # 2. remote file server protocols current supported: + # - nfs + rootfs_protocol: nfs + rootfs_server: 172.168.131.2 + rootfs_path: os-rw + +# The best host for execution is the x86 OpenEuler system +os: +os_version: +os_arch: +os_mount: +testbox: +queue: -- 2.23.0

1 0

[PATCH lkp-tests] lib/bootstrap.sh: delete history remove code
by Liu Shaofei 22 Apr '21

22 Apr '21

[why] function callend by "is_clearlinux" is not defined, by checking history commit record, the function is removed and the patch codes is removed. [ 63.427966] [ 63.428704] + is_clearlinux [ 63.428705] [ 63.431226] /lkp/lkp/src/lib/bootstrap.sh: line 104: is_clearlinux: command not found [ 63.431228] Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- lib/bootstrap.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 7983f638e..0bd01198e 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -101,13 +101,13 @@ network_ok() [ "$(cat $i/carrier 2>/dev/null)" = '1' ] && test_ping && return 0 done - is_clearlinux && { - net_devices_link up - - # in case of systemd-networkd.service was masked - systemctl unmask systemd-networkd.service - systemctl start systemd-networkd.service - } +# is_clearlinux && { +# net_devices_link up +# +# # in case of systemd-networkd.service was masked +# systemctl unmask systemd-networkd.service +# systemctl start systemd-networkd.service +# } return 1 } -- 2.23.0

1 0

[PATCH compass-ci] sparrow/0-package: add deps iputils
by Liu Yinsi 22 Apr '21

22 Apr '21

[why] ping command is used in $CCI_SRC/lib/check_public_network.sh Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/0-package/os/centos | 1 + sparrow/0-package/os/debian | 1 + sparrow/0-package/os/openEuler | 1 + 3 files changed, 3 insertions(+) diff --git a/sparrow/0-package/os/centos b/sparrow/0-package/os/centos index b8ef7694..d4d10457 100755 --- a/sparrow/0-package/os/centos +++ b/sparrow/0-package/os/centos @@ -25,6 +25,7 @@ pkgs=( wget curl sudo + iputils ) yum install -y --skip-broken "${pkgs[@]}" diff --git a/sparrow/0-package/os/debian b/sparrow/0-package/os/debian index 947cbdc1..6feb9be2 100755 --- a/sparrow/0-package/os/debian +++ b/sparrow/0-package/os/debian @@ -25,6 +25,7 @@ pkgs=( qemu-system-x86 qemu-system-riscv64 sudo + iputils ) apt-get update diff --git a/sparrow/0-package/os/openEuler b/sparrow/0-package/os/openEuler index e742fa13..8cdadd52 100755 --- a/sparrow/0-package/os/openEuler +++ b/sparrow/0-package/os/openEuler @@ -26,6 +26,7 @@ pkgs=( libvirt curl sudo + iputils ) public_network_ok && { -- 2.23.0

1 0

[PATCH lkp-tests] tests/qcow2rootfs-x86_64: automatically build x86_64 rootfs
by Wang Chenglong 22 Apr '21

22 Apr '21

[Why] automatically build rootfs for x86 from the official QCOW2 image. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/qcow2rootfs-x86_64 | 80 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 tests/qcow2rootfs-x86_64 diff --git a/tests/qcow2rootfs-x86_64 b/tests/qcow2rootfs-x86_64 new file mode 100755 index 000000000..f698fb7ec --- /dev/null +++ b/tests/qcow2rootfs-x86_64 @@ -0,0 +1,80 @@ +#!/bin/bash +# - qcow2_os +# - qcow2_arch +# - qcow2_version +# - rootfs_protocol +# - rootfs_server +# - rootfs_path + +set_qcow2rootfs_tools() +{ + export QCOW2_PATH="/c/compass-ci/container/qcow2rootfs" + git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci || return 1 + rm ${QCOW2_PATH}/root/etc/yum.repos.d/* + wget -O ${QCOW2_PATH}/root/etc/yum.repos.d/CentOS-Base.repo https://repo.huaweicloud.com/repository/conf/CentOS-7-reg.repo + ${QCOW2_PATH}/build + sed -i 's#$EXTRACT_ROOT/bin/extract $QCOW2_FILE $tar_out_file#$EXTRACT_ROOT/bin/extract $EXTRACT_ROOT/qcow2-dir/$QCOW2_NAME $tar_out_file#' \ + ${QCOW2_PATH}/bin/mail + +} + +check_url() +{ + export QCOW2_NAME="${qcow2_os}-${qcow2_version}.${qcow2_arch}.qcow2.xz" + + # Since the 21.03 version of OpenEuler's mirror naming format has changed, + # it has been modified. + if [ "${qcow2_version}" == "21.03" ];then + export QCOW2_NAME="${qcow2_os}-${qcow2_version}-${qcow2_arch}.qcow2.xz" + fi + + export QCOW2_URL="https://repo.openeuler.org/${qcow2_os}-${qcow2_version}/virtual_machine_img…" +} + +download_qcow2_img() +{ + export IMG_PATH="/root/qcow2" + mkdir -p ${IMG_PATH} && cd ${IMG_PATH} + [ -f "${IMG_PATH}/${QCOW2_NAME}" ] || { + wget ${QCOW2_URL} || exit 1 + } +} + +mount_os-rw() +{ + export MOUNT_POINT="${IMG_PATH}/mnt" + mkdir ${MOUNT_POINT} + mount -t ${rootfs_protocol} -o rw ${rootfs_server}:/${rootfs_path} ${MOUNT_POINT} +} + +extract_rootfs() +{ + export ROOTFS_DIR="${qcow2_version}-$(date +%Y-%m-%d-%H-%M-%S)" + export ROOTFS_PATH="${MOUNT_POINT}/openeuler/${qcow2_arch}/automatic-build/${ROOTFS_DIR}" + mkdir -p $ROOTFS_PATH + ${QCOW2_PATH}/run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} +} + +set_resolv_config() +{ + cat >${ROOTFS_PATH}/etc/resolv.conf<<-EOF + nameserver 114.114.114.114 + EOF +} + +set_link() +{ + cd ${MOUNT_POINT}/openeuler/${qcow2_arch} + ln -fs automatic-build/${ROOTFS_DIR} ${qcow2_version}-automatic + + umount ${MOUNT_POINT} + echo $ROOTFS_PATH +} + +set_qcow2rootfs_tools +check_url +download_qcow2_img +mount_os-rw +extract_rootfs +set_resolv_config +set_link -- 2.23.0

1 0

[PATCH compass-ci] container/mail-robot: optimize method
by Luan Shengde 22 Apr '21

22 Apr '21

use File.exist? instead of File.exists? Lint/DeprecatedClassMethods: File.exists? is deprecated in favor of File.exist?. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/apply-account.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/mail-robot/lib/apply-account.rb b/container/mail-robot/lib/apply-account.rb index d8cb4da4..f64a3de9 100755 --- a/container/mail-robot/lib/apply-account.rb +++ b/container/mail-robot/lib/apply-account.rb @@ -56,7 +56,7 @@ class ApplyAccount @es_port = ES_PORT # email info file for account issuers. - @account_issuer = File.exists?(ENV['ACCOUNT_ISSUER']) ? YAML.load_file(ENV['ACCOUNT_ISSUER']) : {} + @account_issuer = File.exist?(ENV['ACCOUNT_ISSUER']) ? YAML.load_file(ENV['ACCOUNT_ISSUER']) : {} @my_info = {} end -- 2.23.0

1 0

[PATCH compass-ci] scheduler/elasticsearch_client.cr: fix es bug
by Wu Zhende 22 Apr '21

22 Apr '21

<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN"> <html><head> <title>404 Not Found</title> </head><body> <h1>Not Found</h1> <p>The requested URL was not found on this server.</p> <hr> <address>Apache Server at http Port 80</address> </body></html> Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/scheduler/elasticsearch_client.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scheduler/elasticsearch_client.cr b/src/scheduler/elasticsearch_client.cr index 1ef79a6..0a7bc44 100644 --- a/src/scheduler/elasticsearch_client.cr +++ b/src/scheduler/elasticsearch_client.cr @@ -35,7 +35,7 @@ class Elasticsearch::Client if auth user = ENV["ES_USER"]? password = ENV["ES_PASSWORD"]? - host = "http://#{user}:#{URI.encode_www_form(password)}@#{host}" if user && password + host = "#{user}:#{URI.encode_www_form(password)}@#{host}" if user && password end @client = Elasticsearch::API::Client.new({:host => host, :port => port}) end -- 2.23.0

1 0

[PATCH lkp-tests] add job yaml file for building/testing mysql
by Luan Shengde 22 Apr '21

22 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/mysql-8-0-20.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 jobs/mysql-8-0-20.yaml diff --git a/jobs/mysql-8-0-20.yaml b/jobs/mysql-8-0-20.yaml new file mode 100644 index 000000000..4b0e97f6e --- /dev/null +++ b/jobs/mysql-8-0-20.yaml @@ -0,0 +1,36 @@ +suite: cci-makepkg +category: functional + +testbox: taishan200-2280-2s48p-256g--a18 + +os: openeuler +os_version: 20.03 +os_mount: initramfs + +mysql_user: root +mysql_host: localhost +mysql_port: 3306 +mysql_db: sysbench_test +mysql_password: mysql_password +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +sysbench-mysql: + oltp_test_mode: complex + max_requests: 0 + mysql_table_engine: innodb + oltp_tables_count: 1000 + oltp_table_size: 100000 + nr_threads: 256 + runtime: 60 + rand_type: special + rand_spec_pct: 100 + oltp_read_only: false + report_interval: 1 + events: 20000 + +sleep: 72000 -- 2.23.0

1 0

[PATCH lkp-tests 3/4] pkg/mysql-8-0-20: compile install mysql-8.0.20
by Luan Shengde 22 Apr '21

22 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- pkg/mysql-8-0-20/PKGBUILD | 77 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 pkg/mysql-8-0-20/PKGBUILD diff --git a/pkg/mysql-8-0-20/PKGBUILD b/pkg/mysql-8-0-20/PKGBUILD new file mode 100644 index 000000000..d6b06f6f8 --- /dev/null +++ b/pkg/mysql-8-0-20/PKGBUILD @@ -0,0 +1,77 @@ +pkgname=mysql-server-mysql +pkgver=8.0.20 +gccdir=gcc +pkgrel=1 +arch=('aarch64' 'x86_64') +url="http://www.mysql.com" +license=('GPLv2') +source=("https://gitee.com/li-yancheng/my-sql_with_boostkit_patch.git" + "https://gitee.com/li-yancheng/gcc.git" + "my.cnf") +md5sums=('SKIP' 'SKIP' 'd90ba2c701507eeac43b06fcfa49c2c1') + +build() +{ + # build_gcc + build_mysql +} + +build_gcc() +{ + mkdir -p $HOME/rpmbuild/SOURCES/ + cd $srcdir/gcc && cp -r * $HOME/rpmbuild/SOURCES/ + cd $HOME/rpmbuild/SOURCES/ && { + rpmbuild -bp gcc.spec + } + cd $HOME/rpmbuild/BUILD/gcc-9.3.0 && { + ./configure --prefix=/usr/ \ + --enable-shared \ + --enable-threads=posix \ + --enable-checking=release \ + --with-system-zlib \ + --enable-__cxa_atexit \ + --disable-libunwind-exceptions \ + --enable-gnu-unique-object \ + --enable-linker-build-id \ + --with-linker-hash-style=gnu \ + --enable-languages=c,c++,objc,obj-c++,fortran,lto \ + --enable-plugin \ + --enable-initfini-array \ + --disable-libgcj \ + --without-isl \ + --without-cloog \ + --enable-gnu-indirect-function \ + --with-stage1-ldflags='-Wl,-z,relro,-z,now' \ + --with-boot-ldflags='-Wl,-z,relro,-z,now' \ + --with-multilib-list=lp64 + make -j96 + make install + } +} + +build_mysql() +{ + unset CPPFLAGS + unset CFLAGS + unset CXXFLAGS + cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver + + CMAKE=cmake + grep -sqF "CentOS Linux release 7" /etc/centos-release && CMAKE=cmake3 + + mkdir -p build + cd build && { + $CMAKE .. -DBUILD_CONFIG=mysql_release -DCMAKE_INSTALL_PREFIX=/usr/local/mysql -DMYSQL_DATADIR=/data/mysql/data -DSYSCONFDIR=/etc -DWITH_BOOST=../boost/boost_1_70_0/ + make -j96 + } +} + +package() +{ + mkdir -p $pkgdir/etc + cp my.cnf $pkgdir/etc/ + + cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver/build + # make DESTDIR=$pkgdir install + make install +} -- 2.23.0

2 2

[PATCH compass-ci] sbin/auto_submit: fix some errors
by Li Yuanchao 21 Apr '21

21 Apr '21

[errmsg] ./auto_submit:66:in `get_argvs_for_competition': wrong number of arguments (given 3, expected 4) (ArgumentError) ./auto_submit:74:in `get_argvs_for_competition': undefined method `chomp' for nil:NilClass (NoMethodError) Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sbin/auto_submit b/sbin/auto_submit index 28e3715..dcf5b80 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -55,7 +55,7 @@ class AutoSubmit tag = %x(git -C #{mirror_dir} tag --points-at #{commit_id}) real_argvs.push("upstream_tag=#{tag}") unless tag.empty? if git_repo.include?('my-sql_with_boostkit_patch') - competi_argvs = get_argvs_for_competition(mirror_dir, branch, commit_id) + competi_argvs = get_argvs_for_competition(real_argvs, mirror_dir, branch, commit_id) real_argvs.push(competi_argvs) end real_argvs.push("upstream_commit=#{commit_id}") @@ -70,8 +70,10 @@ class AutoSubmit if committer_email.include?('noreply') emails = %x(git -C #{mirror_dir} log --format=%ae -2 #{commit_id}) - committer_email = emails.lines[0].chomp - author_email = emails.lines[1].chomp + if emails.lines.size == 2 + committer_email = emails.lines[0].chomp + author_email = emails.lines[1].chomp + end end base_commit = %x(git -C #{mirror_dir} merge-base master #{branch}).chomp -- 2.23.0

1 0

[PATCH compass-ci] container/auto-submit: add es authentication
by Li Yuanchao 21 Apr '21

21 Apr '21

For config_account Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/auto-submit/start | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/container/auto-submit/start b/container/auto-submit/start index 79463f6..ad4ba9d 100755 --- a/container/auto-submit/start +++ b/container/auto-submit/start @@ -3,16 +3,26 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig.rb' +names = Set.new %w[ + ES_USER + ES_PASSWORD +] + +service_authentication = relevant_service_authentication(names) +env = docker_env(service_authentication) + DOCKER_CCI = '/c/compass-ci' DOCKER_LKP = '/c/lkp-tests' docker_rm "auto-submit" -cmd = %W[ +cmd = %w[ docker run --restart=always --name auto-submit +] + env + %W[ -u 1090:1090 -d -e CCI_SRC=#{DOCKER_CCI} -- 2.23.0

1 0

[PATCH compass-ci] sbin/download-rootfs: download osimage
by Liu Yinsi 21 Apr '21

21 Apr '21

the default os_mount=initramfs, so download osimage to run qemu kvm. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/download-rootfs | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/sbin/download-rootfs b/sbin/download-rootfs index 68a6803..d936dbe 100755 --- a/sbin/download-rootfs +++ b/sbin/download-rootfs @@ -19,22 +19,35 @@ os_version=${3:-20.03} load_cci_defaults rootfs_dir=/srv/os/${os}/${os_arch}/${os_version} +osimage_dir=/srv/initrd/osimage/${os}/${os_arch}/${os_version} -download_os_rootfs() -{ - mkdir -p $rootfs_dir - cd $rootfs_dir +mkdir -p $rootfs_dir $osimage_dir - download_url="http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" - wget $download_url +download_one() +{ + local download_url=$1 + wget $download_url + + [ "$?" -ne 0 ] && { + log_error "failed to download os rootfs: $download_url" + exit 1 + } +} - [ "$?" -ne 0 ] && { - log_error "failed to download os rootfs: $download_url" - exit 1 - } +download_os_rootfs() +{ + cd $rootfs_dir + download_one "http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" + + gzip -dc ${os_version}.cgz | cpio -id + rm -rf ${os_version}.cgz +} - gzip -dc ${os_version}.cgz | cpio -id - rm -rf ${os_version}.cgz +download_osimage() +{ + cd $osimage_dir + download_one "http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/initrd/osimage/…" + download_one "http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/initrd/deps/nfs…" } is_virtual() @@ -43,3 +56,4 @@ is_virtual() } is_virtual || download_os_rootfs +is_virtual || download_osimage -- 2.23.0

1 0

[PATCH v2 compass-ci] qcow2rootfs: fix guestfish connection .xz file error
by Wang Chenglong 21 Apr '21

21 Apr '21

[Error] spawn /usr/bin/guestfish -i --ro -a /root/extract/qcow2-dir/openEuler-20.09.x86_64.qcow2.xz guestfish: no operating system was found on this disk after: spawn /usr/bin/guestfish -i --ro -a /root/extract/qcow2-dir/openEuler-20.09.x86_64.qcow2 Welcome to guestfish, the guest filesystem shell for editing virtual machine filesystems and disk images. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/qcow2rootfs/bin/main | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/qcow2rootfs/bin/main b/container/qcow2rootfs/bin/main index c7527b79..2d71c484 100755 --- a/container/qcow2rootfs/bin/main +++ b/container/qcow2rootfs/bin/main @@ -39,7 +39,7 @@ extract_rootfs() { echo "[$(date +'%D %T')] Converting .qcow2 to .tar.gz, please wait several minutes..." su - "$EXTRACT_USER" -c " export LIBGUESTFS_BACKEND=direct - $EXTRACT_ROOT/bin/extract $QCOW2_FILE $tar_out_file + $EXTRACT_ROOT/bin/extract $EXTRACT_ROOT/qcow2-dir/$QCOW2_NAME $tar_out_file " echo "[$(date +'%D %T')] Depressing rootfs to $ROOTFS_DIR" -- 2.23.0

1 0

[PATCH compass-ci] qcow2rootfs: fix guestfish connection .xz file error
by Wang Chenglong 21 Apr '21

21 Apr '21

[Error] spawn /usr/bin/guestfish -i --ro -a /root/extract/qcow2-dir/openEuler-20.09.x86_64.qcow2.xz guestfish: no operating system was found on this disk after: spawn /usr/bin/guestfish -i --ro -a /root/extract/qcow2-dir/openEuler-20.09.x86_64.qcow2 Welcome to guestfish, the guest filesystem shell for editing virtual machine filesystems and disk images. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/qcow2rootfs/bin/main | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/qcow2rootfs/bin/main b/container/qcow2rootfs/bin/main index c7527b79..3bee1bf2 100755 --- a/container/qcow2rootfs/bin/main +++ b/container/qcow2rootfs/bin/main @@ -39,7 +39,7 @@ extract_rootfs() { echo "[$(date +'%D %T')] Converting .qcow2 to .tar.gz, please wait several minutes..." su - "$EXTRACT_USER" -c " export LIBGUESTFS_BACKEND=direct - $EXTRACT_ROOT/bin/extract $QCOW2_FILE $tar_out_file + $EXTRACT_ROOT/bin/extract /root/extract/qcow2-dir/$QCOW2_NAME $tar_out_file " echo "[$(date +'%D %T')] Depressing rootfs to $ROOTFS_DIR" -- 2.23.0

2 2

[PATCH compass-ci 1/3] service/logging-es: configuring the service account and password
by Wu Zhende 21 Apr '21

21 Apr '21

Improve security Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/defconfig.rb | 8 ++++++++ container/logging-es/Dockerfile | 7 +++++++ container/logging-es/build | 9 ++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/container/defconfig.rb b/container/defconfig.rb index b121de8..ad94940 100755 --- a/container/defconfig.rb +++ b/container/defconfig.rb @@ -21,6 +21,14 @@ def relevant_defaults(names) cci_defaults.select { |k, _| names.include? k } end +def relevant_service_account(names) + hash = {} + Dir.glob(['/etc/compass-ci/passwd.yaml']).each do |file| + hash.update YAML.load_file(file) || {} + end + hash.select { |k, _| names.include? k } +end + def set_local_env hash = cci_defaults hash.map { |k, v| system "export #{k}=#{v}" } diff --git a/container/logging-es/Dockerfile b/container/logging-es/Dockerfile index a0acf94..f66dc35 100644 --- a/container/logging-es/Dockerfile +++ b/container/logging-es/Dockerfile @@ -5,6 +5,8 @@ ARG BASE_IMAGE FROM $BASE_IMAGE ARG MEMORY +ARG USER +ARG PASSWORD # docker image borrowed from hub.docker.com/r/gagara/elasticsearch-oss-arm64 @@ -16,6 +18,9 @@ RUN sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/elastic sed -i '$a cluster.initial_master_nodes: ["node-1"]' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a indices.memory.index_buffer_size: 20%' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a thread_pool.write.queue_size: 2000' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.security.enabled: true' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.license.self_generated.type: basic' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.security.transport.ssl.enabled: true' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i "s/-Xms1g/-Xms${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options && \ sed -i "s/-Xmx1g/-Xmx${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options @@ -24,6 +29,8 @@ RUN mkdir /usr/share/elasticsearch/tmp && \ WORKDIR /usr/share/elasticsearch +RUN ./bin/elasticsearch-users useradd ${USER} -p ${PASSWORD} -r superuser + ENV PATH /usr/share/elasticsearch/bin:$PATH ENV ES_TMPDIR /usr/share/elasticsearch/tmp diff --git a/container/logging-es/build b/container/logging-es/build index b50830e..ff1f102 100755 --- a/container/logging-es/build +++ b/container/logging-es/build @@ -3,8 +3,15 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig.rb' +names = Set.new %w[ + LOGGING_ES_USER + LOGGING_ES_PASSWORD +] + +defaults = relevant_service_account(names) docker_skip_rebuild "logging-es:7.11.1" BASE_IMAGE_DICT = { @@ -16,4 +23,4 @@ BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] available_memory = get_available_memory -system "docker build -t logging-es:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg MEMORY=#{available_memory} ." +system "docker build -t logging-es:7.11.1 --progress=plain --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg MEMORY=#{available_memory} --build-arg USER=#{defaults['LOGGING_ES_USER']} --build-arg PASSWORD=#{defaults['LOGGING_ES_PASSWORD']} ." -- 2.23.0

2 1

[PATCH compass-ci] assist-result-client: fix git bisect email subject is the "["
by Cao Xueliang 21 Apr '21

21 Apr '21

Need use json parse the data, before return the response. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/assist_result_client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/assist_result_client.rb b/lib/assist_result_client.rb index 592c841..2d2e60d 100644 --- a/lib/assist_result_client.rb +++ b/lib/assist_result_client.rb @@ -60,6 +60,6 @@ class AssistResult response = resource.post(Base64.encode64(data.to_json)) return nil unless response.code == 200 - return response.body + return JSON.parse(response.body) end end -- 2.23.0

1 0

[PATCH compass-ci] sparrow/dev-env: add the host/port to the job by scheduler
by Cao Xueliang 21 Apr '21

21 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- sparrow/3-code/dev-env | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index e02de35..a478fff 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -163,6 +163,11 @@ RESULT_WEBDAV_HOST: RESULT_WEBDAV_PORT: DOCKER_REGISTRY_HOST: DOCKER_REGISTRY_PORT: +GIT_MIRROR_HOST: +JUMPER_HOST: +JUMPER_PORT: +MONITOR_HOST: +MONITOR_PORT: EOF cat > /etc/profile.d/compass.sh <<'EOF' -- 2.23.0

1 0

[PATCH lkp-tests 2/4] add config file for mysql-8.0.20 sysbench test
by Luan Shengde 21 Apr '21

21 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- pkg/mysql-8-0-20/my.cnf | 67 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 pkg/mysql-8-0-20/my.cnf diff --git a/pkg/mysql-8-0-20/my.cnf b/pkg/mysql-8-0-20/my.cnf new file mode 100644 index 000000000..c657234df --- /dev/null +++ b/pkg/mysql-8-0-20/my.cnf @@ -0,0 +1,67 @@ +[mysqld_safe] +log-error=/data/mysql/log/mysql.log +pid-file=/data/mysql/run/mysqld.pid +[mysqldump] +quick +[mysql] +[client] +default-character-set=utf8mb4 +socket=/data/mysql/run/mysql.sock + +[mysqld] +socket=/data/mysql/run/mysql.sock +tmpdir=/data/mysql/tmp +datadir=/data/mysql/data +default_authentication_plugin=mysql_native_password +user=mysql +memlock=on +bind-address=0.0.0.0 +skip-name-resolve +skip-grant-tables + +lc-messages=en_US +lc-messages-dir=/usr/local/mysql/share + +innodb_page_size=64k +max_connections=4000 +back_log=2048 +performance_schema=OFF +max_prepared_stmt_count=128000 + +#file +innodb_file_per_table +innodb_log_file_size=2048M +innodb_log_files_in_group=32 +innodb_open_files=10000 +table_open_cache_instances=64 + +#buffers +innodb_buffer_pool_size=128G +innodb_buffer_pool_instances=64 +innodb_log_buffer_size=2048M + +#tune +sync_binlog=0 +innodb_flush_log_at_trx_commit=0 +innodb_use_native_aio=1 +innodb_spin_wait_delay=160 +innodb_sync_spin_loops=20 +innodb_flush_method=O_DIRECT +innodb_io_capacity=30000 +innodb_io_capacity_max=40000 +innodb_lru_scan_depth=9000 +innodb_page_cleaners=1 +innodb_spin_wait_pause_multiplier=25 + +#perf special +innodb_flush_neighbors=0 +innodb_write_io_threads=1 +innodb_read_io_threads=1 +innodb_purge_threads=1 + +sql_mode=STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO,STRICT_ALL_TABLES + +#skop_log_bin +ssl=0 + +table_open_cache=15000 -- 2.23.0

1 0

[PATCH lkp-tests 1/4] tests: script to start mysql
by Luan Shengde 21 Apr '21

21 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/mysql-8-0-20 | 73 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 tests/mysql-8-0-20 diff --git a/tests/mysql-8-0-20 b/tests/mysql-8-0-20 new file mode 100755 index 000000000..9b00766f0 --- /dev/null +++ b/tests/mysql-8-0-20 @@ -0,0 +1,73 @@ +#!/bin/sh +# - data_dev +# mysql_user +# mysql_host +# mysql_port +# mysql_db +# mysql_password +. $LKP_SRC/lib/common.sh + +: "${mysql_user:=root}" +: "${mysql_host:=localhost}" +: "${mysql_port:=3306}" +: "${mysql_db:=sysbench_test}" +: "${mysql_password:=mysql_password}" + +create_basedir() +{ + mkdir -p /data + mkfs.xfs -f /dev/"$data_dev" + mount /dev/"$data_dev" /data + + mkdir -p /data/mysql/{data,share,tmp,run,log} + + useradd mysql + touch /data/mysql/log/mysql.log + chown -R mysql:mysql /data + chown -R mysql:mysql /data/mysql/log/mysql.log +} + +configure_mysql() +{ + if [ ! -f $mysqld ] || [ ! -f $mysql ] || [ ! -f $mysql_server ]; then + echo "mysql is not installed!" && exit 1 + fi + + "$mysqld" --user=mysql --basedir=/data/mysql --datadir=/data/mysql/data/ --initialize --port="$mysql_port" + sleep 10 + + "$mysql_server" start + sleep 10 + + "$mysql" -u "$mysql_user" -e "flush privileges; use mysql; alter user 'root'@'localhost' identified by '$mysql_password'; flush privileges;" + sed -i '/^skip-grant-tables/d' /etc/my.cnf + "$mysql_server" restart + sleep 10 +} + +create_database() +{ + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "use mysql; update user set host ='%' where user='root'; flush privileges;" + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $mysql_db;" + "$mysql_server" stop + sleep 10 +} + +start_mysql() +{ + "$mysqld" --defaults-file=/etc/my.cnf --port="$mysql_port" + sleep 20 +} + +main() +{ + local mysql="/usr/local/mysql/bin/mysql" + local mysqld="/usr/local/mysql/bin/mysqld" + local mysql_server="/usr/local/mysql/support-files/mysql.server" + + create_basedir + configure_mysql + create_database + start_mysql +} +main -- 2.23.0

1 0

[PATCH compass-ci] lib/dump_stat.rb: optimize warning info
by Lu Weitao 21 Apr '21

21 Apr '21

[Why] when a key - value is invalid, should show the key and value Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/dump_stat.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/dump_stat.rb b/lib/dump_stat.rb index a497d461..1b56cc35 100644 --- a/lib/dump_stat.rb +++ b/lib/dump_stat.rb @@ -56,7 +56,7 @@ module DumpStat if value.is_a?(String) value = check_string_value(k, value, @monitor) next unless value - return nil unless number?(value, @invalid_records, @record_index, @monitor) + return nil unless number?(k, value, @invalid_records, @record_index, @monitor) value = value.index('.') ? value.to_f : value.to_i elsif value.is_a?(Array) @@ -65,7 +65,7 @@ module DumpStat value[i] = check_string_value(k, value[i], @monitor) next unless value[i] - return nil unless number?(value[i], @invalid_records, @record_index, @monitor) + return nil unless number?(k, value[i], @invalid_records, @record_index, @monitor) value[i] = value[i].index('.') ? value[i].to_f : value[i].to_i valid_stats_verification(k, value[i]) @@ -164,10 +164,10 @@ def check_string_value(key, value, monitor) end # only number is valid -def number?(value, invalid_records, record_index, monitor) +def number?(key, value, invalid_records, record_index, monitor) unless value.numeric? invalid_records.push record_index - warn_stat "invalid stats value: #{value}", monitor + warn_stat "invalid stats key-value: \n key: #{key} \n value: #{value}", monitor return nil end -- 2.23.0

1 0

[PATCH compass-ci 5/5] es_query: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

Everying where "ESQuery.new" is used needs to be adjusted Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/assign-account/start | 4 ++++ container/assist-result/start | 6 ++++++ container/delimiter/start | 6 ++++++ container/mail-robot/start | 5 +++++ lib/commit_status.rb | 2 +- lib/compare.rb | 4 ++-- lib/es_jobs.rb | 2 +- lib/es_query.rb | 6 ++---- sbin/es-find | 2 +- src/lib/web_backend.rb | 2 +- 10 files changed, 29 insertions(+), 10 deletions(-) diff --git a/container/assign-account/start b/container/assign-account/start index b926853..c8c55da 100755 --- a/container/assign-account/start +++ b/container/assign-account/start @@ -4,6 +4,8 @@ . ../defconfig.sh +load_service_authentication + docker_rm assign-account cmd=( @@ -11,6 +13,8 @@ cmd=( --restart=always --name=assign-account -d + -e ES_USER=$ES_USER + -e ES_PASSWORD=$ES_PASSWORD -v /etc:/etc:rw -v /srv/account-info:/opt/account_data/:rw -v /home:/home:rw diff --git a/container/assist-result/start b/container/assist-result/start index af812aa..9998588 100755 --- a/container/assist-result/start +++ b/container/assist-result/start @@ -9,11 +9,17 @@ require_relative '../defconfig.rb' names = Set.new %w[ ES_HOST ES_PORT + ES_USER + ES_PASSWORD ASSIST_RESULT_HOST ASSIST_RESULT_PORT ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' diff --git a/container/delimiter/start b/container/delimiter/start index 9485750..53adc03 100755 --- a/container/delimiter/start +++ b/container/delimiter/start @@ -9,9 +9,15 @@ require_relative '../defconfig.rb' names = Set.new %w[ TASKQUEUE_HOST TASKQUEUE_PORT + ES_USER + ES_PASSWORD ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' diff --git a/container/mail-robot/start b/container/mail-robot/start index ce18498..07aea39 100755 --- a/container/mail-robot/start +++ b/container/mail-robot/start @@ -15,6 +15,8 @@ names = Set.new %w[ SCHED_HTTPS_HOST SCHED_HTTPS_PORT ES_PORT + ES_USER + ES_PASSWORD MAILDIR ACCOUNT_ISSUER lab @@ -22,6 +24,9 @@ names = Set.new %w[ defaults = relevant_defaults(names) +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) cmd = %w[ diff --git a/lib/commit_status.rb b/lib/commit_status.rb index 81aeb29..baae530 100644 --- a/lib/commit_status.rb +++ b/lib/commit_status.rb @@ -26,7 +26,7 @@ def query_latest_good_commit(job, error_id) end def query_jobs_from_es(items) - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new result = es.multi_field_query items jobs_list = result['hits']['hits'] jobs_list.map! { |job| job['_source'] } diff --git a/lib/compare.rb b/lib/compare.rb index 7c182c4..7386cae 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -38,7 +38,7 @@ end def create_matrices_list(conditions) matrices_list = [] suite_list = [] - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new conditions.each do |condition| query_results = es.multi_field_query(condition, desc_keyword: 'start_time') matrix, suites = combine_query_data(query_results) @@ -71,7 +71,7 @@ def compare_group(argv, dimensions, options) end def create_groups_matrices_list(conditions, dims) - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new query_results = es.multi_field_query(conditions, desc_keyword: 'start_time') combine_group_query_data(query_results['hits']['hits'], dims) end diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index 25e54b4..3c29b66 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -23,7 +23,7 @@ class ESJobs end def query_jobs_from_es - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new result = es.multi_field_query @es_query jobs = result['hits']['hits'] jobs.map! { |job| job['_source'] } diff --git a/lib/es_query.rb b/lib/es_query.rb index 22d68a9..dd6d97d 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -7,12 +7,10 @@ require_relative 'constants.rb' # build multiple query request body class ESQuery - HOST = (ENV.key?('ES_HOST') ? ENV['ES_HOST'] : ES_HOST) - PORT = (ENV.key?('ES_PORT') ? ENV['ES_PORT'] : ES_PORT).to_i - def initialize(host = HOST, port = PORT, index: 'jobs') + def initialize(hosts = ES_HOSTS, index: 'jobs') @index = index @scroll_id = '' - @client = Elasticsearch::Client.new url: "http://#{host}:#{port}" + @client = Elasticsearch::Client.new hosts: hosts raise 'Connect Elasticsearch error!' unless @client.ping end diff --git a/sbin/es-find b/sbin/es-find index f548fc1..78e30a8 100755 --- a/sbin/es-find +++ b/sbin/es-find @@ -77,7 +77,7 @@ options = { nr_jobs: 10, desc_keyword: 'start_time' } range = nil count = nil -es = ESQuery.new(ES_HOST, ES_PORT) +es = ESQuery.new(ES_HOSTS) opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: es-find [options] search_key1=val1[,val2..] ..' diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index c4d4edd..91287b4 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -657,7 +657,7 @@ def get_error_objects(filter_items) end def get_job_list(items) - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new query_results = es.multi_field_query(items) extract_jobs_list(query_results['hits']['hits']) -- 2.23.0

1 0

[PATCH compass-ci 4/5] ruby-services: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

Two ruby services: web-backend git_mirror Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/git-mirror/start | 12 +++++++++++- container/web-backend/start | 4 ++++ lib/constants.rb | 22 ++++++++++++++++++++++ lib/git_mirror.rb | 2 +- src/lib/web_backend.rb | 2 +- 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/container/git-mirror/start b/container/git-mirror/start index f01789c..9c9e3e2 100755 --- a/container/git-mirror/start +++ b/container/git-mirror/start @@ -3,8 +3,17 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig.rb' +names = Set.new %w[ + ES_USER + ES_PASSWORD +] + +service_authentication = relevant_service_authentication(names) +env = docker_env(service_authentication) + DOCKER_CCI = '/c/compass-ci' DOCKER_REPO = '/c/git-repos' docker_rm "git-mirror" @@ -12,10 +21,11 @@ docker_rm "git-mirror" lkp = %x(id -u lkp) group_id = %x(cat /etc/group | grep '^committer' | awk -F ':' '{print $3}') -cmd = %W[ +cmd = %w[ docker run --restart=always --name git-mirror +] + env + %W[ -u #{lkp.to_i}:#{group_id.to_i} -d -e CCI_SRC=#{DOCKER_CCI} diff --git a/container/web-backend/start b/container/web-backend/start index cebf949..250c524 100755 --- a/container/web-backend/start +++ b/container/web-backend/start @@ -13,6 +13,8 @@ load_cci_defaults [[ $SRV_HTTP_RESULT_HOST ]] || SRV_HTTP_RESULT_HOST=$(curl ip.sb) [[ $SRV_HTTP_RESULT_PORT ]] || SRV_HTTP_RESULT_PORT=20007 +load_service_authentication + cmd=( docker run --name web-backend @@ -26,6 +28,8 @@ cmd=( -e CCI_SRC=/c/compass-ci -e LKP_SRC=/c/lkp-tests -e UPSTREAM_REPOS_PATH=/c/upstream-repos + -e ES_USER=$ES_USER + -e ES_PASSWORD=$ES_PASSWORD -v $CCI_SRC:/c/compass-ci -v $LKP_SRC:/c/lkp-tests diff --git a/lib/constants.rb b/lib/constants.rb index 11927f9..c71c3de 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -5,9 +5,31 @@ require_relative '../container/defconfig.rb' config = cci_defaults + +names = Set.new %w[ + ES_USER + ES_PASSWORD +] + ES_HOST ||= config['ES_HOST'] || '172.17.0.1' ES_PORT ||= config['ES_PORT'] || 9200 +if ENV['ES_USER'] + ES_USER = ENV['ES_USER'] + ES_PASSWORD = ENV['ES_PASSWORD'] +else + service_authentication = relevant_service_authentication(names) + ES_USER = service_authentication['ES_USER'] + ES_PASSWORD = service_authentication['ES_PASSWORD'] +end + +ES_HOSTS = [{ + host: ES_HOST, + port: ES_PORT, + user: ES_USER, + password: ES_PASSWORD +}] + SEND_MAIL_HOST ||= config['SEND_MAIL_HOST'] || '172.17.0.1' SEND_MAIL_PORT ||= config['SEND_MAIL_PORT'] || 10001 diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 6100d6b..71d6956 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -125,7 +125,7 @@ class MirrorMain @defaults = {} @git_queue = Queue.new @log = JSONLogger.new - @es_client = Elasticsearch::Client.new(url: "http://#{ES_HOST}:#{ES_PORT}") + @es_client = Elasticsearch::Client.new(hosts: ES_HOSTS) clone_upstream_repo load_fork_info connection_init diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 932d362..c4d4edd 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -34,7 +34,7 @@ NOT_SHOW_FIELDS = %w[result_root].freeze ALL_FIELDS = FIELDS + NOT_SHOW_FIELDS NOT_NEED_EXIST_FIELDS = %w[error_ids upstream_repo].freeze PREFIX_SEARCH_FIELDS = ['tbox_group'].freeze -ES_CLIENT = Elasticsearch::Client.new(url: "http://#{ES_HOST}:#{ES_PORT}") +ES_CLIENT = Elasticsearch::Client.new(hosts: ES_HOSTS) COMPARE_RECORDS_NUMBER = 100 def es_query(query) -- 2.23.0

1 0

[PATCH compass-ci 3/5] services: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

services: lifecycle monitoring extract-stats Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/extract-stats/start | 6 ++++++ container/lifecycle/start | 6 ++++++ container/monitoring/start | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/container/extract-stats/start b/container/extract-stats/start index 621099a..7a0377e 100755 --- a/container/extract-stats/start +++ b/container/extract-stats/start @@ -12,6 +12,8 @@ names = Set.new %w[ REDIS_PORT ES_HOST ES_PORT + ES_USER + ES_PASSWORD SEND_MAIL_HOST SEND_MAIL_PORT SRV_HTTP_RESULT_HOST @@ -20,6 +22,10 @@ names = Set.new %w[ ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' diff --git a/container/lifecycle/start b/container/lifecycle/start index 6e9b4fb..484584f 100755 --- a/container/lifecycle/start +++ b/container/lifecycle/start @@ -8,9 +8,15 @@ require_relative '../defconfig.rb' names = Set.new %w[ LIFECYCLE_PORT + ES_USER + ES_PASSWORD ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) LIFECYCLE_PORT = defaults['LIFECYCLE_PORT'] || '11312' diff --git a/container/monitoring/start b/container/monitoring/start index f4214ed..23d5cb4 100755 --- a/container/monitoring/start +++ b/container/monitoring/start @@ -10,9 +10,15 @@ names = Set.new %w[ MONITOR_PORT MQ_PORT MQ_HOST + ES_USER + ES_PASSWORD ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) DEFAULT_CCI = '/c/cci' -- 2.23.0

1 0

[PATCH compass-ci 2/5] service/scheduler: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/scheduler/my-start | 6 ++++++ container/scheduler/start | 6 ++++++ src/scheduler/elasticsearch_client.cr | 8 +++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/container/scheduler/my-start b/container/scheduler/my-start index e5855af..5dd871e 100755 --- a/container/scheduler/my-start +++ b/container/scheduler/my-start @@ -13,6 +13,8 @@ names = Set.new %w[ REDIS_PORT ES_HOST ES_PORT + ES_USER + ES_PASSWORD OS_HTTP_HOST OS_HTTP_PORT INITRD_HTTP_HOST @@ -38,6 +40,10 @@ defaults['SCHED_PORT'] ||= '3000' defaults['SCHED_HOST'] ||= '172.17.0.1' defaults['MASTER_FLUENTD_PORT'] ||= '24224' defaults['SRV_HTTP_CCI_PORT'] ||= '11307' + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) CCI_REPOS = ENV['CCI_REPOS'] || '/c' diff --git a/container/scheduler/start b/container/scheduler/start index 598b263..029dda8 100755 --- a/container/scheduler/start +++ b/container/scheduler/start @@ -14,6 +14,8 @@ names = Set.new %w[ REDIS_PORT ES_HOST ES_PORT + ES_USER + ES_PASSWORD OS_HTTP_HOST OS_HTTP_PORT INITRD_HTTP_HOST @@ -44,6 +46,10 @@ SCHED_PORT = defaults['SCHED_PORT'] sched_nodes.each do |node| docker_rm "s001-alpine-#{node['port']}" end + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) CCI_REPOS = ENV['CCI_REPOS'] || '/c' diff --git a/src/scheduler/elasticsearch_client.cr b/src/scheduler/elasticsearch_client.cr index 0e83ff1..1ef79a6 100644 --- a/src/scheduler/elasticsearch_client.cr +++ b/src/scheduler/elasticsearch_client.cr @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +require "uri" require "yaml" require "json" require "any_merge" @@ -30,7 +31,12 @@ class Elasticsearch::Client HOST = (ENV.has_key?("ES_HOST") ? ENV["ES_HOST"] : JOB_ES_HOST) PORT = (ENV.has_key?("ES_PORT") ? ENV["ES_PORT"] : JOB_ES_PORT).to_i32 - def initialize(host = HOST, port = PORT) + def initialize(host = HOST, port = PORT, auth = true) + if auth + user = ENV["ES_USER"]? + password = ENV["ES_PASSWORD"]? + host = "http://#{user}:#{URI.encode_www_form(password)}@#{host}" if user && password + end @client = Elasticsearch::API::Client.new({:host => host, :port => port}) end -- 2.23.0

1 0

[PATCH compass-ci 1/5] service/es: configuring authentication
by Wu Zhende 21 Apr '21

21 Apr '21

To improve security, need to enter the account and password to access the kibana web page. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/defconfig.sh | 19 +++++++++++++++++++ container/es/Dockerfile | 7 +++++++ container/es/build | 9 ++++++++- container/es/start | 2 +- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/container/defconfig.sh b/container/defconfig.sh index 50446d8..64a434f 100755 --- a/container/defconfig.sh +++ b/container/defconfig.sh @@ -20,6 +20,12 @@ load_cci_defaults() done } +load_service_authentication() +{ + shopt -s nullglob + create_yaml_variables '/etc/compass-ci/passwd.yaml' +} + docker_rm() { container=$1 @@ -28,6 +34,19 @@ docker_rm() docker rm -f $container } +check_auth_es_ready() +{ + local port=$1 + load_service_authentication + local i + for i in {1..30} + do + + curl -s localhost:$port -u $ES_USER:$ES_PASSWORD> /dev/null && return + sleep 2 + done +} + check_service_ready() { local port=$1 diff --git a/container/es/Dockerfile b/container/es/Dockerfile index 44d87ee..f29e677 100644 --- a/container/es/Dockerfile +++ b/container/es/Dockerfile @@ -4,11 +4,16 @@ FROM elasticsearch:7.11.1@sha256:d52cda1e73d1b1915ba2d76ca1e426620c7b5d6942d9d2f432259503974ba786 ARG MEMORY +ARG USER +ARG PASSWORD RUN sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a path.data: /srv/es' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a node.name: node-1' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a cluster.initial_master_nodes: ["node-1"]' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.security.enabled: true' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.license.self_generated.type: basic' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.security.transport.ssl.enabled: true' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i "s/-Xms256m/-Xms${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options && \ sed -i "s/-Xmx256m/-Xmx${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options @@ -17,6 +22,8 @@ RUN mkdir /usr/share/elasticsearch/tmp && \ WORKDIR /usr/share/elasticsearch +RUN ./bin/elasticsearch-users useradd ${USER} -p ${PASSWORD} -r superuser + ENV PATH /usr/share/elasticsearch/bin:$PATH ENV ES_TMPDIR /usr/share/elasticsearch/tmp diff --git a/container/es/build b/container/es/build index db5145f..e023227 100755 --- a/container/es/build +++ b/container/es/build @@ -3,10 +3,17 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig.rb' +names = Set.new %w[ + ES_USER + ES_PASSWORD +] + +defaults = relevant_service_authentication(names) docker_skip_rebuild "es:7.11.1" available_memory = get_available_memory -system "docker build -t es:7.11.1 --build-arg MEMORY=#{available_memory} --network=host ." +system "docker build -t es:7.11.1 --build-arg MEMORY=#{available_memory} --build-arg USER=#{defaults['ES_USER']} --build-arg PASSWORD=#{defaults['ES_PASSWORD']} --network=host ." diff --git a/container/es/start b/container/es/start index 3aa9525..cff44ea 100755 --- a/container/es/start +++ b/container/es/start @@ -21,5 +21,5 @@ cmd=( "${cmd[@]}" # set index -check_service_ready 9200 +check_auth_es_ready 9200 find $CCI_SRC/sbin/ -name "es-*-mapping.sh" -exec sh {} \; -- 2.23.0

1 0

[PATCH compass-ci] delimiter: delete es_query in bisect script
by Cao Xueliang 20 Apr '21

20 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/git_bisect.rb | 8 +++----- src/delimiter/find-commit/bisect_run_script.rb | 18 +++++++++++------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/git_bisect.rb b/lib/git_bisect.rb index 6aedda5..c0c0a60 100644 --- a/lib/git_bisect.rb +++ b/lib/git_bisect.rb @@ -3,7 +3,6 @@ # frozen_string_literal: true require 'json' -require_relative 'es_query' require_relative 'error_messages' require_relative 'sched_client' require_relative '../src/delimiter/utils' @@ -12,7 +11,6 @@ require_relative "#{ENV['LKP_SRC']}/lib/monitor" # find the first bad commit class GitBisect def initialize(task) - @es = ESQuery.new @task = task end @@ -54,7 +52,7 @@ class GitBisect puts "upstream_commit: #{@upstream_commit}" raise 'upstream info is null' unless @upstream_repo || @upstream_commit - @upstream_repo_git = "git://#{GIT_MIRROR_HOST}/#{@upstream_repo}" + @upstream_repo_git = "git://#{GIT_MIRROR_HOST}/upstream/#{@upstream_repo}" end def set_upstream_url @@ -66,7 +64,7 @@ class GitBisect def set_work_dir @work_dir = Utils.clone_repo(@upstream_repo_git, @upstream_commit) puts "work_dir: #{@work_dir}" - raise "checkout repo: #{@upstream_repo} to commit: #{@upstream_commit} failed!" unless @work_dir + raise "checkout repo: #{@upstream_repo_git} to commit: #{@upstream_commit} failed!" unless @work_dir end def set_pkgbuild_repo @@ -104,7 +102,7 @@ class GitBisect return nil unless temp[0].include?('is the first bad commit') || temp[-1].include?('bisect run success') first_bad_commit = Utils.parse_first_bad_commit(result) - bisect_error = ErrorMessages.new(@build_pkg_dir).obtain_error_messages_by_error_id(@error_id) + bisect_error = Utils.get_bisect_error(@bad_job_id, @error_id) all_errors = Utils.obt_errors(@work_dir, first_bad_commit) first_bad_commit_result_root = Utils.obt_result_root_by_commit(first_bad_commit) diff --git a/src/delimiter/find-commit/bisect_run_script.rb b/src/delimiter/find-commit/bisect_run_script.rb index 8cdf675..05c529e 100755 --- a/src/delimiter/find-commit/bisect_run_script.rb +++ b/src/delimiter/find-commit/bisect_run_script.rb @@ -5,13 +5,11 @@ # frozen_string_literal: true require 'json' -require_relative "#{ENV['CCI_SRC']}/lib/es_query" require_relative "#{ENV['CCI_SRC']}/src/delimiter/utils" # git bisect run class GitBisectRun def initialize(job_id, error_id, work_dir) - @es = ESQuery.new @job_id = job_id @error_id = error_id @work_dir = work_dir @@ -27,12 +25,17 @@ class GitBisectRun private def get_bisect_status(job) - status = Utils.get_job_status(job, @error_id) - exit 125 unless status + begin + status = Utils.get_job_status(job, @error_id) + exit 125 unless status - exit 1 if status.eql?('bad') + exit 1 if status.eql?('bad') - exit 0 + exit 0 + rescue StandardError => e + puts e.backtrace.inspect + exit -1 + end end end @@ -43,6 +46,7 @@ begin run = GitBisectRun.new job_id, error_id, work_dir run.git_bisect -rescue +rescue StandardError => e + puts e.backtrace.inspect exit -1 end -- 2.23.0

1 0

[PATCH compass-ci] container/assist-result: add new function in assist-result
by Cao Xueliang 20 Apr '21

20 Apr '21

add get_compare_result and get_error_messages inassist-result, use them in delimiter utils. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assist-result/routes.rb | 23 +++++++++++++++++++ .../assist-result/views/get_compare_result.rb | 10 ++++++++ .../assist-result/views/get_error_messages.rb | 9 ++++++++ lib/assist_result_client.rb | 20 ++++++++++++++++ src/delimiter/utils.rb | 15 ++++++++---- 5 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 container/assist-result/views/get_compare_result.rb create mode 100644 container/assist-result/views/get_error_messages.rb diff --git a/container/assist-result/routes.rb b/container/assist-result/routes.rb index 287db25..c0712a9 100755 --- a/container/assist-result/routes.rb +++ b/container/assist-result/routes.rb @@ -9,6 +9,8 @@ require 'sinatra' require_relative './views/get_job_yaml' require_relative './views/get_job_content' require_relative './views/check_job_credible' +require_relative './views/get_compare_result' +require_relative './views/get_error_messages' configure do set :bind, '0.0.0.0' @@ -45,3 +47,24 @@ get '/get_job_content/:job_id' do return [200, result.to_json] end + +get '/get_compare_errors/*,*' do |pre_id, cur_id| + begin + result = get_compare_errors(pre_id, cur_id) + rescue StandardError => e + return [400, e.backtrace.inspect] + end + + return [200, result.to_json] +end + +post '/get_error_messages' do + begin + data = JSON.parse(Base64.decode64(request.body.read)) + result = get_error_messages(data['job_id'], data['error_id']) + rescue StandardError => e + return [400, e.backtrace.inspect] + end + + return [200, result.to_json] +end diff --git a/container/assist-result/views/get_compare_result.rb b/container/assist-result/views/get_compare_result.rb new file mode 100644 index 0000000..cf9e1de --- /dev/null +++ b/container/assist-result/views/get_compare_result.rb @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative "#{ENV['CCI_SRC']}/lib/compare_error_messages" + +def get_compare_errors(pre_id, cur_id) + _, errors = CEM.get_compare_result(pre_id, cur_id) + return errors +end diff --git a/container/assist-result/views/get_error_messages.rb b/container/assist-result/views/get_error_messages.rb new file mode 100644 index 0000000..e429dfe --- /dev/null +++ b/container/assist-result/views/get_error_messages.rb @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative "#{ENV['CCI_SRC']}/lib/compare_error_messages" + +def get_error_messages(job_id, error_id) + return CEM.get_error_messages(job_id, error_id) +end diff --git a/lib/assist_result_client.rb b/lib/assist_result_client.rb index 9c42b97..592c841 100644 --- a/lib/assist_result_client.rb +++ b/lib/assist_result_client.rb @@ -42,4 +42,24 @@ class AssistResult return response.body end + + def get_compare_errors(pre_id, cur_id) + resource = RestClient::Resource.new("http://#{@host}:#{@port}/get_compare_errors/#{pre_id},#{cur_id}") + response = resource.get() + return nil unless response.code == 200 + + return response.body + end + + def get_error_messages(job_id, error_id) + data = { + 'job_id' => job_id, + 'error_id' => error_id + } + resource = RestClient::Resource.new("http://#{@host}:#{@port}/get_error_messages") + response = resource.post(Base64.encode64(data.to_json)) + return nil unless response.code == 200 + + return response.body + end end diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index f960c7e..c782ea2 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -67,7 +67,7 @@ module Utils def submit_job(job) save_job_to_yaml(job, PROCESS_JOB_YAML) - response = %x(#{LKP_SRC}/sbin/submit #{PROCESS_JOB_YAML}) + response = %x(#{ENV['LKP_SRC']}/sbin/submit #{PROCESS_JOB_YAML}) puts "submit job response: #{response}" return nil if response =~ /job id=0/ return $1 if response =~ /job id=(.*)/ @@ -87,6 +87,7 @@ module Utils query = { 'job_id': new_job_id, 'job_state': 'extract_finished' } extract_finished = monitor_run_stop(query) return nil unless extract_finished.zero? + puts "#{bad_job_id}, #{new_job_id}, #{error_id}" check_result = AssistResult.new.check_job_credible(bad_job_id, new_job_id, error_id) raise "check job credible failed: #{bad_job_id}, #{new_job_id}, #{error_id}" if check_result == nil @@ -185,13 +186,17 @@ module Utils def obt_errors_by_commits(cur_commit, pre_commit) cur_id = obt_id_by_commit(cur_commit) pre_id = obt_id_by_commit(pre_commit) - _, errors = get_compare_result(pre_id, cur_id) - return errors + return AssistResult.new.get_compare_errors(pre_id, cur_id) end def obt_result_root_by_commit(commit) - id = obt_id_by_commit(commit) - ESQuery.new.query_by_id(id)['result_root'] + job_id = obt_id_by_commit(commit) + job_content = AssistResult.new.get_job_content(job_id) + return JSON.parse(job_content)['result_root'] + end + + def get_bisect_error(job_id, error_id) + return AssistResult.new.get_error_messages(job_id, error_id) end end end -- 2.23.0

1 0

[PATCH v3 lkp-tests] sbin/submit: support -n
by Li Yuanchao 20 Apr '21

20 Apr '21

If -n $number, and $number is a integer and $number > 1, submit $number times. And 'nr_run' would be added in the job.yaml. Else, submit once. [usage] submit xxx xxx -n 3 xxx.yaml Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/submit | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/sbin/submit b/sbin/submit index edb207cc0..a35ca0dac 100755 --- a/sbin/submit +++ b/sbin/submit @@ -19,6 +19,7 @@ opt_monitor_query = {} opt_my_queue = false actions = ['output', 'stop'] result_roots = [] +nr_run = 1 options = OptionParser.new do |opts| opts.banner = 'Usage: submit [options] job1.yaml job2.yaml ...' @@ -57,6 +58,11 @@ options = OptionParser.new do |opts| actions << 'mirror_result' end + opts.on('-n job_number', '--number job_number', 'number to submit job') do |number| + nr_run = number.to_i + nr_run = 1 if nr_run < 1 + end + opts.on('-m', '--monitor', "monitor job status: use -m 'KEY: VALUE' to add rule") do opt_monitor = true k, v = ARGV[0].sub(' ', '').split(':', 2) if ARGV[0] @@ -195,16 +201,19 @@ ARGV.each do |jobfile| scheduler_client = SchedulerClient.new(job['SCHED_HOST'], job['SCHED_PORT']) # submit job + job_hash['nr_run'] = nr_run job_json = job_hash.to_json - messages = scheduler_client.submit_job(job_json) - JSON.parse(messages).each do |msg| - if msg['message'].empty? - result_roots << msg['result_root'] - job_ids << msg['job_id'].to_s - puts("submit #{jobfile}, got job id=#{msg['job_id']}") - else - opt_monitor = false - puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + nr_run.times do + messages = scheduler_client.submit_job(job_json) + JSON.parse(messages).each do |msg| + if msg['message'].empty? + result_roots << msg['result_root'] + job_ids << msg['job_id'].to_s + puts("submit #{jobfile}, got job id=#{msg['job_id']}") + else + opt_monitor = false + puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + end end end end -- 2.23.0

1 0

[PATCH compass-ci] lib: use module as namespace in compare_error_messages.rb
by Cao Xueliang 20 Apr '21

20 Apr '21

1. use module CEM. 2. add get_error_messages function. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- .../assist-result/views/check_job_credible.rb | 2 +- lib/compare_error_messages.rb | 138 ++++++++++-------- 2 files changed, 75 insertions(+), 65 deletions(-) diff --git a/container/assist-result/views/check_job_credible.rb b/container/assist-result/views/check_job_credible.rb index a8d387f..c5fc453 100644 --- a/container/assist-result/views/check_job_credible.rb +++ b/container/assist-result/views/check_job_credible.rb @@ -5,5 +5,5 @@ require_relative "#{ENV['CCI_SRC']}/lib/compare_error_messages" def check_job_credible(pre_job_id, cur_job_id, error_id) - return credible?(pre_job_id, cur_job_id, error_id) + return CEM.credible?(pre_job_id, cur_job_id, error_id) end diff --git a/lib/compare_error_messages.rb b/lib/compare_error_messages.rb index 9c864a3..93b38ed 100644 --- a/lib/compare_error_messages.rb +++ b/lib/compare_error_messages.rb @@ -8,91 +8,101 @@ require_relative 'es_query' require_relative 'error_messages' require_relative "#{ENV['LKP_SRC']}/lib/common" -# previous_job_id's error_ids include error_id -# If later_job_id's output include error filename that extract from error_id, -# the later_job_id is credible. -def credible?(previous_job_id, later_job_id, error_id) - es = ESQuery.new - later_es_result = es.query_by_id(later_job_id) +module CEM + extend self - return true if later_es_result['job_state'] == 'finished' + # previous_job_id's error_ids include error_id + # If later_job_id's output include error filename that extract from error_id, + # the later_job_id is credible. + def credible?(previous_job_id, later_job_id, error_id) + es = ESQuery.new + later_es_result = es.query_by_id(later_job_id) - previous_result_file = File.join('/srv', es.query_by_id(previous_job_id)['result_root'], 'build-pkg') - later_result_file = File.join('/srv', later_es_result['result_root'], 'build-pkg') + return true if later_es_result['job_state'] == 'finished' - filenames_check = filenames_check(previous_result_file, later_result_file, error_id) + previous_result_file = File.join('/srv', es.query_by_id(previous_job_id)['result_root'], 'build-pkg') + later_result_file = File.join('/srv', later_es_result['result_root'], 'build-pkg') - return false if filenames_check.empty? || filenames_check.value?(false) + filenames_check = filenames_check(previous_result_file, later_result_file, error_id) - return true -end - -def filenames_check(previous_result_file, later_result_file, error_id) - filenames = Set.new - filenames_check = Hash.new { |h, k| h[k] = false } + return false if filenames_check.empty? || filenames_check.value?(false) - error_lines = ErrorMessages.new(previous_result_file).obtain_error_messages_by_error_id(error_id, true) - error_lines.each do |error_line| - # "src/ssl_sock.c:1454:104: warning: unused parameter 'al' [-Wunused-parameter]" => "src/ssl_sock" - filenames << $1.chomp(File.extname($1)) if error_line =~ /(.*)(:\d+){2}: (error|warning):/ + return true end - File.open(later_result_file).each_line do |line| - filenames.each do |filename| - filenames_check[filename] - filenames_check[filename] = true if line.include?(filename) + def filenames_check(previous_result_file, later_result_file, error_id) + filenames = Set.new + filenames_check = Hash.new { |h, k| h[k] = false } + + error_lines = ErrorMessages.new(previous_result_file).obtain_error_messages_by_error_id(error_id, true) + error_lines.each do |error_line| + # "src/ssl_sock.c:1454:104: warning: unused parameter 'al' [-Wunused-parameter]" => "src/ssl_sock" + filenames << $1.chomp(File.extname($1)) if error_line =~ /(.*)(:\d+){2}: (error|warning):/ + end + + File.open(later_result_file).each_line do |line| + filenames.each do |filename| + filenames_check[filename] + filenames_check[filename] = true if line.include?(filename) + end end + + return filenames_check end - return filenames_check -end + def get_compare_result(previous_job_id, later_job_id) + es = ESQuery.new -def get_compare_result(previous_job_id, later_job_id) - es = ESQuery.new + previous_error_ids = es.query_by_id(previous_job_id)['error_ids'] + later_es_result = es.query_by_id(later_job_id) + later_error_ids = later_es_result['error_ids'] + later_result_file = File.join('/srv', later_es_result['result_root'], 'build-pkg') - previous_error_ids = es.query_by_id(previous_job_id)['error_ids'] - later_es_result = es.query_by_id(later_job_id) - later_error_ids = later_es_result['error_ids'] - later_result_file = File.join('/srv', later_es_result['result_root'], 'build-pkg') + new_error_ids = get_new_error_ids(later_error_ids, previous_error_ids) + error_messages = ErrorMessages.new(later_result_file).obtain_error_messages + new_error_number, formatted_error_messages = format_error_messages(error_messages, new_error_ids) - new_error_ids = get_new_error_ids(later_error_ids, previous_error_ids) - error_messages = ErrorMessages.new(later_result_file).obtain_error_messages - new_error_number, formatted_error_messages = format_error_messages(error_messages, new_error_ids) + # new error use ">>" as an identifier in the error header + return new_error_number, formatted_error_messages + end - # new error use ">>" as an identifier in the error header - return new_error_number, formatted_error_messages -end + def get_error_messages(job_id, error_id) + es_result = ESQuery.new.query_by_id(job_id) + result_file = File.join('/srv', es_result['result_root'], 'build-pkg') + return ErrorMessages.new(result_file).obtain_error_messages_by_error_id(error_id) + end -def get_new_error_ids(later_error_ids, previous_error_ids) - return nil_to_empty_array(later_error_ids) - nil_to_empty_array(previous_error_ids) -end + def get_new_error_ids(later_error_ids, previous_error_ids) + return nil_to_empty_array(later_error_ids) - nil_to_empty_array(previous_error_ids) + end -def nil_to_empty_array(array) - array = [] if array.nil? - return array -end + def nil_to_empty_array(array) + array = [] if array.nil? + return array + end -def format_error_messages(error_messages, new_error_ids) - formatted_error_messages = Terminal::Table.new - formatted_error_messages.style = { border_x: '', border_y: '', border_i: '', padding_left: 0 } - new_error_number = 0 - - error_messages.each do |k, v| - if new_error_ids.include?("build-pkg.#{build_pkg_error_id(k)}") - new_error_number += 1 - formatted_error_messages = add_sign(formatted_error_messages, '>>', v) - else - formatted_error_messages = add_sign(formatted_error_messages, ' ', v) + def format_error_messages(error_messages, new_error_ids) + formatted_error_messages = Terminal::Table.new + formatted_error_messages.style = { border_x: '', border_y: '', border_i: '', padding_left: 0 } + new_error_number = 0 + + error_messages.each do |k, v| + if new_error_ids.include?("build-pkg.#{build_pkg_error_id(k)}") + new_error_number += 1 + formatted_error_messages = add_sign(formatted_error_messages, '>>', v) + else + formatted_error_messages = add_sign(formatted_error_messages, ' ', v) + end end + + return new_error_number, formatted_error_messages.to_s end - return new_error_number, formatted_error_messages.to_s -end + def add_sign(formatted_error_messages, sign, set) + set.each do |value| + formatted_error_messages.add_row([sign, value]) + end -def add_sign(formatted_error_messages, sign, set) - set.each do |value| - formatted_error_messages.add_row([sign, value]) + return formatted_error_messages end - - return formatted_error_messages end -- 2.23.0

1 0

[PATCH compass-ci] sbin/auto_submit: support argvs for competition
by Li Yuanchao 20 Apr '21

20 Apr '21

Get true author_email and committer_email for PR. And get base_commit for branch, which is the commit when branch is created. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/sbin/auto_submit b/sbin/auto_submit index 4adce89..28e3715 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -51,17 +51,35 @@ class AutoSubmit real_argvs = Array.new(submit_argv) real_argvs.push("upstream_branch=#{branch}") - real_argvs.push("upstream_commit=#{commit_id}") real_argvs.push("upstream_dir=#{belong}") tag = %x(git -C #{mirror_dir} tag --points-at #{commit_id}) real_argvs.push("upstream_tag=#{tag}") unless tag.empty? - author_email = %x(git -C #{mirror_dir} log --format=%ae -1) - committer_email = %x(git -C #{mirror_dir} log --format=%ce -1) - real_argvs.push("author_email=#{author_email} committer_email=#{committer_email}") + if git_repo.include?('my-sql_with_boostkit_patch') + competi_argvs = get_argvs_for_competition(mirror_dir, branch, commit_id) + real_argvs.push(competi_argvs) + end + real_argvs.push("upstream_commit=#{commit_id}") system(real_argvs.join(' ')) end + def get_argvs_for_competition(real_argvs, mirror_dir, branch, commit_id) + argvs_for_base = Array.new(real_argvs) + author_email = %x(git -C #{mirror_dir} log --format=%ae -1 #{commit_id}).chomp + committer_email = %x(git -C #{mirror_dir} log --format=%ce -1 #{commit_id}).chomp + + if committer_email.include?('noreply') + emails = %x(git -C #{mirror_dir} log --format=%ae -2 #{commit_id}) + committer_email = emails.lines[0].chomp + author_email = emails.lines[1].chomp + end + + base_commit = %x(git -C #{mirror_dir} merge-base master #{branch}).chomp + argvs_for_base.push("upstream_commit=#{base_commit}") + system(argvs_for_base.join(' ')) + return "author_email=#{author_email} committer_email=#{committer_email} base_commit=#{base_commit}" + end + def submit_specific_branch(submit_argv, newrefs_info, branches) branches.each do |branch_name| branch = "refs/heads/#{branch_name}" -- 2.23.0

1 0

[PATCH compass-ci] extract-stats: fix expect ":" but got "="
by Cao Xueliang 20 Apr '21

20 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/stats_worker.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index 47c3a59..f385fee 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -99,7 +99,7 @@ class StatsWorker sample_error_id = new_error_ids.sample @log.info("send a delimiter task: job_id is #{job_id}") queue = "#{DELIMITER_TASK_QUEUE}/#{job_id}" - value = {"job_id" => job_id, "error_id" => sample_error_id} + value = %({"job_id": "#{job_id}", "error_id": "#{sample_error_id}"}) @etcd.put(queue, value) msg = %({"job_id": "#{job_id}", "new_error_id": "#{sample_error_id}"}) -- 2.23.0

1 0

[PATCH compass-ci] container/scheduler-http: fix unfriendly error
by Liu Yinsi 20 Apr '21

20 Apr '21

before: /c/compass-ci/container/scheduler-https# ./start Traceback (most recent call last): ./start:14:in `<main>': CA certificate is missing. Ignore if the HTTPS scheduler is not required (RuntimeError) after: /c/compass-ci/container/scheduler-https# ./start CA certificate is missing. Ignore if the HTTPS scheduler is not required Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/scheduler-https/start | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/container/scheduler-https/start b/container/scheduler-https/start index 4ec84a28..374300c0 100755 --- a/container/scheduler-https/start +++ b/container/scheduler-https/start @@ -11,7 +11,10 @@ crt_path = "#{ca_path}/server.crt" key_path = "#{ca_path}/server.key" warn_msg = 'CA certificate is missing. Ignore if the HTTPS scheduler is not required' -raise warn_msg unless File.exist?(crt_path) && File.exist?(key_path) +unless File.exist?(crt_path) && File.exist?(key_path) + puts warn_msg + exit +end names = Set.new %w[ SCHED_HOST -- 2.23.0

1 0

[PATCH v2 compass-ci] lib/rpm-repo.rb: do not move to pub directory until test completed
by Li Ping 20 Apr '21

20 Apr '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/rpm-repo.rb | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/rpm-repo.rb b/lib/rpm-repo.rb index 658a5270..c8d6b087 100755 --- a/lib/rpm-repo.rb +++ b/lib/rpm-repo.rb @@ -6,6 +6,7 @@ require 'fileutils' require 'json' require './mq_client' +require_relative 'json_logger.rb' MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || '172.17.0.1' MQ_PORT = ENV['MQ_PORT'] || 5672 @@ -29,31 +30,47 @@ MQ_PORT = ENV['MQ_PORT'] || 5672 # change /srv/rpm/pub/**/repodate # @mq.ack(info) class HandleRepo + @@upload_dir_prefix = "/srv/rpm/upload/" def initialize @mq = MQClient.new(MQ_HOST, MQ_PORT) - @update = [] + @log = JSONLogger.new end def handle_new_rpm queue = @mq.queue("update_repo") queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| - rpm_info = JSON.parse(msg) - rpm_info["upload_rpms"].each do |rpm| - rpm_path = File.dirname(rpm).sub("upload", "testing") - FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) + begin + rpm_info = JSON.parse(msg) + check_upload_rpms(rpm_info) + rpm_info["upload_rpms"].each do |rpm| + rpm_path = File.dirname(rpm).sub("upload", "testing") + FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) - dest = File.join(rpm_path.to_s, File.basename(rpm)) - @update << dest - FileUtils.mv(rpm, dest) - system("createrepo --update $(dirname #{rpm_path})") + dest = File.join(rpm_path.to_s, File.basename(rpm)) + FileUtils.mv(rpm, dest) + system("createrepo --update $(dirname #{rpm_path})") + end + @mq.ack(info) + rescue StandardError => e + @log.warn({ + "error message": e.message + }.to_json) + @mq.ack(info) end - update_pub_dir - @mq.ack(info) end end - def update_pub_dir - @update.each do |rpm| + def check_upload_rpms(data) + raise JSON.dump({ "errcode" => "200", "errmsg" => "no upload_rpms params" }) unless data.key?("upload_rpms") + raise JSON.dump({ "errcode" => "200", "errmsg" => "upload_rpms params type error" }) if data["upload_rpms"].class != Array + data["upload_rpms"].each do |rpm| + raise JSON.dump({ "errcode" => "200", "errmsg" => "#{rpm} not exist" }) unless File.exist?(rpm) + raise JSON.dump({ "errcode" => "200", "errmsg" => "the upload directory is incorrect" }) unless File.dirname(rpm).start_with?(@@upload_dir_prefix) + end + end + + def update_pub_dir(update) + update.each do |rpm| pub_path = File.dirname(rpm).sub("testing", "pub") FileUtils.mkdir_p(pub_path) unless File.directory?(pub_path) -- 2.23.0

1 0

[PATCH compass-ci 3/3] service/kibana-logging: configuring ES authentication information
by Wu Zhende 20 Apr '21

20 Apr '21

logging-es set up the certification. kibana-logging needs to configure the connection information to connect to the logging-es. When logging to the kibana web page, need to enter the account and password to enhance security. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana-logging/Dockerfile | 7 +++++-- container/kibana-logging/build | 10 +++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/container/kibana-logging/Dockerfile b/container/kibana-logging/Dockerfile index 970eb5a..7085618 100644 --- a/container/kibana-logging/Dockerfile +++ b/container/kibana-logging/Dockerfile @@ -5,8 +5,11 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -# docker image borrowed from hub.docker.com/r/gagara/kibana-oss-arm64 +ARG USER +ARG PASSWORD MAINTAINER Wu Zhende <wuzhende666(a)163.com> -RUN sed -i 's/server.host: "0"/server.host: "0.0.0.0"/' config/kibana.yml +RUN sed -i 's/server.host: "0"/server.host: "0.0.0.0"/' config/kibana.yml && \ + sed -i "6i\elasticsearch.username: ${USER}" config/kibana.yml && \ + sed -i "6i\elasticsearch.password: ${PASSWORD}" config/kibana.yml diff --git a/container/kibana-logging/build b/container/kibana-logging/build index 9c75aeb..238a88d 100755 --- a/container/kibana-logging/build +++ b/container/kibana-logging/build @@ -3,8 +3,16 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig' +names = Set.new %w[ + LOGGING_ES_USER + LOGGING_ES_PASSWORD +] + +defaults = relevant_service_account(names) + docker_skip_rebuild "kibana:7.11.1" BASE_IMAGE_DICT = { @@ -13,4 +21,4 @@ BASE_IMAGE_DICT = { BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] -system "docker build -t kibana:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} ." +system "docker build -t kibana:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg USER=#{defaults['LOGGING_ES_USER']} --build-arg PASSWORD=#{defaults['LOGGING_ES_PASSWORD']} ." -- 2.23.0

1 0

[PATCH compass-ci 2/3] service/master-fluentd: configuring ES authentication information
by Wu Zhende 20 Apr '21

20 Apr '21

logging-es set up the certification. fluentd needs to configure the connection information to connect to the logging-es. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/master-fluentd/docker-fluentd.conf | 2 ++ container/master-fluentd/start | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/container/master-fluentd/docker-fluentd.conf b/container/master-fluentd/docker-fluentd.conf index 410f2eb..73d7103 100644 --- a/container/master-fluentd/docker-fluentd.conf +++ b/container/master-fluentd/docker-fluentd.conf @@ -42,6 +42,8 @@ @type elasticsearch host "#{ENV['LOGGING_ES_HOST']}" port "#{ENV['LOGGING_ES_PORT']}" + user "#{ENV['LOGGING_ES_USER']}" + password "#{ENV['LOGGING_ES_PASSWORD']}" suppress_type_name true flush_interval 1s num_threads 10 diff --git a/container/master-fluentd/start b/container/master-fluentd/start index a05881c..27aece9 100755 --- a/container/master-fluentd/start +++ b/container/master-fluentd/start @@ -9,11 +9,17 @@ require_relative '../defconfig.rb' names = Set.new %w[ LOGGING_ES_HOST LOGGING_ES_PORT + LOGGING_ES_USER + LOGGING_ES_PASSWORD ] +account_info = relevant_service_account(names) + defaults = relevant_defaults(names) defaults['LOGGING_ES_HOST'] ||= '172.17.0.1' defaults['LOGGING_ES_PORT'] ||= '9202' +defaults.merge!(account_info) + env = docker_env(defaults) docker_rm 'master-fluentd' -- 2.23.0

1 0

[PATCH lab-z9] add new info for a125
by Bai Jing 20 Apr '21

20 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- hosts/taishan200-2280-2s64p-256g--a125 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/hosts/taishan200-2280-2s64p-256g--a125 b/hosts/taishan200-2280-2s64p-256g--a125 index 5c5862e..f2cb708 100644 --- a/hosts/taishan200-2280-2s64p-256g--a125 +++ b/hosts/taishan200-2280-2s64p-256g--a125 @@ -2,11 +2,7 @@ memory: 256G nr_hdd_partitions: 2 hdd_partitions: - /dev/disk/by-id/scsi-35000039988707ca1 - - scsi-35000039988707ca10 - - scsi-35000039988707ca1-part* - - /dev/disk/by-id/scsi-35000039988707d25-part1 - - /dev/disk/by-id/scsi-35000039988707d25-part2 - - /dev/disk/by-id/scsi-35000039988707d25-part3 + - /dev/disk/by-id/scsi-35000039988707d25 mac_addr: - 44:67:47:85:f4:65 - 44:67:47:85:f4:66 -- 2.23.0

1 0

[PATCH v2 lkp-tests] sbin/submit: support nr_run
by Li Yuanchao 20 Apr '21

20 Apr '21

If nr_run is a integer and nr_run > 1, submit nr_run times. Else, submit once. [usage] submit xxx xxx nr_run=3 xxx.yaml Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/submit | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/sbin/submit b/sbin/submit index edb207cc0..bcc155f3e 100755 --- a/sbin/submit +++ b/sbin/submit @@ -195,16 +195,21 @@ ARGV.each do |jobfile| scheduler_client = SchedulerClient.new(job['SCHED_HOST'], job['SCHED_PORT']) # submit job + nr_run = job_hash['nr_run'] || 1 + nr_run = nr_run.to_i + nr_run = 1 if nr_run < 1 job_json = job_hash.to_json - messages = scheduler_client.submit_job(job_json) - JSON.parse(messages).each do |msg| - if msg['message'].empty? - result_roots << msg['result_root'] - job_ids << msg['job_id'].to_s - puts("submit #{jobfile}, got job id=#{msg['job_id']}") - else - opt_monitor = false - puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + nr_run.times do + messages = scheduler_client.submit_job(job_json) + JSON.parse(messages).each do |msg| + if msg['message'].empty? + result_roots << msg['result_root'] + job_ids << msg['job_id'].to_s + puts("submit #{jobfile}, got job id=#{msg['job_id']}") + else + opt_monitor = false + puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + end end end end -- 2.23.0

1 0

[PATCH compass-ci] sparrow/3-code/dev-env: set default value
by Liu Yinsi 20 Apr '21

20 Apr '21

[error] submit_id=e628930d-f8ce-4ea1-b323-f8593c6f60cf 2021-04-19 20:59:08 +0800 WARN -- skip non-executable /c/lkp-tests/tests/self-test submit /c/lkp-tests/jobs/host-info.yaml failed, got job id=0, error: The /srv/initrd/lkp/latest/lkp-aarch64.cgz does not exist. [why] when selftest, if not to define $lkp_initrd_user in /etc/compass-ci/defaults/*.yaml, when submit job, it will not to find /srv/initrd/lkp/$lkp_initrd_user/lkp-$arch.cgz. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 969889ed..e02de355 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -55,12 +55,14 @@ server_name=$(hostname | cut -f1 -d.) : ${MAILDIR:=/srv/cci/Maildir/.compass-ci} : ${SENT_MAILDIR:=/srv/cci/Maildir/.sent} : ${NETDATA_PORT:=20013} +: ${lkp_initrd_user:=latest} mkdir -p /etc/compass-ci/defaults cat > /etc/compass-ci/defaults/$server_name.yaml <<EOF SCHED_HOST: $sched_host SCHED_PORT: $sched_port lab: $lab +lkp_initrd_user: $lkp_initrd_user EOF [[ -f /etc/compass-ci/defaults/upstream-config ]] || { -- 2.23.0

1 0

[PATCH compass-ci] sparrow/1-storage/permission: set lkp owner and group
by Liu Yinsi 20 Apr '21

20 Apr '21

/srv/initrd/deps and /srv/initrd/pkg permission must be lkp:lkp, or build pkg and deps are unable to transfer to the directories. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/1-storage/permission | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sparrow/1-storage/permission b/sparrow/1-storage/permission index db33a93..3c229c5 100755 --- a/sparrow/1-storage/permission +++ b/sparrow/1-storage/permission @@ -13,6 +13,9 @@ chmod_dir=( /srv/rabbitmq /c/git-repos /srv/tmp/public + /srv/initrd + /srv/initrd/pkg + /srv/initrd/deps ) chmod 775 "${chmod_dir[@]}" @@ -20,6 +23,8 @@ chmod 775 "${chmod_dir[@]}" chown_dir=( /srv/result /srv/initrd + /srv/initrd/pkg + /srv/initrd/deps /srv/es /srv/cache/netdata_cache /srv/cache/netdata_lib -- 2.23.0

1 0

[PATCH lkp-tests] sbin/submit: support nr_run
by Li Yuanchao 20 Apr '21

20 Apr '21

If nr_run is a integer and nr_run > 1, submit nr_run times. Else, submit once. [usage] submit xxx xxx nr_run=3 xxx.yaml Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/submit | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/sbin/submit b/sbin/submit index edb207cc0..2e6ad8960 100755 --- a/sbin/submit +++ b/sbin/submit @@ -195,16 +195,21 @@ ARGV.each do |jobfile| scheduler_client = SchedulerClient.new(job['SCHED_HOST'], job['SCHED_PORT']) # submit job - job_json = job_hash.to_json - messages = scheduler_client.submit_job(job_json) - JSON.parse(messages).each do |msg| - if msg['message'].empty? - result_roots << msg['result_root'] - job_ids << msg['job_id'].to_s - puts("submit #{jobfile}, got job id=#{msg['job_id']}") - else - opt_monitor = false - puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + nr_run = job_hash['nr_run'] || 1 + nr_run = nr_run.to_i + nr_run = 1 if nr_run < 1 + nr_run.times do + job_json = job_hash.to_json + messages = scheduler_client.submit_job(job_json) + JSON.parse(messages).each do |msg| + if msg['message'].empty? + result_roots << msg['result_root'] + job_ids << msg['job_id'].to_s + puts("submit #{jobfile}, got job id=#{msg['job_id']}") + else + opt_monitor = false + puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + end end end end -- 2.23.0

3 4

[PATCH lkp-tests 2/2] install-iso-firstboot: modify how job's IPXE response is retrieved
by Yu Chuan 19 Apr '21

19 Apr '21

[Why] Vmlinuz, initrd, reboot.cgz is all that is required for firstboot after ISO is installed. Therefore, the original os_mount=local is changed to custom_ipxe to obtain the ipxe response. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/install-iso-firstboot.yaml | 9 +-------- tests/install-iso-pre | 14 +++++++++----- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/jobs/install-iso-firstboot.yaml b/jobs/install-iso-firstboot.yaml index 7be9eaaab184..c170b8a07f8a 100644 --- a/jobs/install-iso-firstboot.yaml +++ b/jobs/install-iso-firstboot.yaml @@ -1,10 +1,3 @@ suite: install-iso-firstboot -sleep: 1 - -# os_mount must be equal to 'local' -os_mount: local - -# kernel_custom_params must have 'boot_lv_suffix' and 'src_lv_suffix', -# and they must be the same. so can boot from the specify logical volume -kernel_custom_params: +custom_ipxe: diff --git a/tests/install-iso-pre b/tests/install-iso-pre index 768ffb2cca45..c89895a994cc 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -225,14 +225,18 @@ submit_install_iso_firstboot_yaml() local firstboot_yaml="${LKP_SRC}/jobs/install-iso-firstboot.yaml" [ -f "${firstboot_yaml}" ] || die "cannot find ${firstboot_yaml}" + custom_ipxe="#!ipxe\\\\n" + custom_ipxe="${custom_ipxe}\\\\ndhcp" + custom_ipxe="${custom_ipxe}\\\\ninitrd http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/${iso_os}/${iso_arch}/${iso_version}/initrd.lkp" + custom_ipxe="${custom_ipxe}\\\\ninitrd http://${INITRD_HTTP_HOST}:${INITRD_HTTP_PORT}/initrd/custom_bootstrap/reboot.cgz" + custom_ipxe="${custom_ipxe}\\\\nkernel http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/${iso_os}/${iso_arch}/${iso_version}/boot/vmlinuz ip=dhcp root=/dev/mapper/os-${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX} initrd=initrd.lkp initrd=reboot.cgz local rw" + custom_ipxe="${custom_ipxe}\\\\nboot\\\\n" + + sed -i "/custom_ipxe:/ccustom_ipxe: \"${custom_ipxe}\"" $firstboot_yaml + ${LKP_SRC}/sbin/submit \ ${firstboot_yaml} \ suite=install-iso-firstboot-${ROOTFS_SUFFIX} \ - os=${iso_os} \ - os_arch=${iso_arch} \ - os_version=${iso_version} \ - os_mount=local \ - kernel_custom_params="src_lv_suffix=${ROOTFS_SUFFIX} boot_lv_suffix=${ROOTFS_SUFFIX}" \ testbox=$testbox || die "submit ${firstboot_yaml} failed" } -- 2.23.0

1 0

[PATCH lkp-tests 1/2] install-iso-pre: fix some bugs
by Yu Chuan 19 Apr '21

19 Apr '21

[What Bugs] 1. fix the use of secrets fields. 2. add mkfs.ext4 step after create logical volume. 3. vmlinux => vmlinuz 4. ks upload dir: - before: /srv/os/kickstart/ks/${iso_os}_${iso_arch}_${iso_version}.cfg - after: /srv/os/install/ks/${iso_os}_${iso_arch}_${iso_version}.ks 5. second-level hash problem when submit. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index ef4a0659fe02..768ffb2cca45 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -21,7 +21,7 @@ check_params() "HTTP_SERVER_HOST" "HTTP_SERVER_PORT" "HTTP_ROOT_DIR" - "ROOTFS_ROOT_PASSWD" + "secrets_ROOTFS_ROOT_PASSWD" "rootfs_disk" "RSYNC_SERVER_HOST" @@ -31,7 +31,7 @@ check_params() "NFS_ROOT_DIR" "LAB_Z9_URL" - "MY_TOKEN" + "secrets_MY_TOKEN" ) for i in "${required_vars[@]}" @@ -65,6 +65,8 @@ create_lv() log_info "start create lv:$LV_NAME" lvcreate -y -L 10G -n ${LV_NAME} ${VG_NAME} || die "create logical volume failed." + + mkfs.ext4 /dev/mapper/${VG_NAME}-${LV_NAME} } prepare_lv() @@ -111,7 +113,7 @@ generate_ks() logvol / --vgname=${VG_NAME} --name=${LV_NAME} --useexisting # Root password - rootpw --iscrypted ${ROOTFS_ROOT_PASSWD} + rootpw --iscrypted ${secrets_ROOTFS_ROOT_PASSWD} # Reboot after installation reboot --eject @@ -180,7 +182,7 @@ prepare_submit() SCHED_PORT: ${SCHED_PORT} my_name: ${my_name} my_email: ${my_email} - my_token: ${MY_TOKEN} + my_token: ${secrets_MY_TOKEN} lab: z9 EOF } @@ -195,7 +197,7 @@ prepare_install_iso_yaml() custom_ipxe="#!ipxe\\\\n" custom_ipxe="${custom_ipxe}\\\\ndhcp" custom_ipxe="${custom_ipxe}\\\\ninitrd ${tftp_root}/initrd.img" - custom_ipxe="${custom_ipxe}\\\\nkernel ${tftp_root}/vmlinux ro inst.geoloc=0 console=ttyS0 smmu.bypassdev=0x1000:0x17 smmu.bypassde1000:0x15 viff video=VGA-1:640x480-32@60me initrd=initrd.img ip=dhcp inst.ks=http://${HTTP_SERVER_HOST}:${HTTP_SERVER_PORT}/os/kickstart/ks/${iso_os}_${iso_arch}_${iso_version}.cfg" + custom_ipxe="${custom_ipxe}\\\\nkernel ${tftp_root}/vmlinuz ro inst.geoloc=0 console=ttyS0 smmu.bypassdev=0x1000:0x17 smmu.bypassde1000:0x15 viff video=VGA-1:640x480-32@60me initrd=initrd.img ip=dhcp inst.ks=http://${HTTP_SERVER_HOST}:${HTTP_SERVER_PORT}/os/install/ks/${iso_os}_${iso_arch}_${iso_version}.ks" custom_ipxe="${custom_ipxe}\\\\nboot\\\\n" log_info "custom_ipxe: $custom_ipxe" @@ -244,13 +246,13 @@ submit_install_iso_sync_rootfs_yaml() ${LKP_SRC}/sbin/submit \ ${sync_rootfs_yaml} \ suite=install-iso-sync-rootfs-${ROOTFS_SUFFIX} \ - iso_os=${iso_os} \ - iso_arch=${iso_arch} \ - iso_version=${iso_version} \ - NFS_SERVER_HOST=${NFS_SERVER_HOST} \ - NFS_ROOT_DIR=${NFS_ROOT_DIR} \ - ROOTFS_SUFFIX=${ROOTFS_SUFFIX} \ - VG_NAME=${VG_NAME} \ + install-iso-sync-rootfs.iso_os=${iso_os} \ + install-iso-sync-rootfs.iso_arch=${iso_arch} \ + install-iso-sync-rootfs.iso_version=${iso_version} \ + install-iso-sync-rootfs.NFS_SERVER_HOST=${NFS_SERVER_HOST} \ + install-iso-sync-rootfs.NFS_ROOT_DIR=${NFS_ROOT_DIR} \ + install-iso-sync-rootfs.ROOTFS_SUFFIX=${ROOTFS_SUFFIX} \ + install-iso-sync-rootfs.VG_NAME=${VG_NAME} \ testbox=$testbox || die "submit ${sync_rootfs_yaml} failed" } -- 2.23.0

1 0

[PATCH compass-ci] lib/rpm-repo.rb: do not move to pub directory until test completed
by Li Ping 19 Apr '21

19 Apr '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/rpm-repo.rb | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/lib/rpm-repo.rb b/lib/rpm-repo.rb index 658a5270..f2344dcc 100755 --- a/lib/rpm-repo.rb +++ b/lib/rpm-repo.rb @@ -31,29 +31,41 @@ MQ_PORT = ENV['MQ_PORT'] || 5672 class HandleRepo def initialize @mq = MQClient.new(MQ_HOST, MQ_PORT) - @update = [] end def handle_new_rpm queue = @mq.queue("update_repo") queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| - rpm_info = JSON.parse(msg) - rpm_info["upload_rpms"].each do |rpm| - rpm_path = File.dirname(rpm).sub("upload", "testing") - FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) + begin + rpm_info = JSON.parse(msg) + check_upload_rpms(rpm_info) + rpm_info["upload_rpms"].each do |rpm| + rpm_path = File.dirname(rpm).sub("upload", "testing") + FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) - dest = File.join(rpm_path.to_s, File.basename(rpm)) - @update << dest - FileUtils.mv(rpm, dest) - system("createrepo --update $(dirname #{rpm_path})") + dest = File.join(rpm_path.to_s, File.basename(rpm)) + FileUtils.mv(rpm, dest) + system("createrepo --update $(dirname #{rpm_path})") + end + @mq.ack(info) + rescue StandardError => e + puts "error message: ", e.message + @mq.ack(info) end - update_pub_dir - @mq.ack(info) end end - def update_pub_dir - @update.each do |rpm| + def check_upload_rpms(data) + raise JSON.dump({ 'errcode' => '200', 'errmsg' => 'no upload_rpms params' }) unless data.key?('upload_rpms') + raise JSON.dump({ 'errcode' => '200', 'errmsg' => "upload_rpms params type error" }) if data["upload_rpms"].class != Array + data["upload_rpms"].each do |rpm| + raise JSON.dump({ "errcode" => "200", "errmsg" => "#{rpm} not exist" }) unless File.exist?(rpm) + raise JSON.dump({ "errcode" => "200", "errmsg" => "the upload directory is incorrect" }) unless File.dirname(rpm).start_with?("/srv/rpm/upload/") + end + end + + def update_pub_dir(update) + update.each do |rpm| pub_path = File.dirname(rpm).sub("testing", "pub") FileUtils.mkdir_p(pub_path) unless File.directory?(pub_path) -- 2.23.0

1 0

[PATCH compass-ci] container/webhook: change the port which listen to
by Li Yuanchao 19 Apr '21

19 Apr '21

Change from 11301 to 20005 as planned Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/webhook/root/webhook.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/webhook/root/webhook.js b/container/webhook/root/webhook.js index 4639e14..6ca836e 100755 --- a/container/webhook/root/webhook.js +++ b/container/webhook/root/webhook.js @@ -17,4 +17,4 @@ http.createServer(function(req, res){ res.statusCode = 404 res.end('no such location') }) -}).listen(11301) +}).listen(20005) -- 2.23.0

1 0

[PATCH lkp-tests] pkg/mysql/PKGBUILD: fix md5sums
by Lin Jiaxin 19 Apr '21

19 Apr '21

reference: commit bc02d3fe0c13cf85d561a26fdef8dd04c8ede4aa Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- pkg/mysql/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mysql/PKGBUILD b/pkg/mysql/PKGBUILD index 304f6a4a6..c13f07d67 100644 --- a/pkg/mysql/PKGBUILD +++ b/pkg/mysql/PKGBUILD @@ -9,7 +9,7 @@ source=("https://cdn.mysql.com/archives/mysql-8.0/mysql-boost-$pkgver.tar.gz" "https://mirrors.huaweicloud.com/kunpeng/archive/openEuler/patch/database/my…" "https://github.com/thkukuk/rpcsvc-proto/releases/download/v1.4/rpcsvc-proto…" "my.cnf") -md5sums=('7472a25d91973cbba5d8a8f176a3080b' '6e853850efbd6f887ac3e13b1bf99cc4' 'f13eb11de53616017f7428cc06d4e242' 'df1022ccc5d0aff30b8e505735b4a98b' '1390631a1ea1832f9c27d07fcc1e536e') +md5sums=('7472a25d91973cbba5d8a8f176a3080b' '6e853850efbd6f887ac3e13b1bf99cc4' 'f13eb11de53616017f7428cc06d4e242' 'df1022ccc5d0aff30b8e505735b4a98b' 'd90ba2c701507eeac43b06fcfa49c2c1') prepare() { -- 2.23.0

1 0

[PATCH compass-ci] es_query.rb: fix _scroll_id change
by Cui Yili 19 Apr '21

19 Apr '21

[why] _scroll_id will change every time query, so you need to save the last _scroll_id. Signed-off-by: Cui Yili <2268260388(a)qq.com> --- lib/es_query.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 22d68a9..6bd39bf 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -47,11 +47,12 @@ class ESQuery }, size: size } result = @client.search index: @index, scroll: '10m', body: query - @scroll_id = result['_scroll_id'] - return result else - @client.scroll scroll: '10m', scroll_id: @scroll_id + result = @client.scroll scroll: '10m', scroll_id: @scroll_id end + + @scroll_id = result['_scroll_id'] unless result.empty? + return result end def query_by_id(id) -- 2.23.0

2 1

[PATCH compass-ci] sbin/auto_submit: add author_email and committer_email
by Li Yuanchao 19 Apr '21

19 Apr '21

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbin/auto_submit b/sbin/auto_submit index faebb2f..4adce89 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -55,6 +55,9 @@ class AutoSubmit real_argvs.push("upstream_dir=#{belong}") tag = %x(git -C #{mirror_dir} tag --points-at #{commit_id}) real_argvs.push("upstream_tag=#{tag}") unless tag.empty? + author_email = %x(git -C #{mirror_dir} log --format=%ae -1) + committer_email = %x(git -C #{mirror_dir} log --format=%ce -1) + real_argvs.push("author_email=#{author_email} committer_email=#{committer_email}") system(real_argvs.join(' ')) end -- 2.23.0

1 0

[PATCH compass-ci 2/2] scheduler: allow some variables to be used in the job
by Yu Chuan 19 Apr '21

19 Apr '21

[Why] allow two variables: - OS_HTTP_HOST - OS_HTTP_PORT Signed-off-by: Yu Chuan <13186087857(a)163.com> --- sparrow/3-code/dev-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 76686a5f5823..969889edc16b 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -151,6 +151,8 @@ SEND_MAIL_HOST: SEND_MAIL_PORT: INITRD_HTTP_HOST: INITRD_HTTP_PORT: +OS_HTTP_HOST: +OS_HTTP_PORT: ASSIST_RESULT_HOST: ASSIST_RESULT_PORT: ASSISTANT_HOST: -- 2.23.0

1 0

[PATCH compass-ci 1/2] scheduler: allow all jobs with suite=install-iso* to use custom_ipxe
by Yu Chuan 19 Apr '21

19 Apr '21

[Why] Job with suite=install-iso-firstboot needs use custom_ipxe feature. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- src/scheduler/find_job_boot.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 41fa89f0056b..1ccd9e6de644 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -287,7 +287,7 @@ class Sched end private def get_boot_ipxe(job : Job) - return job["custom_ipxe"] if job["suite"] == "install-iso" && job.has_key?("custom_ipxe") + return job["custom_ipxe"] if job["suite"].starts_with?("install-iso") && job.has_key?("custom_ipxe") response = "#!ipxe\n\n" -- 2.23.0

1 0

[PATCH compass-ci] add permission 775 lkp:lkp
by Liu Yinsi 17 Apr '21

17 Apr '21

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/1-storage/permission | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sparrow/1-storage/permission b/sparrow/1-storage/permission index db33a93..3c229c5 100755 --- a/sparrow/1-storage/permission +++ b/sparrow/1-storage/permission @@ -13,6 +13,9 @@ chmod_dir=( /srv/rabbitmq /c/git-repos /srv/tmp/public + /srv/initrd + /srv/initrd/pkg + /srv/initrd/deps ) chmod 775 "${chmod_dir[@]}" @@ -20,6 +23,8 @@ chmod 775 "${chmod_dir[@]}" chown_dir=( /srv/result /srv/initrd + /srv/initrd/pkg + /srv/initrd/deps /srv/es /srv/cache/netdata_cache /srv/cache/netdata_lib -- 2.23.0

1 0

[PATCH compass-ci] etcd/build: remove redundant spaces
by Wu Zhende 17 Apr '21

17 Apr '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/etcd/build | 2 -- 1 file changed, 2 deletions(-) diff --git a/container/etcd/build b/container/etcd/build index 6ae25dd..37c2326 100755 --- a/container/etcd/build +++ b/container/etcd/build @@ -8,5 +8,3 @@ version=v3.4.15 docker_skip_rebuild "etcd-${version}:alpine" docker build -t etcd-${version}:alpine --build-arg VERSION=${version} . - - -- 2.23.0

1 0

[PATCH compass-ci] sbin/es-jobs-mapping.sh: add subqueue
by Wu Zhende 17 Apr '21

17 Apr '21

Use subqueue to search Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- sbin/es-jobs-mapping.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbin/es-jobs-mapping.sh b/sbin/es-jobs-mapping.sh index 9926fd8..67b1ed1 100755 --- a/sbin/es-jobs-mapping.sh +++ b/sbin/es-jobs-mapping.sh @@ -100,6 +100,9 @@ else "queue": { "type": "keyword" }, + "subqueue": { + "type": "keyword" + }, "testbox": { "type": "keyword" }, -- 2.23.0

1 0

[PATCH compass-ci] scheduler/scheduler.cr: add warning logs
by Wu Zhende 17 Apr '21

17 Apr '21

Because set "logging false", so need to print all the logs ourselves. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/scheduler/scheduler.cr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr index ae30937..19a3092 100644 --- a/src/scheduler/scheduler.cr +++ b/src/scheduler/scheduler.cr @@ -40,11 +40,15 @@ module Scheduler env.response.headers["Connection"] = "close" env.create_log env.create_sched + rescue e + env.log.warn(e.inspect_with_backtrace) end after_all do |env| env.sched.etcd_close env.log.info(%({"from": "#{env.request.remote_address}", "message": "access_record"})) + rescue e + env.log.warn(e.inspect_with_backtrace) end # echo alive -- 2.23.0

1 0

[PATCH compass-ci] lib/dump_stat.rb: add message key in cols verifation
by Lu Weitao 17 Apr '21

17 Apr '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/dump_stat.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/dump_stat.rb b/lib/dump_stat.rb index 04a7632..a497d46 100644 --- a/lib/dump_stat.rb +++ b/lib/dump_stat.rb @@ -121,8 +121,6 @@ module DumpStat @max_cols_stat = '' zero_stats = [] @result.each do |key, val| - next if key.end_with?('.message', '.log') - if @max_cols < val.size @max_cols = val.size @max_cols_stat = key -- 2.23.0

1 0

[PATCH compass-ci] add work dir /srv/tmp/public
by Liu Yinsi 17 Apr '21

17 Apr '21

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/1-storage/tiny | 1 + 1 file changed, 1 insertion(+) diff --git a/sparrow/1-storage/tiny b/sparrow/1-storage/tiny index 0eb7d39..c7636b0 100755 --- a/sparrow/1-storage/tiny +++ b/sparrow/1-storage/tiny @@ -26,6 +26,7 @@ dirs=( /srv/cci/Maildir/.compass-ci/cur /srv/cci/Maildir/.compass-ci/sent /srv/rabbitmq + /srv/tmp/public ) mkdir -p "${dirs[@]}" -- 2.23.0

1 0

[PATCH compass-ci] container/assist-result: refactor get_job_stats to get_job_content
by Cao Xueliang 17 Apr '21

17 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assist-result/routes.rb | 6 +++--- .../views/{get_job_stats.rb => get_job_content.rb} | 5 ++--- lib/assist_result_client.rb | 8 ++++++++ src/delimiter/utils.rb | 7 +++++-- 4 files changed, 18 insertions(+), 8 deletions(-) rename container/assist-result/views/{get_job_stats.rb => get_job_content.rb} (67%) diff --git a/container/assist-result/routes.rb b/container/assist-result/routes.rb index 02956aa..287db25 100755 --- a/container/assist-result/routes.rb +++ b/container/assist-result/routes.rb @@ -7,7 +7,7 @@ require 'json' require 'sinatra' require_relative './views/get_job_yaml' -require_relative './views/get_job_stats' +require_relative './views/get_job_content' require_relative './views/check_job_credible' configure do @@ -36,9 +36,9 @@ post '/check_job_credible' do return [200, {'credible' => result}.to_json] end -get '/get_job_stats/:job_id' do +get '/get_job_content/:job_id' do begin - result = get_job_stats(params[:job_id]) + result = get_job_content(params[:job_id]) rescue StandardError => e return [400, e.backtrace.inspect] end diff --git a/container/assist-result/views/get_job_stats.rb b/container/assist-result/views/get_job_content.rb similarity index 67% rename from container/assist-result/views/get_job_stats.rb rename to container/assist-result/views/get_job_content.rb index d294df3..643beb5 100644 --- a/container/assist-result/views/get_job_stats.rb +++ b/container/assist-result/views/get_job_content.rb @@ -4,7 +4,6 @@ require_relative "#{ENV['CCI_SRC']}/lib/es_query" -def get_job_stats(job_id) - content = ESQuery.new.query_by_id(job_id) - return content['stats'] +def get_job_content(job_id) + return ESQuery.new.query_by_id(job_id) end diff --git a/lib/assist_result_client.rb b/lib/assist_result_client.rb index fcf5d9d..9c42b97 100644 --- a/lib/assist_result_client.rb +++ b/lib/assist_result_client.rb @@ -34,4 +34,12 @@ class AssistResult return JSON.parse(response.body) end + + def get_job_content(job_id) + resource = RestClient::Resource.new("http://#{@host}:#{@port}/get_job_content/#{job_id}") + response = resource.get() + return nil unless response.code == 200 + + return response.body + end end diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index dbc42ef..a39d9de 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -108,9 +108,12 @@ module Utils # this is a temporary solution, the extract container will be improved in future. def query_stats(job_id, times) (1..times).each do |i| - new_job = ESQuery.new.query_by_id(job_id) + job_content = AssistResult.new.get_job_content(job_id) puts "query stats times: #{i}" - return new_job['stats'] if new_job['stats'] + if job_content + job_content = JSON.parse(job_content) + return job_content['stats'] if job_content.key?('stats') + end sleep 60 end -- 2.23.0

1 0

[PATCH v2 lkp-tests] script to deploy ceph for multi server tests
by Luan Shengde 16 Apr '21

16 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/ceph-init | 155 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100755 tests/ceph-init diff --git a/tests/ceph-init b/tests/ceph-init new file mode 100755 index 000000000..59398d1bd --- /dev/null +++ b/tests/ceph-init @@ -0,0 +1,155 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# - cephuser_password +# - cephuser + +: "${cephuser:=cephuser}" +: "${cephuser_password:=cephuser_password}" + +### ===== on each node ===== +# stop firewalld and disable selinx +systemctl stop firewalld +systemctl disable firewalld + +sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/sysconfig/selinux + +until [[ -n $node_roles ]]; do + sleep 1 +done + +hosts=(node1 node2 node3) + +[[ $node_roles == node1 ]] && { + hostnamectl set-hostname node1 +} + +[[ $node_roles == node2 ]] && { + hostnamectl set-hostname node2 +} + +[[ $node_roles == node3 ]] && { + hostnamectl set-hostname node3 +} + +yum update && { + yum install ntp wget gcc make ceph lvm2 python3 -y +} + +# add user and set sudo permissions +useradd -d /home/$cephuser -m $cephuser && { + echo $cephuser:$cephuser_password | chpasswd +} + +cat >> /etc/sudoers <<-EOF +$cephuser ALL=(ALL) NOPASSWD:ALL +EOF + +# check and install pip +[[ $(command -v pip) ]] || { + until wget https://bootstrap.pypa.io/get-pip.py; do + sleep 2 + done + python get-pip.py +} + +pip install prettytable + +### ===== on ceph-deploy node ===== +# get node IPs is first +[[ $node_roles == node1 ]] || exit 0 + +until [[ -n $node1 ]]; do + sleep 1 +done + +until [[ -n $node2 ]]; do + sleep 1 +done + +until [[ -n $node3 ]]; do + sleep 1 +done + +# config hosts and set default ssh users +cat >> /etc/hosts <<-EOF +$node1 node1 node1 +$node2 node2 node2 +$node3 node3 node3 +EOF + +for host in "${hosts[@]}"; do + cat >> "$HOME/.ssh/config" <<-EOF + Host $host + Hostname $host + User $cephuser + EOF +done + +chmod 0600 "$HOME/.ssh/config" + +network=$(ip route | grep default | awk '{print $3}' | awk -F'.' '{print $1"."$2}') +net_mask=$(ip route | grep "^$network" | awk '{print $1}') + +# do the deploy until all the servers are up +# for host in "${host[@]}"; do +# until ping -c 1 -W 1 172.168.131.100> /dev/null; do +# sleep 2 +# done +# done + +# set server trust +[[ $(command -v sshpass) ]] || { + until wget http://sourceforge.net/projects/sshpass/files/latest/download -O sshpass.tar.gz; do + sleep 2 + done +} + +tar xzvf sshpass.tar.gz && { + cd sshpass-1.08 || exit 1 + ./configure + make + make install +} + +rm -rf "$HOME/.ssh/id_rsa" "$HOME/.ssh/id_rsa.pub" && { + ssh-keygen -f "$HOME/.ssh/id_rsa" -t rsa -N '' +} + +sed -i '/StrictHostKeyChecking/c StrictHostKeyChecking no' /etc/ssh/ssh_config +for host in "${hosts[@]}"; do +{ + sshpass -p $cephuser_password ssh-copy-id "$cephuser"@"$host" &> /dev/null +}& +done +wait + +# install +pip install ceph-deploy + +pip_dir=$(pip show ceph-deploy | awk '/^Location/ {print $NF}') +ceph_init_file="${pip_dir}/ceph_deploy/hosts/__init__.py" +[[ -f $ceph_init_file ]]&& { + sed -i "/distributions = {/a 'openeuler': fedora," "$ceph_init_file" +} + +# deploy ceph cluster +disk_name=$(sed -e '1d' -e '/^$/d' /proc/partitions | grep " .da" | awk '{print $NF}') +disk_device="/dev/${disk_name}" + +[[ -d "$HOME/ceph-deploy" ]] || { + mkdir -p "$HOME/ceph-deploy" +} + +cd "$HOME/ceph-deploy" && { + ceph-deploy new "${hosts[@]}" + echo "public_network = ${net_mask}" >> "$HOME/ceph-deploy/ceph.conf" + ceph-deploy mon create-initial + ceph-deploy --overwrite-conf admin "${hosts[@]}" + ceph-deploy mgr create "${hosts[@]}" + for host in "${hosts[@]}"; do + ceph-deploy osd create --data "$disk_device" "$host" + done + wait + ceph osd pool create ceph-osd-pool 128 +} -- 2.23.0

2 2

[PATCH v2 compass-ci] delimiter: consume bisect process job by the bad job tbox_group
by Cao Xueliang 16 Apr '21

16 Apr '21

Sometimes we set a queue in a job, then we delete the testbox who can consume the queue, if we submit the job to the queue, no testbox can consume it, so we consume the bisect process job by the bad job tbox_group. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter/utils.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index dbc42ef..1aac6b3 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -137,9 +137,10 @@ module Utils job['my_email'] = ENV['my_email'] job['my_token'] = ENV['secrets_my_token'] job['bad_job_id'] = job_id - job['queue'] = "#{job['testbox'].split('.')[0]}-bisect" + job['testbox'] = job['tbox_group'] job.delete('id') + job.delete('queue') return job end -- 2.23.0

1 0

[PATCH v1 compass-ci 1/2] delimiter: consume job by the bad job queue
by Cao Xueliang 16 Apr '21

16 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter/utils.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index dbc42ef..a0d4196 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -137,7 +137,6 @@ module Utils job['my_email'] = ENV['my_email'] job['my_token'] = ENV['secrets_my_token'] job['bad_job_id'] = job_id - job['queue'] = "#{job['testbox'].split('.')[0]}-bisect" job.delete('id') -- 2.23.0

1 1

[PATCH v1 compass-ci 2/2] container/etcd: fix Error reading socket: Connection reset by peer
by Cao Xueliang 16 Apr '21

16 Apr '21

We should pass the start script params to /bin/etcd program. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/etcd/entry.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container/etcd/entry.sh b/container/etcd/entry.sh index 4bfea1e..16c8d00 100755 --- a/container/etcd/entry.sh +++ b/container/etcd/entry.sh @@ -3,6 +3,6 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. crond -/bin/etcd -exec "$@" +/bin/etcd "$@" +exec -- 2.23.0

1 0

[PATCH compass-ci 4/4] sparrow: expose ASSISTANT_PORT/HOST in scheduler
by Cao Xueliang 16 Apr '21

16 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- sparrow/3-code/dev-env | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index bb576c7..76686a5 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -28,6 +28,8 @@ server_name=$(hostname | cut -f1 -d.) : ${INITRD_HTTP_PORT:=8800} : ${assist_result_host:=$server_ip} : ${ASSIST_RESULT_PORT:=8102} +: ${assistant_host:=$server_ip} +: ${ASSISTANT_PORT:=8101} : ${result_webdav_host:=$server_ip} : ${RESULT_WEBDAV_PORT:=3080} : ${MASTER_FLUENTD_HOST:=$server_ip} @@ -107,6 +109,8 @@ INITRD_HTTP_HOST: $initrd_http_host INITRD_HTTP_PORT: $INITRD_HTTP_PORT ASSIST_RESULT_HOST: $assist_result_host ASSIST_RESULT_PORT: $ASSIST_RESULT_PORT +ASSISTANT_HOST: $assistant_host +ASSISTANT_PORT: $ASSISTANT_PORT RESULT_WEBDAV_HOST: $result_webdav_host RESULT_WEBDAV_PORT: $RESULT_WEBDAV_PORT MASTER_FLUENTD_HOST: $MASTER_FLUENTD_HOST @@ -149,6 +153,8 @@ INITRD_HTTP_HOST: INITRD_HTTP_PORT: ASSIST_RESULT_HOST: ASSIST_RESULT_PORT: +ASSISTANT_HOST: +ASSISTANT_PORT: RESULT_WEBDAV_HOST: RESULT_WEBDAV_PORT: DOCKER_REGISTRY_HOST: -- 2.23.0

1 0

[PATCH compass-ci 3/4] lib: add assistant_client and use it in mail_bisect_result
by Cao Xueliang 16 Apr '21

16 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/assistant_client.rb | 24 ++++++++++++++++++++++++ lib/constants.rb | 3 +++ lib/mail_bisect_result.rb | 4 ++-- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 lib/assistant_client.rb diff --git a/lib/assistant_client.rb b/lib/assistant_client.rb new file mode 100644 index 0000000..c951f2b --- /dev/null +++ b/lib/assistant_client.rb @@ -0,0 +1,24 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'base64' +require 'rest-client' +require_relative 'constants' + +class AssistantClient + def initialize(host = ASSISTANT_HOST, port = ASSISTANT_PORT) + @host = host + @port = port + end + + def get_mail_list(type) + resource = RestClient::Resource.new("http://#{@host}:#{@port}/get_mail_list/#{type}") + response = resource.get() + return nil unless response.code == 200 + + return JSON.parse(response.body) + end +end diff --git a/lib/constants.rb b/lib/constants.rb index 03b1e40..11927f9 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -20,6 +20,9 @@ SRV_HTTP_GIT_PORT ||= config['SRV_HTTP_GIT_PORT'] || ENV['SRV_HTTP_GIT_PORT'] || SRV_HTTP_DOMAIN ||= config['SRV_HTTP_DOMAIN'] || ENV['SRV_HTTP_DOMAIN'] || 'api.compass-ci.openeuler.org' +ASSISTANT_HOST ||= config['ASSISTANT_HOST'] || ENV['ASSISTANT_HOST'] || '172.17.0.1' +ASSISTANT_PORT ||= config['ASSISTANT_PORT'] || ENV['ASSISTANT_PORT'] || 8101 + ASSIST_RESULT_HOST ||= config['ASSIST_RESULT_HOST'] || ENV['ASSIST_RESULT_HOST'] || '172.17.0.1' ASSIST_RESULT_PORT ||= config['ASSIST_RESULT_PORT'] || ENV['ASSIST_RESULT_PORT'] || 8102 diff --git a/lib/mail_bisect_result.rb b/lib/mail_bisect_result.rb index 782c702..5a71225 100644 --- a/lib/mail_bisect_result.rb +++ b/lib/mail_bisect_result.rb @@ -6,7 +6,7 @@ require_relative 'git' require_relative 'es_query' require_relative 'constants' require_relative 'mail_client' -require_relative 'parse_mail_list' +require_relative 'assistant_client' require_relative 'assign_account_client' # compose and send email for bisect result @@ -25,7 +25,7 @@ class MailBisectResult end def parse_mail_info - mail_hash = parse_mail_list('delimiter') + mail_hash = AssistantClient.new.get_mail_list('delimiter') @to = mail_hash['to'] if mail_hash.key?('to') @bcc = mail_hash['bcc'] if mail_hash.key?('bcc') raise 'Need to add bcc email for bisect report.' unless @bcc -- 2.23.0

1 0

[PATCH compass-ci 2/4] container/assistant: refactor assistant container
by Cao Xueliang 16 Apr '21

16 Apr '21

1. Refactor assistant container. 2. Add get_mail_list function. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assistant/Dockerfile | 17 ++++---- container/assistant/build | 4 +- container/assistant/routes.rb | 20 ++++++--- container/assistant/start | 51 +++++++++++++++------- container/assistant/views/get_mail_list.rb | 9 ++++ 5 files changed, 69 insertions(+), 32 deletions(-) create mode 100644 container/assistant/views/get_mail_list.rb diff --git a/container/assistant/Dockerfile b/container/assistant/Dockerfile index 04d2da0..3299858 100644 --- a/container/assistant/Dockerfile +++ b/container/assistant/Dockerfile @@ -1,16 +1,15 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM debian +FROM alpine:3.11 MAINTAINER Cao Xueliang <caoxl78320(a)163.com> -ENV DEBIAN_FRONTEND noninteractive +RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories && \ + apk update && \ + apk add --no-cache 'ruby-dev' 'ruby-etc' \ + 'g++' 'gcc' 'pcre' 'libevent' 'make' 'git' 'bash' 'grep' 'coreutils' 'curl' 'util-linux' -RUN apt-get update && \ - apt-get install -y ruby-json ruby-sinatra curl - -COPY routes.rb /usr/local/bin/ -COPY views /usr/local/bin/views/ - -CMD ["/usr/local/bin/routes.rb"] +RUN umask 002 && \ + gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ + gem install rest-client activesupport json yaml elasticsearch sinatra puma terminal-table diff --git a/container/assistant/build b/container/assistant/build index 89a8a97..6f739d6 100755 --- a/container/assistant/build +++ b/container/assistant/build @@ -4,6 +4,6 @@ . ../defconfig.sh -docker_skip_rebuild "debian:assistant" +docker_skip_rebuild "assistant" -docker build -t debian:assistant . +docker build -t assistant . diff --git a/container/assistant/routes.rb b/container/assistant/routes.rb index 54d3670..232ec8c 100755 --- a/container/assistant/routes.rb +++ b/container/assistant/routes.rb @@ -8,24 +8,32 @@ require 'json' require 'open3' require_relative './views/locate_files' +require_relative './views/get_mail_list' -set :bind, '0.0.0.0' -set :port, 8101 +configure do + set :bind, '0.0.0.0' + set :port, ENV['ASSISTANT_PORT'] +end post '/locate_files' do request.body.rewind begin data = JSON.parse request.body.read - rescue JSON::ParserError - return [400, 'parse json params error'] + result = locate_files(data) + rescue StandardError => e + return [400, e.backtrace.inspect] end + [200, result.to_json] +end + +get '/get_mail_list/:type' do begin - result = locate_files(data) + result = get_mail_list(params[:type]) rescue StandardError => e return [400, e.backtrace.inspect] end - [200, result.to_json] + return [200, result.to_json] end diff --git a/container/assistant/start b/container/assistant/start index 5f7822e..87c1801 100755 --- a/container/assistant/start +++ b/container/assistant/start @@ -1,21 +1,42 @@ -#!/bin/bash +#!/usr/bin/env ruby # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true -. $CCI_SRC/container/defconfig.sh +require 'set' +require_relative '../defconfig.rb' -docker_rm assistant +names = Set.new %w[ + ASSISTANT_HOST + ASSISTANT_PORT +] -cmd=( - docker run - --restart=always - --name assistant - -u nobody - -d - -p 8101:8101 - -v /srv/initrd:/srv/initrd - -v /etc/localtime:/etc/localtime:ro - debian:assistant -) +defaults = relevant_defaults(names) +env = docker_env(defaults) -"${cmd[@]}" +DEFAULT_LKP = '/c/lkp-tests' +DEFAULT_CCI = '/c/compass-ci' +DEFAULT_CONFIG_DIR = '/etc/compass-ci' +ASSISTANT_PORT = defaults['ASSISTANT_PORT'] +docker_rm 'assistant' + +cmd = %w[ + docker run + --name assistant + --restart=always + -d +] + env + %W[ + -e LKP_SRC=#{DEFAULT_LKP} + -e CCI_SRC=#{DEFAULT_CCI} + -p #{ASSISTANT_PORT}:#{ASSISTANT_PORT} + -v #{ENV['LKP_SRC']}:#{DEFAULT_LKP} + -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} + -v #{DEFAULT_CONFIG_DIR}:#{DEFAULT_CONFIG_DIR}:ro + -v /etc/localtime:/etc/localtime:ro + -w #{DEFAULT_CCI}/container/assistant/ + assistant +] + +cmd += ['sh', '-c', 'umask 002 && ruby ./routes.rb'] + +system(*cmd) diff --git a/container/assistant/views/get_mail_list.rb b/container/assistant/views/get_mail_list.rb new file mode 100644 index 0000000..66a1872 --- /dev/null +++ b/container/assistant/views/get_mail_list.rb @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative "#{ENV['CCI_SRC']}/lib/parse_mail_list" + +def get_mail_list(type) + return parse_mail_list(type) +end -- 2.23.0

1 0

[PATCH compass-ci 1/4] delimiter: consume job by the bad job queue
by Cao Xueliang 16 Apr '21

16 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter/utils.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index dbc42ef..a0d4196 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -137,7 +137,6 @@ module Utils job['my_email'] = ENV['my_email'] job['my_token'] = ENV['secrets_my_token'] job['bad_job_id'] = job_id - job['queue'] = "#{job['testbox'].split('.')[0]}-bisect" job.delete('id') -- 2.23.0

1 0

[PATCH compass-ci] container/crystal-shards: fix executing command: shards version got exit 1
by Cao Xueliang 16 Apr '21

16 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/crystal-shards/shard.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/container/crystal-shards/shard.yml b/container/crystal-shards/shard.yml index 3466bc5..b9662af 100644 --- a/container/crystal-shards/shard.yml +++ b/container/crystal-shards/shard.yml @@ -35,3 +35,4 @@ dependencies: branch: master amqp-client: github: cloudamqp/amqp-client.cr + version: 1.0.0 -- 2.23.0

1 0

[PATCH compass-ci 2/2] lib/sched.cr: add tbox_group to testbox
by Wu Zhende 16 Apr '21

16 Apr '21

It can be used to display statistics on a certain type of machine. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 1359f95..7a9e4d8 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -174,10 +174,12 @@ class Sched job["job_state"] = "boot" state = "booting" job_id = job["id"] + tbox_group = job["tbox_group"] else deadline = nil job_id = "" state = "requesting" + tbox_group = JobHelper.match_tbox_group(testbox.to_s) end type = get_type(testbox) @@ -189,7 +191,8 @@ class Sched "deadline" => deadline, "queues" => queues, "type" => type, - "name" => testbox + "name" => testbox, + "tbox_group" => tbox_group } @redis.update_wtmp(testbox.to_s, hash) -- 2.23.0

1 0

[PATCH compass-ci 1/2] lib/job.cr: fix tbox_group generation rule
by Wu Zhende 16 Apr '21

16 Apr '21

[Why] The tbox generated by this method is different from the tbox generated by the lkp submit method. testbox: dc-8g.wuzhende-3455481 this method get: dc-8g.wuzhende lkp mothed get: dc-8g Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/job.cr | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index b6bb730..200ba2b 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -26,11 +26,7 @@ end module JobHelper def self.match_tbox_group(testbox : String) - tbox_group = testbox - if testbox =~ /(.*)-\d+$/ - tbox_group = $1 - end - return tbox_group + testbox.split(/\.|--/)[0] end def self.service_path(path) -- 2.23.0

1 0

[PATCH compass-ci] container/dracut-initrd: set pre-mount service to be wanted by kernel param local
by Xu Xijian 15 Apr '21

15 Apr '21

The local mount is dealt in dracut-pre-mount stage in dracut, however sometime this stage is skipped. So add "ConditionKernelCommandLine=|local" to make sure kernel param local can trigger dracut-pre-mount service. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index a57d681..f149556 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -20,6 +20,9 @@ sed -i "/install() {/a\ inst /usr/bin/awk" /usr/lib/dracut/modules.d/40networ tools_00bash="/sbin/e2fsck /sbin/mke2fs /usr/bin/basename /sbin/lvm /sbin/reboot" sed -i "/install() {/a\ inst $tools_00bash" /usr/lib/dracut/modules.d/00bash/module-setup.sh -pre_mount_file="/usr/lib/dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh" +dracut_systemd_dir="/usr/lib/dracut/modules.d/98dracut-systemd" +pre_mount_service="${dracut_systemd_dir}/dracut-pre-mount.service" +sed -i "/Description/aConditionKernelCommandLine=|local" "$pre_mount_service" +pre_mount_file="${dracut_systemd_dir}/dracut-pre-mount.sh" [ "$(sed -n '$p' $pre_mount_file)" = "exit 0" ] && sed -i '$d' "$pre_mount_file" cat set-local-sysroot.sh >> "$pre_mount_file" -- 2.23.0

1 0

help star
by Wu Fengguang 15 Apr '21

15 Apr '21

Wei Jihui is our help star, thanks Wei Jihui! baijing@crystal ~/todo/bin% ./help-star 5.9 Wei Jihui 4.8 Liu Shaofei 3.1 Xu Xijian 2.8 Yu Chuan 2.7 Zhang Dewan 2.3 Wang Yong 2.0 Wu Fengguang 1.8 Cao Xueliang 1.8 Luan Shengde 1.7 Liu Yinsi 1.3 Wu Zhende 0.0 Bai Jing 0.0 Du Kaitian 0.0 Lu Weitao 0.0 Zhang Yale 0.0 Cui Yili 0.0 Wang Chenglong 0.0 Li Yuanchao 0.0 Lin Jiaxin 0.0 Li Ping

1 0

review star
by Wu Fengguang 15 Apr '21

15 Apr '21

: wqUser-Agent: Mutt/1.10.1 (2018-07-13) User-Agent: Mutt/1.10.1 (2018-07-13) Cao Xueliang is our review star, thanks Cao Xueliang! baijing@crystal ~/todo/bin% ./review-star Review email records in the last 7 days total review paragraph point people 75 4 4 15 Cao Xueliang 15 4 10 12 Yu Chuan 21 5 5 8 Wei Jihui 24 6 11 6 Liu Shaofei 11 7 16 4 Luan Shengde 3 1 3 3 Xu Xijian 13 3 3 1 Lin Jiaxin 4 3 6 1 Li Yuanchao 12 1 1 1 Bai Jing 9 0 0 0 Wu Zhende 14 0 0 0 Cui Yili 9 1 8 0 Wang Yong 3 0 0 0 Li Ping 10 2 3 0 Lu Weitao 5 0 0 0 Wu Fengguang 21 0 0 0 Liu Yinsi 11 0 0 0 Wang Chenglong

1 0

git count
by Wu Fengguang 15 Apr '21

15 Apr '21

Cao Xueliang is our code star, thanks Cao Xueliang! baijing@crystal ~/todo/bin% ./git-count 13 Cao Xueliang 7 Wu Zhende 6 Liu Shaofei 6 Lu Weitao 6 Liu Yinsi 4 Wei Jihui 3 Bai Jing 2 Xu Xijian 2 Cui Yili 2 Wang Yong 2 Li Yuanchao 2 Lin Jiaxin 1 Yu Chuan 1 Wang Chenglong 1 Luan Shengde 0 Du Kaitian 0 Wu Fengguang 0 Zhang Yale 0 Zhang Dewan 0 Li Ping

1 0

[PATCH lkp-tests] setup cluster for multi server(3 or more)
by Luan Shengde 15 Apr '21

15 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- cluster/cs-s1-a122-c2 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 cluster/cs-s1-a122-c2 diff --git a/cluster/cs-s1-a122-c2 b/cluster/cs-s1-a122-c2 new file mode 100644 index 000000000..a69179eab --- /dev/null +++ b/cluster/cs-s1-a122-c2 @@ -0,0 +1,14 @@ +switch: Switch-P12 +ip0: 1 +nodes: + taishan200-2280-2s64p-256g--a122: + roles: [ node1 ] + macs: [ "44:a1:91:9d:d8:25" ] + + taishan200-2280-2s64p-256g--a123: + roles: [ node2 ] + macs: [ "44:a1:91:9d:f5:4a" ] + + taishan200-2280-2s64p-256g--a125: + roles: [ node3 ] + macs: [ "44:67:47:85:f4:65" ] -- 2.23.0

1 0

[PATCH lkp-tests] tests/build-pkg: matching url is not printed in function replace_source
by Lin Jiaxin 15 Apr '21

15 Apr '21

Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- tests/build-pkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build-pkg b/tests/build-pkg index 350d8fdf..b399fdc3 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -122,7 +122,7 @@ replace_source() echo "source=(" > $1 for url in ${source[@]} do - echo "$url" | egrep 'git\+|\.git$' && { + echo "$url" | egrep 'git\+|\.git$' > /dev/null && { [ -n "$repo_dir" ] && { url="${repo_dir}git://${LKP_SERVER}/${upstream_dir}/${upstream_repo}#commit=${upstream_commit}" [ -n "$git_tag" ] && { -- 2.23.0

1 0

[PATCH v2 lkp-tests] jobs/rpmbuild-pkg.yaml: remove upstream_repo
by Lin Jiaxin 15 Apr '21

15 Apr '21

Assigning a null value to upstream_repo is meaningless. And when use "submit upstream_repo=xxx rpmbuild-pkg.yaml" will make upstream_repo be null. Unless we put "upstream_repo=xxx" behind. Or declare it in yaml. [example] linjiaxin@crystal ~%/home/linjiaxin/lkp-tests/sbin/submit upstream_repo="a/azk/azk" upstream_url=https://github.com/azukiapp/azk pkgbuild_repo=archlinux/aur-a/azk-git testbox=dc-16g os=openeuler os_version=20.03-LTS os_mount=container os_arch=aarch64 docker_image=openeuler:20.03-LTS repo_name=aur rpmbuild-pkg.yaml upstream_branch=master upstream_commit=e72adb039e4dd64fd6ed81d82fd597c1d4efdf14 upstream_dir=upstream submit_id=d18fabcd-6302-464c-aa99-3ed9e610704b submit /home/linjiaxin/lkp-tests/jobs/rpmbuild-pkg.yaml, got job id=crystal.1594407 linjiaxin@crystal ~% es-find id=crystal.1594407 | grep upstream_repo "upstream_repo": null, Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- jobs/rpmbuild-pkg.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/jobs/rpmbuild-pkg.yaml b/jobs/rpmbuild-pkg.yaml index a70590af..5860ba02 100644 --- a/jobs/rpmbuild-pkg.yaml +++ b/jobs/rpmbuild-pkg.yaml @@ -1,5 +1,4 @@ suite: rpmbuild-pkg category: functional -upstream_repo: rpmbuild-pkg: -- 2.23.0

2 2

[PATCH lkp-tests] jobs/rpmbuild-pkg.yaml: remove upstream_repo
by Lin Jiaxin 14 Apr '21

14 Apr '21

Assigning a null value to upstream_repo is meaningless. And when use "submit upstream_repo=xxx rpmbuild-pkg" will make upstream_repo be null. Unless we put "upstream_repo=xxx" behind. Or declare it in yaml. [example] linjiaxin@crystal ~%/home/linjiaxin/lkp-tests/sbin/submit upstream_repo="a/azk/azk" upstream_url=https://github.com/azukiapp/azk pkgbuild_repo=archlinux/aur-a/azk-git testbox=dc-16g os=openeuler os_version=20.03-LTS os_mount=container os_arch=aarch64 docker_image=openeuler:20.03-LTS repo_name=aur rpmbuild-pkg.yaml upstream_branch=master upstream_commit=e72adb039e4dd64fd6ed81d82fd597c1d4efdf14 upstream_dir=upstream submit_id=d18fabcd-6302-464c-aa99-3ed9e610704b submit /home/linjiaxin/lkp-tests/jobs/rpmbuild-pkg.yaml, got job id=crystal.1594407 linjiaxin@crystal ~% es-find id=crystal.1594407 | grep upstream_repo "upstream_repo": null, Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- jobs/rpmbuild-pkg.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/jobs/rpmbuild-pkg.yaml b/jobs/rpmbuild-pkg.yaml index a70590af..5860ba02 100644 --- a/jobs/rpmbuild-pkg.yaml +++ b/jobs/rpmbuild-pkg.yaml @@ -1,5 +1,4 @@ suite: rpmbuild-pkg category: functional -upstream_repo: rpmbuild-pkg: -- 2.23.0

1 0

[PATCH lkp-tests] tests: script to deploy ceph for multi server test
by Luan Shengde 14 Apr '21

14 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/ceph-init | 152 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100755 tests/ceph-init diff --git a/tests/ceph-init b/tests/ceph-init new file mode 100755 index 000000000..735376df6 --- /dev/null +++ b/tests/ceph-init @@ -0,0 +1,152 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# - cephuser_password +# - cephuser + +set -x + +: "${cephuser:=cephuser}" +: "${cephuser_password:=cephuser_password}" + +### ===== on each node ===== +# stop firewalld and disable selinx +systemctl stop firewalld +systemctl disable firewalld + +sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/sysconfig/selinux + +until [[ -n $node_roles ]]; do + sleep 1 +done + +hosts=(cephnode1 cephnode2 cephnode3) + +[[ $node_roles == cephnode1 ]] && { + hostnamectl set-hostname cephnode1 +} + +[[ $node_roles == cephnode2 ]] && { + hostnamectl set-hostname cephnode2 +} + +[[ $node_roles == cephnode3 ]] && { + hostnamectl set-hostname cephnode3 +} + +yum update +yum install ntp wget gcc make ceph lvm2 python3 -y + +# add user and set sudo permissions +useradd -d /home/$cephuser -m $cephuser +echo $cephuser:$cephuser_password | chpasswd + +cat >> /etc/sudoers <<-EOF +$cephuser ALL=(ALL) NOPASSWD:ALL +EOF + +# check and install pip +[[ $(command -v pip) ]] || { + until wget https://bootstrap.pypa.io/get-pip.py; do + sleep 2 + done + python get-pip.py +} + +pip install prettytable + +### ===== on ceph-deploy node ===== +# get node IPs is first +[[ $node_roles == cephnode1 ]] || exit 0 + +until [[ -n $cephnode1 ]]; do + sleep 1 +done + +until [[ -n $cephnode2 ]]; do + sleep 1 +done + +until [[ -n $cephnode3 ]]; do + sleep 1 +done + +# config hosts and set default ssh users +cat >> /etc/hosts <<-EOF +$cephnode1 cephnode1 cephnode1 +$cephnode2 cephnode2 cephnode2 +$cephnode3 cephnode3 cephnode3 +EOF + +for host in "${hosts[@]}"; do + cat >> "$HOME/.ssh/config" <<-EOF + Host $host + Hostname $host + User $cephuser + EOF +done + +chmod 0600 "$HOME/.ssh/config" + + +network=$(ip route | grep default | awk '{print $3}' | awk -F'.' '{print $1"."$2}') +net_mask=$(ip route | grep "^$network" | awk '{print $1}') + +# do the deploy until all the servers are up +# for host in "${host[@]}"; do +# until ping -c 1 -W 1 172.168.131.100> /dev/null; do +# sleep 2 +# done +# done + +# set server trust +[[ $(command -v sshpass) ]] || { + until wget http://sourceforge.net/projects/sshpass/files/latest/download -O sshpass.tar.gz; do + sleep 2 + done +} + +tar xzvf sshpass.tar.gz && { + cd sshpass-1.08 || exit 1 + ./configure + make + make install +} + +rm -rf "$HOME/.ssh/id_rsa" "$HOME/.ssh/id_rsa.pub" +ssh-keygen -f "$HOME/.ssh/id_rsa" -t rsa -N '' + +sed -i '/StrictHostKeyChecking/c StrictHostKeyChecking no' /etc/ssh/ssh_config +for host in "${hosts[@]}"; do +{ + sshpass -p $cephuser_password ssh-copy-id "$cephuser"@"$host" &> /dev/null +}& +done +wait + +# install +pip install ceph-deploy + +pip_dir=$(pip show ceph-deploy | awk '/^Location/ {print $NF}') +ceph_init_file="${pip_dir}/ceph_deploy/hosts/__init__.py" +[[ -f $ceph_init_file ]]&& { + sed -i "/distributions = {/a 'openeuler': fedora," "$ceph_init_file" +} + +# deploy ceph cluster +disk_name=$(sed -e '1d' -e '/^$/d' /proc/partitions | grep " .da" | awk '{print $NF}') +disk_device="/dev/${disk_name}" + +mkdir -p "$HOME/ceph-deploy" +cd "$HOME/ceph-deploy" && { + ceph-deploy new "${hosts[@]}" + echo "public_network = ${net_mask}" >> "$HOME/ceph-deploy/ceph.conf" + ceph-deploy mon create-initial + ceph-deploy --overwrite-conf admin "${hosts[@]}" + ceph-deploy mgr create "${hosts[@]}" + for host in "${hosts[@]}"; do + ceph-deploy osd create --data "$disk_device" "$host" + done + wait + ceph osd pool create ceph-osd-pool 128 +} -- 2.23.0

2 2

[PATCH compass-ci] service/scheduler: print access logs in JSON format
by Wu Zhende 14 Apr '21

14 Apr '21

Kermal log format is as follows: 172.17.0.1 - - [13/Apr/2021:07:53:18 +0800] "GET/~lkp/cgi-bin/lkp-post-run?job_id=crystal.673&source=lifecycle&job_ste=timeout HTTP/1.1" 200 4 "-" "Crystal" change to json: {"level_num":1,"level":"INFO","time":"2021-04-13T16:29:15.064+0800","from":"172.168.131.113:60512","message":"access_record","status_code":200,"method":"PUT","resource":"el_host2queues?host=dc-8g.wuzhende-361524","elapsed":"496.32µs"} Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 1 - src/scheduler/scheduler.cr | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index f62f83e..58ab8e6 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -54,7 +54,6 @@ class Sched end def alive(version) - debug_message("Env= {\n#{`export`}}") "LKP Alive! The time is #{Time.local}, version = #{version}" rescue e @log.warn(e) diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr index ee13fd3..ae30937 100644 --- a/src/scheduler/scheduler.cr +++ b/src/scheduler/scheduler.cr @@ -31,6 +31,7 @@ require "../lib/json_logger" # module Scheduler VERSION = "0.2.0" + logging false add_context_storage_type(Time::Span) @@ -43,6 +44,7 @@ module Scheduler after_all do |env| env.sched.etcd_close + env.log.info(%({"from": "#{env.request.remote_address}", "message": "access_record"})) end # echo alive -- 2.23.0

2 1

[PATCH compass-ci 1/2] container/srv-http: add /srv/initrd/osimage
by Liu Yinsi 14 Apr '21

14 Apr '21

for user locally deploy compass-ci, can download osimage to run qemu kvm when os_mount=initramfs, and add env SRV_HTTP_INITRD_PORT Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/srv-http/start-initrd | 1 + sparrow/3-code/dev-env | 2 ++ 2 files changed, 3 insertions(+) diff --git a/container/srv-http/start-initrd b/container/srv-http/start-initrd index 93be548..457465b 100755 --- a/container/srv-http/start-initrd +++ b/container/srv-http/start-initrd @@ -14,6 +14,7 @@ cmd=( -v /srv/initrd/qemu-image:/srv/initrd/qemu-image:ro -v /srv/initrd/dockerimage:/srv/initrd/dockerimage:ro -v /srv/initrd/rpmbuild-pkg:/srv/initrd/rpmbuild-pkg:ro + -v /srv/initrd/osimage:/srv/initrd/osimage:ro -v /etc/localtime:/etc/localtime:ro -d srv-http diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 3aeba90..bb576c7 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -18,6 +18,7 @@ server_name=$(hostname | cut -f1 -d.) : ${srv_http_git_host:=$server_ip} : ${srv_http_cci_host:=$server_ip} : ${SRV_HTTP_RESULT_PORT:=20007} +: ${SRV_HTTP_INITRD_PORT:=20008} : ${SRV_HTTP_OS_PORT:=20009} : ${SRV_HTTP_GIT_PORT:=20010} : ${SRV_HTTP_CCI_PORT:=20011} @@ -92,6 +93,7 @@ SRV_HTTP_OS_HOST: $srv_http_os_host SRV_HTTP_GIT_HOST: $srv_http_git_host SRV_HTTP_CCI_HOST: $srv_http_cci_host SRV_HTTP_RESULT_PORT: $SRV_HTTP_RESULT_PORT +SRV_HTTP_INITRD_PORT: $SRV_HTTP_INITRD_PORT SRV_HTTP_OS_PORT: $SRV_HTTP_OS_PORT SRV_HTTP_GIT_PORT: $SRV_HTTP_GIT_PORT SRV_HTTP_CCI_PORT: $SRV_HTTP_CCI_PORT -- 2.23.0

2 1

[PATCH compass-ci 2/2] sbin: add download-osimage script
by Liu Yinsi 14 Apr '21

14 Apr '21

for user locally deploy compass-ci, if testbox=vm-*, default os_mount=initramfs, so need to use osimage instead of os rootfs, delete download-rootfs script. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/download-osimage | 36 ++++++++++++++++++++++++++++++++++++ sbin/download-rootfs | 40 ---------------------------------------- sparrow/5-build/os-ready | 2 +- 3 files changed, 37 insertions(+), 41 deletions(-) create mode 100755 sbin/download-osimage delete mode 100755 sbin/download-rootfs diff --git a/sbin/download-osimage b/sbin/download-osimage new file mode 100755 index 0000000..f041846 --- /dev/null +++ b/sbin/download-osimage @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +# download osimage. +# 1. download default osimage openeuler aarch64 20.03 +# example usage: +# ./download-osimage +# 2. download osimage you want by add params $os, $os_arch, $os_version. +# example usage: +# ./download-osimage debian aarch64 sid + +os=${1:-openeuler} +os_arch=${2:-$(arch)} +os_version=${3:-20.03} + +. ${CCI_SRC}/container/defconfig.sh + +load_cci_defaults + +rootfs_dir=/initrd/osimage/${os}/${os_arch}/${os_version} + +download_osimage() +{ + mkdir -p /srv/$rootfs_dir + cd $rootfs_dir + + wget http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/$rootfs_dir/cur… + wget http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/$rootfs_dir/run… +} + +is_virtual() +{ + dmidecode -s system-product-name | grep -iq "virtual" +} + +is_virtual || download_osimage diff --git a/sbin/download-rootfs b/sbin/download-rootfs deleted file mode 100755 index f8c245d..0000000 --- a/sbin/download-rootfs +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. - -# download os rootfs. -# 1. download default os rootfs openeuler aarch64 20.03 -# example usage: -# ./download-rootfs -# 2. download os rootfs you want by add params $os, $os_arch, $os_version. -# example usage: -# ./download-rootfs debian aarch64 sid - -os=${1:-openeuler} -os_arch=${2:-$(arch)} -os_version=${3:-20.03} - -. ${CCI_SRC}/container/defconfig.sh -. ${CCI_SRC}/lib/log.sh - -load_cci_defaults - -rootfs_dir=/srv/os/${os}/${os_arch}/${os_version} - -download_os_rootfs() -{ - mkdir -p $rootfs_dir - cd $rootfs_dir - - download_url="http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" - wget $download_url - - [ "$?" -ne 0 ] && { - log_error "failed to download os rootfs: $download_url" - exit 1 - } - - gzip -dc ${os_version}.cgz | cpio -id - rm -rf ${os_version}.cgz -} - -download_os_rootfs diff --git a/sparrow/5-build/os-ready b/sparrow/5-build/os-ready index 5e1acc4..d4ed2b8 100755 --- a/sparrow/5-build/os-ready +++ b/sparrow/5-build/os-ready @@ -4,4 +4,4 @@ . $CCI_SRC/lib/check_public_network.sh -public_network_ok && $CCI_SRC/sbin/download-rootfs +public_network_ok && $CCI_SRC/sbin/download-osimage -- 2.23.0

2 2

[PATCH compass-ci] sbin: add config account yaml script
by Cao Xueliang 14 Apr '21

14 Apr '21

delimiter/delimiter.cr needs it to create delimiter account. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- sbin/config_account_yaml.rb | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 sbin/config_account_yaml.rb diff --git a/sbin/config_account_yaml.rb b/sbin/config_account_yaml.rb new file mode 100755 index 0000000..a555276 --- /dev/null +++ b/sbin/config_account_yaml.rb @@ -0,0 +1,8 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative "../lib/config_account" + +config_yaml ARGV[0] -- 2.23.0

1 0

[PATCH compass-ci] container/etcd: use a scheduled task for compacting in etcd
by Cao Xueliang 14 Apr '21

14 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/etcd/Dockerfile | 6 ++++++ container/etcd/entry.sh | 8 ++++++++ container/etcd/etcd_compact.sh | 7 +++++++ container/etcd/start | 2 +- 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100755 container/etcd/entry.sh create mode 100755 container/etcd/etcd_compact.sh diff --git a/container/etcd/Dockerfile b/container/etcd/Dockerfile index b3957e8..f4746ab 100644 --- a/container/etcd/Dockerfile +++ b/container/etcd/Dockerfile @@ -14,3 +14,9 @@ RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweiclou rm -Rf etcd-${VERSION}-linux-arm64* /var/cache/apk/* VOLUME /data + +COPY etcd_compact.sh / +COPY entry.sh / +RUN echo '* 0 * * * /test.sh > /etcd_compact.log 2>&1' >> /var/spool/cron/crontabs/root + +ENTRYPOINT ["/entry.sh"] diff --git a/container/etcd/entry.sh b/container/etcd/entry.sh new file mode 100755 index 0000000..4bfea1e --- /dev/null +++ b/container/etcd/entry.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +crond +/bin/etcd +exec "$@" + diff --git a/container/etcd/etcd_compact.sh b/container/etcd/etcd_compact.sh new file mode 100755 index 0000000..acc20d0 --- /dev/null +++ b/container/etcd/etcd_compact.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +rev=$(etcdctl endpoint status --write-out="json"|egrep -o '"revision":[0-9]*'|egrep -o '[0-9].*') +etcdctl compact $rev && etcdctl defrag + diff --git a/container/etcd/start b/container/etcd/start index db94833..6d32cd2 100755 --- a/container/etcd/start +++ b/container/etcd/start @@ -14,10 +14,10 @@ cmd=( -p 4001:4001 -p 7001:7001 -v /srv/etcd/dir:/data + -v /etc/localtime:/etc/localtime:ro -e ETCD_UNSUPPORTED_ARCH=arm64 --name etcd-server etcd-v3.4.15:alpine - /bin/etcd --name node-1 --data-dir /data --initial-cluster-state new -- 2.23.0

1 0

[PATCH lkp-tests] sbin/submit: set default values
by Liu Yinsi 14 Apr '21

14 Apr '21

before: submit host-info.yaml os_arch=x86_64 arch=x86_64 after : submit host-info.yaml [why] when submit job in x86_64 machine, must execute like this: `submit host-info.yaml os_arch=x86_64 arch=x86_64`, then the job can be consumed by docker testbox, so add default values, it will be more convenient. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/submit | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sbin/submit b/sbin/submit index d2d9bec76..47b377001 100755 --- a/sbin/submit +++ b/sbin/submit @@ -10,7 +10,10 @@ require "#{LKP_SRC}/lib/scheduler_client" require 'optparse' require 'yaml' -opt_set_key_value = {} +opt_set_key_value = { + 'os_arch' => `arch`.chomp, + 'arch' => `arch`.chomp +} opt_cmdline_defaults = {} opt_output_dir = nil opt_auto_define_files = false -- 2.23.0

3 4

[PATCH compass-ci] container/dracut-initrd: set yes for choices when pvcreate and vgcreate
by Xu Xijian 13 Apr '21

13 Apr '21

[why] When create physical volume, some choices need to be made, we should set it as yes(-y) to keep going instead of skipping [errlog] [ 260.644037] dracut-pre-mount[4008]: /bin/dracut-pre-mount@50(sync_src_lv): lvm pvcreate /dev/sda [ 260.664078] dracut-pre-mount[4008]: WARNING: ext4 signature detected on /dev/sda at offset 1080. Wipe it? [y/n]: [n] [ 260.684036] dracut-pre-mount[4008]: Aborted wiping of ext4. [ 260.700034] dracut-pre-mount[4008]: 1 existing signature left on the device. [ 260.716076] dracut-pre-mount[4008]: /bin/dracut-pre-mount@50(sync_src_lv): reboot Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- container/dracut-initrd/bin/set-local-sysroot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container/dracut-initrd/bin/set-local-sysroot.sh b/container/dracut-initrd/bin/set-local-sysroot.sh index 2a07270..9e83742 100644 --- a/container/dracut-initrd/bin/set-local-sysroot.sh +++ b/container/dracut-initrd/bin/set-local-sysroot.sh @@ -29,13 +29,13 @@ sync_src_lv() { # ensure the physical disk has been initialized as physical volume real_pv_device="$(lvm pvs | grep -w $pv_device | awk '{print $1}')" [ "$real_pv_device" = "$pv_device" ] || { - lvm pvcreate "$pv_device" || reboot + lvm pvcreate -y "$pv_device" || reboot } # ensure the volume group $vg_name exists real_vg_name="$(lvm pvs | grep -w $vg_name | awk '{print $2}')" [ "$real_vg_name" = "$vg_name" ] || { - lvm vgcreate "$vg_name" "$pv_device" || reboot + lvm vgcreate -y "$vg_name" "$pv_device" || reboot } } -- 2.23.0

1 0

[PATCH compass-ci] es-jobs: reuse results of other programs
by Wu Zhende 13 Apr '21

13 Apr '21

[Why] es-jobs's result may need use in other programs. But it's hard to get it right now. [How] use: es_jobs.generate_result can get the hash result Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/es_jobs.rb | 6 +++++- sbin/es-jobs | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index b0d0fb2..25e54b4 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -168,6 +168,10 @@ class ESJobs end def output + output_yaml('', @result) + end + + def generate_result if @jobs.empty? puts "No query result is found: #{@es_query}" return @@ -175,6 +179,6 @@ class ESJobs @result = query_jobs_state(@jobs) @result['kvcount'] = @result['kvcount'].sort.to_h @result['raw.id'] = @result['raw.id'].sort.to_h - output_yaml('', @result) + @result end end diff --git a/sbin/es-jobs b/sbin/es-jobs index ff0d540..6d17d6b 100755 --- a/sbin/es-jobs +++ b/sbin/es-jobs @@ -52,4 +52,5 @@ items = parse_argv raise 'Please enter a query' if items.empty? es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter) +es_jobs.generate_result es_jobs.output -- 2.23.0

1 0

[PATCH compass-ci] lib/sched.cr: change testbox's queues
by Wu Zhende 13 Apr '21

13 Apr '21

[Before] ["sched/dc-8g.aarch64/ready"] [After] ["dc-8g.aarch64"] [Why] "sched/.*/ready" is a fixed format. The length of each column on kibana is limited, and removing the fixed format can also highlinght the point. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index f62f83e..35315ce 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -158,6 +158,16 @@ class Sched spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end + # input: ["sched/dc-8g.aarch64/ready"] + # output: ["dc-8g.aarch64"] + def fetch_queues(queues) + new_queues = [] of String + queues.each do |queue| + new_queues << queue.split("/")[1] + end + new_queues + end + def set_lifecycle(job, testbox, queues) if job deadline = job.get_deadline @@ -172,7 +182,7 @@ class Sched end type = get_type(testbox) - queues = JSON.parse(queues.to_json) + queues = JSON.parse(fetch_queues(queues).to_json) hash = { "job_id" => job_id, "state" => state, -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] sparrow: call public_network_ok
by Liu Yinsi 13 Apr '21

13 Apr '21

[why] For user locally deploy Compass CI, if public network not ok, they will install pkg by themselves, in order to fix error message, so call function public_network_ok(). Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/0-package/common | 8 ++++++-- sparrow/0-package/os/openEuler | 14 +++++++++----- sparrow/5-build/os-ready | 4 +++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/sparrow/0-package/common b/sparrow/0-package/common index 9fdb4be..eb86b24 100755 --- a/sparrow/0-package/common +++ b/sparrow/0-package/common @@ -2,8 +2,12 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ -gem install --no-user-install git activesupport rest-client cucumber json faye-websocket elasticsearch bunny +. $CCI_SRC/lib/check_public_network.sh + +public_network_ok && { + gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ + gem install --no-user-install git activesupport rest-client cucumber json faye-websocket elasticsearch bunny +} grep -q '^lkp:' /etc/passwd || useradd -u 1090 lkp grep -q '^mailer:' /etc/passwd || useradd -u 1144 mailer diff --git a/sparrow/0-package/os/openEuler b/sparrow/0-package/os/openEuler index 9cb7b04..e742fa1 100755 --- a/sparrow/0-package/os/openEuler +++ b/sparrow/0-package/os/openEuler @@ -2,6 +2,8 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +. $CCI_SRC/lib/check_public_network.sh + pkgs=( vim git @@ -26,10 +28,12 @@ pkgs=( sudo ) -yum install -y --skip-broken "${pkgs[@]}" +public_network_ok && { + yum install -y --skip-broken "${pkgs[@]}" -wget http://abf-downloads.openmandriva.org/rock/repository/$(arch)/main/release/… -wget http://abf-downloads.openmandriva.org/rock/repository/$(arch)/main/release/… + wget http://abf-downloads.openmandriva.org/rock/repository/$(arch)/main/release/… + wget http://abf-downloads.openmandriva.org/rock/repository/$(arch)/main/release/… -rpm -i --force lib64lockfile1-1.16-1-omv4002.$(arch).rpm lockfile-progs-0.1.18-5-omv4002.$(arch).rpm -rm -rf *.rpm + rpm -i --force lib64lockfile1-1.16-1-omv4002.$(arch).rpm lockfile-progs-0.1.18-5-omv4002.$(arch).rpm + rm -rf *.rpm +} diff --git a/sparrow/5-build/os-ready b/sparrow/5-build/os-ready index 48a740b..5e1acc4 100755 --- a/sparrow/5-build/os-ready +++ b/sparrow/5-build/os-ready @@ -2,4 +2,6 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -$CCI_SRC/sbin/download-rootfs +. $CCI_SRC/lib/check_public_network.sh + +public_network_ok && $CCI_SRC/sbin/download-rootfs -- 2.23.0

2 4

[PATCH compass-ci] lib/git_mirror.rb: retry es query
by Li Yuanchao 13 Apr '21

13 Apr '21

[errmsg] git-mirror: /usr/lib/ruby/2.5.0/socket.rb:452:in `__read_nonblock': Connection reset by peer (Faraday::ConnectionFailed) Sometimes the error above occurs, and container git-mirror restart. It's not a stable error, and next time es query can be success. So retry here. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index bcdb077..6100d6b 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -432,7 +432,14 @@ class MirrorMain query = { query: { match: { _id: git_repo } } } return fork_stat unless @es_client.count(index: 'repo', body: query)['count'].positive? - result = @es_client.search(index: 'repo', body: query)['hits'] + begin + result = @es_client.search(index: 'repo', body: query)['hits'] + rescue StandardError + puts $ERROR_INFO + sleep 1 + retry + end + fork_stat.each_key do |key| fork_stat[key] = result['hits'][0]['_source'][key.to_s] || fork_stat[key] end -- 2.23.0

1 0

[PATCH v2 lkp-tests] tests/rpmbuild-pkg: post data to rpm-repo docker
by Wang Yong 13 Apr '21

13 Apr '21

post rpm data to rpm-repo in order to createrepo ref: commit 5dd22c1e010734bc5e610f4abe46baec0f36f5bb Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/rpmbuild-pkg | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index a91ae4a08..312582d16 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -108,6 +108,32 @@ upload_rpm_pkg() done } +handle_rpm_list() +{ + rpm_list=($(show_rpm_files | xargs basename -a)) + srpm_list=($(show_src_rpm_files | xargs basename -a)) + full_list=(${rpm_list[@]} ${srpm_list[@]}) + j=0 + + for rpm_name in ${full_list[@]} + do + full_list[$j]="\"/srv${rpm_dest}/${rpm_name}\"" + [[ ${rpm_name} =~ "src.rpm" ]] && full_list[$j]="\"/srv${src_rpm_dest}/${rpm_name}\"" + ((j++)) + done + + full_list=$(echo ${full_list[@]} | sed 's; ;,;g') +} + +update_repo_mq() +{ + handle_rpm_list + + curl -H 'Content-Type: Application/json' -XPOST "${LKP_SERVER}:10003/upload" \ + -d "{\"upload_rpms\": ["${full_list}"]}" +} + [ -n "$repo_name" ] && from_srpm || from_git build_rpm upload_rpm_pkg +update_repo_mq -- 2.23.0

1 0

[PATCH compass-ci] container/scheduler: fix Missing ENV key: NODE_PORT
by Cao Xueliang 13 Apr '21

13 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/scheduler/my-start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/scheduler/my-start b/container/scheduler/my-start index 8d800f9..e5855af 100755 --- a/container/scheduler/my-start +++ b/container/scheduler/my-start @@ -65,6 +65,7 @@ cmd = %W[ -e CCI_REPOS=#{CCI_REPOS} -e LKP_SRC=#{DEFAULT_LKP} -e CCI_SRC=#{DEFAULT_CCI} + -e NODE_PORT=#{SCHED_PORT} -v #{ENV['LKP_SRC']}:#{DEFAULT_LKP} -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} -v #{CCI_REPOS}/lab-#{LAB}:#{CCI_REPOS}/lab-#{LAB} -- 2.23.0

1 0

[PATCH compass-ci] scheduler: use NODE_PORT to run kemal
by Cao Xueliang 13 Apr '21

13 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/scheduler/start | 5 +++-- src/scheduler.cr | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/container/scheduler/start b/container/scheduler/start index b13b396..598b263 100755 --- a/container/scheduler/start +++ b/container/scheduler/start @@ -7,6 +7,7 @@ require 'set' require_relative '../defconfig.rb' names = Set.new %w[ + SCHED_HOST SCHED_PORT SCHED_NODES REDIS_HOST @@ -38,7 +39,7 @@ defaults['SCHED_PORT'] ||= '3000' defaults['MASTER_FLUENTD_PORT'] ||= '24224' defaults['SRV_HTTP_CCI_PORT'] ||= '11307' sched_nodes = defaults.delete('SCHED_NODES') -SCHED_PORT = defaults.delete('SCHED_PORT') +SCHED_PORT = defaults['SCHED_PORT'] sched_nodes.each do |node| docker_rm "s001-alpine-#{node['port']}" @@ -69,7 +70,7 @@ sched_nodes.each do |node| -e CCI_REPOS=#{CCI_REPOS} -e LKP_SRC=#{DEFAULT_LKP} -e CCI_SRC=#{DEFAULT_CCI} - -e SCHED_PORT=#{node_port} + -e NODE_PORT=#{node_port} -v /srv/tmp/public:/c/cci/scheduler/public -v #{ENV['LKP_SRC']}:#{DEFAULT_LKP} -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} diff --git a/src/scheduler.cr b/src/scheduler.cr index 07b38a3..4e4bf24 100644 --- a/src/scheduler.cr +++ b/src/scheduler.cr @@ -9,7 +9,7 @@ module Scheduler log = JSONLogger.new begin - Kemal.run(SCHED_PORT) + Kemal.run(ENV["NODE_PORT"].to_i32) rescue e log.error(e) end -- 2.23.0

2 2

[PATCH compass-ci 2/2] src/extract-stats/stats_worker.cr: optimize output
by Lu Weitao 13 Apr '21

13 Apr '21

[Why] standardized output log for kibana Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- src/extract-stats/stats_worker.cr | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index e371bf3..47c3a59 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -2,6 +2,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. require "../lib/etcd_client" +require "../lib/json_logger" require "../scheduler/elasticsearch_client" require "../scheduler/redis_client" require "../scheduler/constants" @@ -16,6 +17,7 @@ class StatsWorker @es = Elasticsearch::Client.new @etcd = EtcdClient.new @rc = RegressionClient.new + @log = JSONLogger.new end def handle(queue_path, channel) @@ -30,7 +32,7 @@ class StatsWorker @etcd.delete(queue_path) rescue e channel.send(queue_path) - STDERR.puts e.message + @log.error(e.message) # incase of many error message when task-queue, ES does not work sleep(10) ensure @@ -95,16 +97,16 @@ class StatsWorker new_error_ids = check_new_error_ids(error_ids, job_id) unless new_error_ids.empty? sample_error_id = new_error_ids.sample - STDOUT.puts "send a delimiter task: job_id is #{job_id}" + @log.info("send a delimiter task: job_id is #{job_id}") queue = "#{DELIMITER_TASK_QUEUE}/#{job_id}" value = {"job_id" => job_id, "error_id" => sample_error_id} @etcd.put(queue, value) msg = %({"job_id": "#{job_id}", "new_error_id": "#{sample_error_id}"}) - system "echo '#{msg}'" + @log.info(msg) end msg = %({"job_id": "#{job_id}", "job_state": "extract_finished"}) - system "echo '#{msg}'" + @log.info(msg) end def check_new_error_ids(error_ids : Array, job_id : String) @@ -113,7 +115,7 @@ class StatsWorker begin is_exists = @rc.check_error_id error_id rescue e - STDERR.puts e.message + @log.error(e.message) next end next if is_exists -- 2.23.0

1 0

[PATCH compass-ci 1/2] lib/dump_stat.rb: optimize useless special chars
by Lu Weitao 13 Apr '21

13 Apr '21

[Why] we already use strip, so needn't chomp Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/dump_stat.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dump_stat.rb b/lib/dump_stat.rb index 383ff09..04a7632 100644 --- a/lib/dump_stat.rb +++ b/lib/dump_stat.rb @@ -38,7 +38,7 @@ module DumpStat stat_result.each do |key, value| key = key.resolve_invalid_bytes - key = key.chomp.strip + key = key.strip next if key[0] == '#' next if value.empty? || value == 0 next if monitor =~ /^(dmesg|kmsg)$/ && key =~ /^(message|pattern):/ -- 2.23.0

1 0

[PATCH lkp-tests] sbin/submit: delete useless require
by Wei Jihui 13 Apr '21

13 Apr '21

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- sbin/submit | 1 - 1 file changed, 1 deletion(-) diff --git a/sbin/submit b/sbin/submit index d2d9bec7..edb207cc 100755 --- a/sbin/submit +++ b/sbin/submit @@ -5,7 +5,6 @@ LKP_SRC = ENV['LKP_SRC'] || File.dirname(File.dirname(File.realpath($PROGRAM_NAME))) require "#{LKP_SRC}/lib/job2sh" -require "#{LKP_SRC}/lib/distro_info" require "#{LKP_SRC}/lib/scheduler_client" require 'optparse' require 'yaml' -- 2.23.0

1 0

[PATCH lkp-tests] tests/rpmbuild-pkg: post data to rpm-repo docker
by Wang Yong 13 Apr '21

13 Apr '21

post rpm data to rpm-repo in order to createrepo ref: commit 5dd22c1e010734bc5e610f4abe46baec0f36f5bb Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/rpmbuild-pkg | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index a91ae4a08..3d63bebae 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -108,6 +108,33 @@ upload_rpm_pkg() done } +update_repo_mq() +{ + rpm_list=($(find ${HOME}/rpmbuild/RPMS -type f -name "*.rpm" -exec basename {} \;)) + srpm_list=($(find ${HOME}/rpmbuild/SRPMS -type f -name "*.rpm" -exec basename {} \;)) + j=0 + k=0 + + for rpm_name in ${rpm_list[@]} + do + rpm_list[$j]="\"/srv${rpm_dest}/${rpm_name}\"" + ((j++)) + done + + for src_rpm_name in ${srpm_list[@]} + do + srpm_list[$k]="\"/srv${src_rpm_dest}/${src_rpm_name}\"" + ((k++)) + done + + full_list=(${rpm_list[@]} ${srpm_list[@]}) + full_list=$(echo ${full_list[@]} | sed 's; ;,;g') + + curl -H 'Content-Type: Application/json' -XPOST "${LKP_SERVER}:10003/upload" \ + -d "{\"upload_rpms\": ["${full_list}"]}" +} + [ -n "$repo_name" ] && from_srpm || from_git build_rpm upload_rpm_pkg +update_repo_mq -- 2.23.0

2 2

[PATCH compass-ci 2/2] sbin/download-rootfs: not to download if machine is virtual
by Liu Yinsi 12 Apr '21

12 Apr '21

[why] we not support to run qemu in a virtual machine, so download os rootfs is unnecessary. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/download-rootfs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sbin/download-rootfs b/sbin/download-rootfs index f8c245d..68a6803 100755 --- a/sbin/download-rootfs +++ b/sbin/download-rootfs @@ -37,4 +37,9 @@ download_os_rootfs() rm -rf ${os_version}.cgz } -download_os_rootfs +is_virtual() +{ + dmidecode -s system-product-name | grep -iq "virtual" +} + +is_virtual || download_os_rootfs -- 2.23.0

1 0

[PATCH compass-ci 1/2] sbin/download-rootfs: download os rootfs according to arch
by Liu Yinsi 12 Apr '21

12 Apr '21

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/download-rootfs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/download-rootfs b/sbin/download-rootfs index 74afbf6..f8c245d 100755 --- a/sbin/download-rootfs +++ b/sbin/download-rootfs @@ -10,7 +10,7 @@ # ./download-rootfs debian aarch64 sid os=${1:-openeuler} -os_arch=${2:-aarch64} +os_arch=${2:-$(arch)} os_version=${3:-20.03} . ${CCI_SRC}/container/defconfig.sh -- 2.23.0

1 0

[PATCH lkp-tests] add script for fix dockerfile file
by Bai Jing 12 Apr '21

12 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- lib/handle_dockerfile.sh | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 lib/handle_dockerfile.sh diff --git a/lib/handle_dockerfile.sh b/lib/handle_dockerfile.sh new file mode 100644 index 000000000..b71f9865d --- /dev/null +++ b/lib/handle_dockerfile.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" +} + +handle_epel() +{ + sed -i "s/epel-release/bash/g" "$dockerfile_name" +} + +handle_epelrpm() +{ + sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" +} + +handle_rpm_gpg() +{ + sed -i "s|RPM-GPG-KEY-CentOS-7|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" + sed -i "s|RPM-GPG-KEY-centosofficial|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" +} + +add_base_commands() +{ + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" +} + +handle_epel_repo() +{ + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" +} + +handle_error_exit() +{ + sed '/^RUN/RUN set -e;/g' "$dockerfile_name" +} + +handle_dockerfile() +{ + dockerfile_name=$1 + handle_FROM + handle_epel + handle_epelrpm + handle_rpm_gpg + add_base_commands + handle_epel_repo + handle_error_exit +} -- 2.23.0

2 3