April 2021 - Compass-ci - mailweb.openeuler.org

[PATCH v2 lkp-tests 2/2] lib/email.sh: add selftest_content()
by Liu Yinsi 22 Apr '21

22 Apr '21

This will send email to user when finish selftest. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- lib/email.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/email.sh b/lib/email.sh index 4f1055033..24ef7bb64 100755 --- a/lib/email.sh +++ b/lib/email.sh @@ -80,6 +80,22 @@ Compass-CI " } +selftest_content() +{ + email_content="To: $my_email +Subject: [NOTIFY Compass-CI] Self-test report + +Dear $my_name: + + Your self-test groud_id=selftest job result is: + + $job_state + +Regards +Compass-CI +" +} + send_email() { local subject=$1 -- 2.23.0

1 0

[PATCH compass-ci] contianer/register-accounts: add env ES_USER ES_PASSWORD
by Li Yuanchao 22 Apr '21

22 Apr '21

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/defconfig.rb | 1 + container/register-accounts/start | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/container/defconfig.rb b/container/defconfig.rb index f090351..0a8e4d5 100755 --- a/container/defconfig.rb +++ b/container/defconfig.rb @@ -11,6 +11,7 @@ def cci_defaults '/etc/compass-ci/service/*.yaml', '/etc/compass-ci/accounts/*.yaml', '/etc/compass-ci/register/*.yaml', + '/etc/compass-ci/*.yaml', "#{ENV['HOME']}/.config/compass-ci/defaults/*.yaml"]).each do |file| hash.update YAML.load_file(file) || {} end diff --git a/container/register-accounts/start b/container/register-accounts/start index c2acc75..ad70048 100755 --- a/container/register-accounts/start +++ b/container/register-accounts/start @@ -9,6 +9,8 @@ require_relative '../defconfig.rb' names = Set.new %w[ ES_HOST ES_PORT + ES_USER + ES_PASSWORD lab ] @@ -25,7 +27,7 @@ cmd = %w[ ] + env + %W[ -e CCI_SRC=#{DEFAULT_CCI} -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} - -v /etc/compass-ci/register:/etc/compass-ci/register:ro + -v /etc/compass-ci:/etc/compass-ci:ro -v /etc/localtime:/etc/localtime:ro -w #{DEFAULT_CCI}/container/register-accounts register-accounts -- 2.23.0

1 0

[PATCH lkp-tests 4/4] distro/depends: depends package file for compile installing mysql
by Luan Shengde 22 Apr '21

22 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- distro/depends/mysql-8-0-20-dev | 257 ++++++++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 distro/depends/mysql-8-0-20-dev diff --git a/distro/depends/mysql-8-0-20-dev b/distro/depends/mysql-8-0-20-dev new file mode 100644 index 000000000..2da6fc930 --- /dev/null +++ b/distro/depends/mysql-8-0-20-dev @@ -0,0 +1,257 @@ +make +bison +ncurses +ncurses-devel +libaio-devel +openssl +openssl-devel +gmp +gmp-devel +mpfr +mpfr-devel +libmpc +libmpc-devel +wget +tar +gcc +gcc-c++ +git +rpcgen +cmake +bison +bison-devel +m4 +libtirpc-devel +openldap-devel +protobuf +protobuf-devel +protobuf-lite-devel +dnf-plugins-core +rpm-build +asciidoc +autoconf +automake +bzip2-devel +curl-devel +desktop-file-utils +emacs +expat-devel +flex +gcc-gfortran +gdb +glib2-devel +gnutls-devel +gpm-devel +libX11-devel +libacl-devel +libdb-devel +libpsl-devel +libtool +lksctp-tools-devel +pcre-devel +perl-generators +policycoreutils +python2-devel +python2-setuptools +python3-devel +unixODBC-devel +util-linux-devel +xmlto +xz-devel +cracklib-devel +gpgme-devel +guile-devel +jsoncpp-devel +libarchive-devel +libidn2-devel +libmetalink-devel +libsecret-devel +libtool-devel +libuv-devel +perl-HTTP-Daemon +perl-MailTools +python2-google-apputils +python3-google-apputils +python3-sphinx +qt5-qtbase-devel +rhash-devel +texinfo +ModemManager-glib +SDL +adobe-mappings-cmap +adobe-mappings-cmap-lang +adobe-mappings-pdf +alsa-lib +avahi-glib +babeltrace +boost-regex +cdparanoia-libs +ctags +dbus-glib +dejavu-fonts +diffutils +docbook-dtds +docbook-style-xsl +emacs-common +enchant +fftw-libs-double +fontconfig-devel +freeglut +freetype-devel +gd +gdb-headless +geoclue2 +ghostscript +giflib +glew +google-droid-sans-fonts +gpm +graphviz +gstreamer1 +gstreamer1-plugins-base +hunspell +hunspell-en +hyphen +iso-codes +jbig2dec +libXau-devel +libXdamage-devel +libXext-devel +libXfixes-devel +libXv +libXxf86misc +libXxf86vm +libXxf86vm-devel +libattr-devel +libdrm +libdrm-devel +libffi-devel +libfontenc +libgfortran +libglvnd +libglvnd-devel +libgpg-error-devel +libidn +libijs +libnotify +libogg +libpaper +libpng-devel +librsvg2 +libtasn1-devel +libtheora +libvisual +libvorbis +libwebp +libxcb-devel +libxshmfence +libxslt +linuxconsoletools +lksctp-tools +mailcap +mcpp +mesa-demos +mesa-libEGL +mesa-libEGL-devel +mesa-libGL +mesa-libGL-devel +mesa-libGLU +mesa-libgbm +mesa-libglapi +nettle-devel +openjpeg2 +opus +orc +p11-kit-devel +perl-Encode-Locale +perl-Fedora-VSP +perl-HTTP-Date +perl-HTTP-Message +perl-IO-HTML +perl-IO-Socket-SSL +perl-LWP-MediaTypes +perl-Net-SSLeay +perl-Text-Unidecode +perl-TimeDate +perl-URI +python2 +python2-debug +python2-help +python2-six +python2-tkinter +python2-tools +python3-babel +python3-chardet +python3-idna +python3-jinja2 +python3-markupsafe +python3-pysocks +python3-pytz +python3-requests +python3-rpm-generators +python3-urllib3 +qt-settings +qt5-qtbase +qt5-qtbase-common +sgml-common +source-highlight +tk +unixODBC +urw-base35-bookman-fonts +urw-base35-c059-fonts +urw-base35-d050000l-fonts +urw-base35-fonts +urw-base35-fonts-common +urw-base35-gothic-fonts +urw-base35-nimbus-mono-ps-fonts +urw-base35-nimbus-roman-fonts +urw-base35-nimbus-sans-fonts +urw-base35-p052-fonts +urw-base35-standard-symbols-ps-fonts +urw-base35-z003-fonts +webkit2gtk3 +webkit2gtk3-jsc +woff2 +xcb-util +xorg-x11-font-utils +xorg-x11-fonts +xorg-x11-proto-devel +xorg-x11-server-utils +OpenEXR-libs +gc-devel +ilmbase +libwmf +perl-Class-Inspector +perl-Net-SMTP-SSL +perl-Unicode-EastAsianWidth +perl-libintl-perl +python-sphinx-locale +python2-dateutil +python2-gflags +python2-pytz +python3-docutils +python3-funcsigs +python3-gflags +python3-imagesize +python3-mock +python3-packaging +python3-pbr +python3-pygments +python3-snowballstemmer +python3-sphinx-theme-alabaster +python3-sphinx_rtd_theme +python3-sphinxcontrib-websupport +python3-sqlalchemy +python3-whoosh +qt5-qtbase-gui +qt5-rpm-macros +xcb-util-image +xcb-util-keysyms +xcb-util-renderutil +xcb-util-wm +fontawesome-fonts +lato-fonts +python2-pip +ImageMagick +perl-File-ShareDir +google-roboto-slab-fonts -- 2.23.0

2 2

[PATCH v2 lkp-tests] jobs: add job yaml file for building/testing mysql
by Luan Shengde 22 Apr '21

22 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/mysql-8-0-20.yaml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 jobs/mysql-8-0-20.yaml diff --git a/jobs/mysql-8-0-20.yaml b/jobs/mysql-8-0-20.yaml new file mode 100644 index 000000000..bf93ecdab --- /dev/null +++ b/jobs/mysql-8-0-20.yaml @@ -0,0 +1,34 @@ +suite: cci-makepkg +category: functional + +testbox: taishan200-2280-2s48p-256g--a18 + +os: openeuler +os_version: 20.03 +os_mount: initramfs + +mysql_user: root +mysql_host: localhost +mysql_port: 3306 +mysql_db: sysbench_test +mysql_password: mysql_password +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +sysbench-mysql: + oltp_test_mode: complex + max_requests: 0 + mysql_table_engine: innodb + oltp_tables_count: 1000 + oltp_table_size: 100000 + nr_threads: 256 + runtime: 60 + rand_type: special + rand_spec_pct: 100 + oltp_read_only: false + report_interval: 1 + events: 20000 -- 2.23.0

1 0

[PATCH lkp-tests] jobs/qcow2rootfs-x86_64.yaml: the yaml for qcow2rootfs
by Wang Chenglong 22 Apr '21

22 Apr '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- jobs/qcow2rootfs-x86_64.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 jobs/qcow2rootfs-x86_64.yaml diff --git a/jobs/qcow2rootfs-x86_64.yaml b/jobs/qcow2rootfs-x86_64.yaml new file mode 100644 index 000000000..1002283a3 --- /dev/null +++ b/jobs/qcow2rootfs-x86_64.yaml @@ -0,0 +1,27 @@ +suite: qcow2rootfs-x86_64 +category: benchmark +qcow2rootfs-x86_64: + + # The related fields about qcow2 image to be used to generate rootfs + # it should be the same as the official provided image name + qcow2_os: + qcow2_arch: + qcow2_version: + + # place the result rootfs related fields + # 1. result rootfs will be placed in the following localtion on the + # remote file server: + # - {remote_file_server}/{rootfs_path}/{iso_os}/{iso_arch}/ + # 2. remote file server protocols current supported: + # - nfs + rootfs_protocol: nfs + rootfs_server: 172.168.131.2 + rootfs_path: os-rw + +# The best host for execution is the x86 OpenEuler system +os: +os_version: +os_arch: +os_mount: +testbox: +queue: -- 2.23.0

1 0

[PATCH lkp-tests] lib/bootstrap.sh: delete history remove code
by Liu Shaofei 22 Apr '21

22 Apr '21

[why] function callend by "is_clearlinux" is not defined, by checking history commit record, the function is removed and the patch codes is removed. [ 63.427966] [ 63.428704] + is_clearlinux [ 63.428705] [ 63.431226] /lkp/lkp/src/lib/bootstrap.sh: line 104: is_clearlinux: command not found [ 63.431228] Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- lib/bootstrap.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 7983f638e..0bd01198e 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -101,13 +101,13 @@ network_ok() [ "$(cat $i/carrier 2>/dev/null)" = '1' ] && test_ping && return 0 done - is_clearlinux && { - net_devices_link up - - # in case of systemd-networkd.service was masked - systemctl unmask systemd-networkd.service - systemctl start systemd-networkd.service - } +# is_clearlinux && { +# net_devices_link up +# +# # in case of systemd-networkd.service was masked +# systemctl unmask systemd-networkd.service +# systemctl start systemd-networkd.service +# } return 1 } -- 2.23.0

1 0

[PATCH compass-ci] sparrow/0-package: add deps iputils
by Liu Yinsi 22 Apr '21

22 Apr '21

[why] ping command is used in $CCI_SRC/lib/check_public_network.sh Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/0-package/os/centos | 1 + sparrow/0-package/os/debian | 1 + sparrow/0-package/os/openEuler | 1 + 3 files changed, 3 insertions(+) diff --git a/sparrow/0-package/os/centos b/sparrow/0-package/os/centos index b8ef7694..d4d10457 100755 --- a/sparrow/0-package/os/centos +++ b/sparrow/0-package/os/centos @@ -25,6 +25,7 @@ pkgs=( wget curl sudo + iputils ) yum install -y --skip-broken "${pkgs[@]}" diff --git a/sparrow/0-package/os/debian b/sparrow/0-package/os/debian index 947cbdc1..6feb9be2 100755 --- a/sparrow/0-package/os/debian +++ b/sparrow/0-package/os/debian @@ -25,6 +25,7 @@ pkgs=( qemu-system-x86 qemu-system-riscv64 sudo + iputils ) apt-get update diff --git a/sparrow/0-package/os/openEuler b/sparrow/0-package/os/openEuler index e742fa13..8cdadd52 100755 --- a/sparrow/0-package/os/openEuler +++ b/sparrow/0-package/os/openEuler @@ -26,6 +26,7 @@ pkgs=( libvirt curl sudo + iputils ) public_network_ok && { -- 2.23.0

1 0

[PATCH lkp-tests] tests/qcow2rootfs-x86_64: automatically build x86_64 rootfs
by Wang Chenglong 22 Apr '21

22 Apr '21

[Why] automatically build rootfs for x86 from the official QCOW2 image. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/qcow2rootfs-x86_64 | 80 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 tests/qcow2rootfs-x86_64 diff --git a/tests/qcow2rootfs-x86_64 b/tests/qcow2rootfs-x86_64 new file mode 100755 index 000000000..f698fb7ec --- /dev/null +++ b/tests/qcow2rootfs-x86_64 @@ -0,0 +1,80 @@ +#!/bin/bash +# - qcow2_os +# - qcow2_arch +# - qcow2_version +# - rootfs_protocol +# - rootfs_server +# - rootfs_path + +set_qcow2rootfs_tools() +{ + export QCOW2_PATH="/c/compass-ci/container/qcow2rootfs" + git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci || return 1 + rm ${QCOW2_PATH}/root/etc/yum.repos.d/* + wget -O ${QCOW2_PATH}/root/etc/yum.repos.d/CentOS-Base.repo https://repo.huaweicloud.com/repository/conf/CentOS-7-reg.repo + ${QCOW2_PATH}/build + sed -i 's#$EXTRACT_ROOT/bin/extract $QCOW2_FILE $tar_out_file#$EXTRACT_ROOT/bin/extract $EXTRACT_ROOT/qcow2-dir/$QCOW2_NAME $tar_out_file#' \ + ${QCOW2_PATH}/bin/mail + +} + +check_url() +{ + export QCOW2_NAME="${qcow2_os}-${qcow2_version}.${qcow2_arch}.qcow2.xz" + + # Since the 21.03 version of OpenEuler's mirror naming format has changed, + # it has been modified. + if [ "${qcow2_version}" == "21.03" ];then + export QCOW2_NAME="${qcow2_os}-${qcow2_version}-${qcow2_arch}.qcow2.xz" + fi + + export QCOW2_URL="https://repo.openeuler.org/${qcow2_os}-${qcow2_version}/virtual_machine_img…" +} + +download_qcow2_img() +{ + export IMG_PATH="/root/qcow2" + mkdir -p ${IMG_PATH} && cd ${IMG_PATH} + [ -f "${IMG_PATH}/${QCOW2_NAME}" ] || { + wget ${QCOW2_URL} || exit 1 + } +} + +mount_os-rw() +{ + export MOUNT_POINT="${IMG_PATH}/mnt" + mkdir ${MOUNT_POINT} + mount -t ${rootfs_protocol} -o rw ${rootfs_server}:/${rootfs_path} ${MOUNT_POINT} +} + +extract_rootfs() +{ + export ROOTFS_DIR="${qcow2_version}-$(date +%Y-%m-%d-%H-%M-%S)" + export ROOTFS_PATH="${MOUNT_POINT}/openeuler/${qcow2_arch}/automatic-build/${ROOTFS_DIR}" + mkdir -p $ROOTFS_PATH + ${QCOW2_PATH}/run ${IMG_PATH}/${QCOW2_NAME} ${ROOTFS_PATH} +} + +set_resolv_config() +{ + cat >${ROOTFS_PATH}/etc/resolv.conf<<-EOF + nameserver 114.114.114.114 + EOF +} + +set_link() +{ + cd ${MOUNT_POINT}/openeuler/${qcow2_arch} + ln -fs automatic-build/${ROOTFS_DIR} ${qcow2_version}-automatic + + umount ${MOUNT_POINT} + echo $ROOTFS_PATH +} + +set_qcow2rootfs_tools +check_url +download_qcow2_img +mount_os-rw +extract_rootfs +set_resolv_config +set_link -- 2.23.0

1 0

[PATCH compass-ci] container/mail-robot: optimize method
by Luan Shengde 22 Apr '21

22 Apr '21

use File.exist? instead of File.exists? Lint/DeprecatedClassMethods: File.exists? is deprecated in favor of File.exist?. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/apply-account.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/mail-robot/lib/apply-account.rb b/container/mail-robot/lib/apply-account.rb index d8cb4da4..f64a3de9 100755 --- a/container/mail-robot/lib/apply-account.rb +++ b/container/mail-robot/lib/apply-account.rb @@ -56,7 +56,7 @@ class ApplyAccount @es_port = ES_PORT # email info file for account issuers. - @account_issuer = File.exists?(ENV['ACCOUNT_ISSUER']) ? YAML.load_file(ENV['ACCOUNT_ISSUER']) : {} + @account_issuer = File.exist?(ENV['ACCOUNT_ISSUER']) ? YAML.load_file(ENV['ACCOUNT_ISSUER']) : {} @my_info = {} end -- 2.23.0

1 0

[PATCH compass-ci] scheduler/elasticsearch_client.cr: fix es bug
by Wu Zhende 22 Apr '21

22 Apr '21

<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN"> <html><head> <title>404 Not Found</title> </head><body> <h1>Not Found</h1> <p>The requested URL was not found on this server.</p> <hr> <address>Apache Server at http Port 80</address> </body></html> Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/scheduler/elasticsearch_client.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scheduler/elasticsearch_client.cr b/src/scheduler/elasticsearch_client.cr index 1ef79a6..0a7bc44 100644 --- a/src/scheduler/elasticsearch_client.cr +++ b/src/scheduler/elasticsearch_client.cr @@ -35,7 +35,7 @@ class Elasticsearch::Client if auth user = ENV["ES_USER"]? password = ENV["ES_PASSWORD"]? - host = "http://#{user}:#{URI.encode_www_form(password)}@#{host}" if user && password + host = "#{user}:#{URI.encode_www_form(password)}@#{host}" if user && password end @client = Elasticsearch::API::Client.new({:host => host, :port => port}) end -- 2.23.0

1 0

[PATCH lkp-tests] add job yaml file for building/testing mysql
by Luan Shengde 22 Apr '21

22 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- jobs/mysql-8-0-20.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 jobs/mysql-8-0-20.yaml diff --git a/jobs/mysql-8-0-20.yaml b/jobs/mysql-8-0-20.yaml new file mode 100644 index 000000000..4b0e97f6e --- /dev/null +++ b/jobs/mysql-8-0-20.yaml @@ -0,0 +1,36 @@ +suite: cci-makepkg +category: functional + +testbox: taishan200-2280-2s48p-256g--a18 + +os: openeuler +os_version: 20.03 +os_mount: initramfs + +mysql_user: root +mysql_host: localhost +mysql_port: 3306 +mysql_db: sysbench_test +mysql_password: mysql_password +data_dev: sdb + +cci-makepkg: + benchmark: mysql-8-0-20 + +mysql-8-0-20: + +sysbench-mysql: + oltp_test_mode: complex + max_requests: 0 + mysql_table_engine: innodb + oltp_tables_count: 1000 + oltp_table_size: 100000 + nr_threads: 256 + runtime: 60 + rand_type: special + rand_spec_pct: 100 + oltp_read_only: false + report_interval: 1 + events: 20000 + +sleep: 72000 -- 2.23.0

1 0

[PATCH lkp-tests 3/4] pkg/mysql-8-0-20: compile install mysql-8.0.20
by Luan Shengde 22 Apr '21

22 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- pkg/mysql-8-0-20/PKGBUILD | 77 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 pkg/mysql-8-0-20/PKGBUILD diff --git a/pkg/mysql-8-0-20/PKGBUILD b/pkg/mysql-8-0-20/PKGBUILD new file mode 100644 index 000000000..d6b06f6f8 --- /dev/null +++ b/pkg/mysql-8-0-20/PKGBUILD @@ -0,0 +1,77 @@ +pkgname=mysql-server-mysql +pkgver=8.0.20 +gccdir=gcc +pkgrel=1 +arch=('aarch64' 'x86_64') +url="http://www.mysql.com" +license=('GPLv2') +source=("https://gitee.com/li-yancheng/my-sql_with_boostkit_patch.git" + "https://gitee.com/li-yancheng/gcc.git" + "my.cnf") +md5sums=('SKIP' 'SKIP' 'd90ba2c701507eeac43b06fcfa49c2c1') + +build() +{ + # build_gcc + build_mysql +} + +build_gcc() +{ + mkdir -p $HOME/rpmbuild/SOURCES/ + cd $srcdir/gcc && cp -r * $HOME/rpmbuild/SOURCES/ + cd $HOME/rpmbuild/SOURCES/ && { + rpmbuild -bp gcc.spec + } + cd $HOME/rpmbuild/BUILD/gcc-9.3.0 && { + ./configure --prefix=/usr/ \ + --enable-shared \ + --enable-threads=posix \ + --enable-checking=release \ + --with-system-zlib \ + --enable-__cxa_atexit \ + --disable-libunwind-exceptions \ + --enable-gnu-unique-object \ + --enable-linker-build-id \ + --with-linker-hash-style=gnu \ + --enable-languages=c,c++,objc,obj-c++,fortran,lto \ + --enable-plugin \ + --enable-initfini-array \ + --disable-libgcj \ + --without-isl \ + --without-cloog \ + --enable-gnu-indirect-function \ + --with-stage1-ldflags='-Wl,-z,relro,-z,now' \ + --with-boot-ldflags='-Wl,-z,relro,-z,now' \ + --with-multilib-list=lp64 + make -j96 + make install + } +} + +build_mysql() +{ + unset CPPFLAGS + unset CFLAGS + unset CXXFLAGS + cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver + + CMAKE=cmake + grep -sqF "CentOS Linux release 7" /etc/centos-release && CMAKE=cmake3 + + mkdir -p build + cd build && { + $CMAKE .. -DBUILD_CONFIG=mysql_release -DCMAKE_INSTALL_PREFIX=/usr/local/mysql -DMYSQL_DATADIR=/data/mysql/data -DSYSCONFDIR=/etc -DWITH_BOOST=../boost/boost_1_70_0/ + make -j96 + } +} + +package() +{ + mkdir -p $pkgdir/etc + cp my.cnf $pkgdir/etc/ + + cd $srcdir/my-sql_with_boostkit_patch/$pkgname-$pkgver/build + # make DESTDIR=$pkgdir install + make install +} -- 2.23.0

2 2

[PATCH compass-ci] sbin/auto_submit: fix some errors
by Li Yuanchao 21 Apr '21

21 Apr '21

[errmsg] ./auto_submit:66:in `get_argvs_for_competition': wrong number of arguments (given 3, expected 4) (ArgumentError) ./auto_submit:74:in `get_argvs_for_competition': undefined method `chomp' for nil:NilClass (NoMethodError) Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sbin/auto_submit b/sbin/auto_submit index 28e3715..dcf5b80 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -55,7 +55,7 @@ class AutoSubmit tag = %x(git -C #{mirror_dir} tag --points-at #{commit_id}) real_argvs.push("upstream_tag=#{tag}") unless tag.empty? if git_repo.include?('my-sql_with_boostkit_patch') - competi_argvs = get_argvs_for_competition(mirror_dir, branch, commit_id) + competi_argvs = get_argvs_for_competition(real_argvs, mirror_dir, branch, commit_id) real_argvs.push(competi_argvs) end real_argvs.push("upstream_commit=#{commit_id}") @@ -70,8 +70,10 @@ class AutoSubmit if committer_email.include?('noreply') emails = %x(git -C #{mirror_dir} log --format=%ae -2 #{commit_id}) - committer_email = emails.lines[0].chomp - author_email = emails.lines[1].chomp + if emails.lines.size == 2 + committer_email = emails.lines[0].chomp + author_email = emails.lines[1].chomp + end end base_commit = %x(git -C #{mirror_dir} merge-base master #{branch}).chomp -- 2.23.0

1 0

[PATCH compass-ci] container/auto-submit: add es authentication
by Li Yuanchao 21 Apr '21

21 Apr '21

For config_account Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/auto-submit/start | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/container/auto-submit/start b/container/auto-submit/start index 79463f6..ad4ba9d 100755 --- a/container/auto-submit/start +++ b/container/auto-submit/start @@ -3,16 +3,26 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig.rb' +names = Set.new %w[ + ES_USER + ES_PASSWORD +] + +service_authentication = relevant_service_authentication(names) +env = docker_env(service_authentication) + DOCKER_CCI = '/c/compass-ci' DOCKER_LKP = '/c/lkp-tests' docker_rm "auto-submit" -cmd = %W[ +cmd = %w[ docker run --restart=always --name auto-submit +] + env + %W[ -u 1090:1090 -d -e CCI_SRC=#{DOCKER_CCI} -- 2.23.0

1 0

[PATCH compass-ci] sbin/download-rootfs: download osimage
by Liu Yinsi 21 Apr '21

21 Apr '21

the default os_mount=initramfs, so download osimage to run qemu kvm. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sbin/download-rootfs | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/sbin/download-rootfs b/sbin/download-rootfs index 68a6803..d936dbe 100755 --- a/sbin/download-rootfs +++ b/sbin/download-rootfs @@ -19,22 +19,35 @@ os_version=${3:-20.03} load_cci_defaults rootfs_dir=/srv/os/${os}/${os_arch}/${os_version} +osimage_dir=/srv/initrd/osimage/${os}/${os_arch}/${os_version} -download_os_rootfs() -{ - mkdir -p $rootfs_dir - cd $rootfs_dir +mkdir -p $rootfs_dir $osimage_dir - download_url="http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" - wget $download_url +download_one() +{ + local download_url=$1 + wget $download_url + + [ "$?" -ne 0 ] && { + log_error "failed to download os rootfs: $download_url" + exit 1 + } +} - [ "$?" -ne 0 ] && { - log_error "failed to download os rootfs: $download_url" - exit 1 - } +download_os_rootfs() +{ + cd $rootfs_dir + download_one "http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" + + gzip -dc ${os_version}.cgz | cpio -id + rm -rf ${os_version}.cgz +} - gzip -dc ${os_version}.cgz | cpio -id - rm -rf ${os_version}.cgz +download_osimage() +{ + cd $osimage_dir + download_one "http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/initrd/osimage/…" + download_one "http://api.compass-ci.openeuler.org:${SRV_HTTP_INITRD_PORT}/initrd/deps/nfs…" } is_virtual() @@ -43,3 +56,4 @@ is_virtual() } is_virtual || download_os_rootfs +is_virtual || download_osimage -- 2.23.0

1 0

[PATCH v2 compass-ci] qcow2rootfs: fix guestfish connection .xz file error
by Wang Chenglong 21 Apr '21

21 Apr '21

[Error] spawn /usr/bin/guestfish -i --ro -a /root/extract/qcow2-dir/openEuler-20.09.x86_64.qcow2.xz guestfish: no operating system was found on this disk after: spawn /usr/bin/guestfish -i --ro -a /root/extract/qcow2-dir/openEuler-20.09.x86_64.qcow2 Welcome to guestfish, the guest filesystem shell for editing virtual machine filesystems and disk images. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/qcow2rootfs/bin/main | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/qcow2rootfs/bin/main b/container/qcow2rootfs/bin/main index c7527b79..2d71c484 100755 --- a/container/qcow2rootfs/bin/main +++ b/container/qcow2rootfs/bin/main @@ -39,7 +39,7 @@ extract_rootfs() { echo "[$(date +'%D %T')] Converting .qcow2 to .tar.gz, please wait several minutes..." su - "$EXTRACT_USER" -c " export LIBGUESTFS_BACKEND=direct - $EXTRACT_ROOT/bin/extract $QCOW2_FILE $tar_out_file + $EXTRACT_ROOT/bin/extract $EXTRACT_ROOT/qcow2-dir/$QCOW2_NAME $tar_out_file " echo "[$(date +'%D %T')] Depressing rootfs to $ROOTFS_DIR" -- 2.23.0

1 0

[PATCH compass-ci] qcow2rootfs: fix guestfish connection .xz file error
by Wang Chenglong 21 Apr '21

21 Apr '21

[Error] spawn /usr/bin/guestfish -i --ro -a /root/extract/qcow2-dir/openEuler-20.09.x86_64.qcow2.xz guestfish: no operating system was found on this disk after: spawn /usr/bin/guestfish -i --ro -a /root/extract/qcow2-dir/openEuler-20.09.x86_64.qcow2 Welcome to guestfish, the guest filesystem shell for editing virtual machine filesystems and disk images. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/qcow2rootfs/bin/main | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/qcow2rootfs/bin/main b/container/qcow2rootfs/bin/main index c7527b79..3bee1bf2 100755 --- a/container/qcow2rootfs/bin/main +++ b/container/qcow2rootfs/bin/main @@ -39,7 +39,7 @@ extract_rootfs() { echo "[$(date +'%D %T')] Converting .qcow2 to .tar.gz, please wait several minutes..." su - "$EXTRACT_USER" -c " export LIBGUESTFS_BACKEND=direct - $EXTRACT_ROOT/bin/extract $QCOW2_FILE $tar_out_file + $EXTRACT_ROOT/bin/extract /root/extract/qcow2-dir/$QCOW2_NAME $tar_out_file " echo "[$(date +'%D %T')] Depressing rootfs to $ROOTFS_DIR" -- 2.23.0

2 2

[PATCH compass-ci 1/3] service/logging-es: configuring the service account and password
by Wu Zhende 21 Apr '21

21 Apr '21

Improve security Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/defconfig.rb | 8 ++++++++ container/logging-es/Dockerfile | 7 +++++++ container/logging-es/build | 9 ++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/container/defconfig.rb b/container/defconfig.rb index b121de8..ad94940 100755 --- a/container/defconfig.rb +++ b/container/defconfig.rb @@ -21,6 +21,14 @@ def relevant_defaults(names) cci_defaults.select { |k, _| names.include? k } end +def relevant_service_account(names) + hash = {} + Dir.glob(['/etc/compass-ci/passwd.yaml']).each do |file| + hash.update YAML.load_file(file) || {} + end + hash.select { |k, _| names.include? k } +end + def set_local_env hash = cci_defaults hash.map { |k, v| system "export #{k}=#{v}" } diff --git a/container/logging-es/Dockerfile b/container/logging-es/Dockerfile index a0acf94..f66dc35 100644 --- a/container/logging-es/Dockerfile +++ b/container/logging-es/Dockerfile @@ -5,6 +5,8 @@ ARG BASE_IMAGE FROM $BASE_IMAGE ARG MEMORY +ARG USER +ARG PASSWORD # docker image borrowed from hub.docker.com/r/gagara/elasticsearch-oss-arm64 @@ -16,6 +18,9 @@ RUN sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/elastic sed -i '$a cluster.initial_master_nodes: ["node-1"]' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a indices.memory.index_buffer_size: 20%' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a thread_pool.write.queue_size: 2000' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.security.enabled: true' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.license.self_generated.type: basic' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.security.transport.ssl.enabled: true' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i "s/-Xms1g/-Xms${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options && \ sed -i "s/-Xmx1g/-Xmx${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options @@ -24,6 +29,8 @@ RUN mkdir /usr/share/elasticsearch/tmp && \ WORKDIR /usr/share/elasticsearch +RUN ./bin/elasticsearch-users useradd ${USER} -p ${PASSWORD} -r superuser + ENV PATH /usr/share/elasticsearch/bin:$PATH ENV ES_TMPDIR /usr/share/elasticsearch/tmp diff --git a/container/logging-es/build b/container/logging-es/build index b50830e..ff1f102 100755 --- a/container/logging-es/build +++ b/container/logging-es/build @@ -3,8 +3,15 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig.rb' +names = Set.new %w[ + LOGGING_ES_USER + LOGGING_ES_PASSWORD +] + +defaults = relevant_service_account(names) docker_skip_rebuild "logging-es:7.11.1" BASE_IMAGE_DICT = { @@ -16,4 +23,4 @@ BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] available_memory = get_available_memory -system "docker build -t logging-es:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg MEMORY=#{available_memory} ." +system "docker build -t logging-es:7.11.1 --progress=plain --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg MEMORY=#{available_memory} --build-arg USER=#{defaults['LOGGING_ES_USER']} --build-arg PASSWORD=#{defaults['LOGGING_ES_PASSWORD']} ." -- 2.23.0

2 1

[PATCH compass-ci] assist-result-client: fix git bisect email subject is the "["
by Cao Xueliang 21 Apr '21

21 Apr '21

Need use json parse the data, before return the response. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/assist_result_client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/assist_result_client.rb b/lib/assist_result_client.rb index 592c841..2d2e60d 100644 --- a/lib/assist_result_client.rb +++ b/lib/assist_result_client.rb @@ -60,6 +60,6 @@ class AssistResult response = resource.post(Base64.encode64(data.to_json)) return nil unless response.code == 200 - return response.body + return JSON.parse(response.body) end end -- 2.23.0

1 0

[PATCH compass-ci] sparrow/dev-env: add the host/port to the job by scheduler
by Cao Xueliang 21 Apr '21

21 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- sparrow/3-code/dev-env | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index e02de35..a478fff 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -163,6 +163,11 @@ RESULT_WEBDAV_HOST: RESULT_WEBDAV_PORT: DOCKER_REGISTRY_HOST: DOCKER_REGISTRY_PORT: +GIT_MIRROR_HOST: +JUMPER_HOST: +JUMPER_PORT: +MONITOR_HOST: +MONITOR_PORT: EOF cat > /etc/profile.d/compass.sh <<'EOF' -- 2.23.0

1 0

[PATCH lkp-tests 2/4] add config file for mysql-8.0.20 sysbench test
by Luan Shengde 21 Apr '21

21 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- pkg/mysql-8-0-20/my.cnf | 67 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 pkg/mysql-8-0-20/my.cnf diff --git a/pkg/mysql-8-0-20/my.cnf b/pkg/mysql-8-0-20/my.cnf new file mode 100644 index 000000000..c657234df --- /dev/null +++ b/pkg/mysql-8-0-20/my.cnf @@ -0,0 +1,67 @@ +[mysqld_safe] +log-error=/data/mysql/log/mysql.log +pid-file=/data/mysql/run/mysqld.pid +[mysqldump] +quick +[mysql] +[client] +default-character-set=utf8mb4 +socket=/data/mysql/run/mysql.sock + +[mysqld] +socket=/data/mysql/run/mysql.sock +tmpdir=/data/mysql/tmp +datadir=/data/mysql/data +default_authentication_plugin=mysql_native_password +user=mysql +memlock=on +bind-address=0.0.0.0 +skip-name-resolve +skip-grant-tables + +lc-messages=en_US +lc-messages-dir=/usr/local/mysql/share + +innodb_page_size=64k +max_connections=4000 +back_log=2048 +performance_schema=OFF +max_prepared_stmt_count=128000 + +#file +innodb_file_per_table +innodb_log_file_size=2048M +innodb_log_files_in_group=32 +innodb_open_files=10000 +table_open_cache_instances=64 + +#buffers +innodb_buffer_pool_size=128G +innodb_buffer_pool_instances=64 +innodb_log_buffer_size=2048M + +#tune +sync_binlog=0 +innodb_flush_log_at_trx_commit=0 +innodb_use_native_aio=1 +innodb_spin_wait_delay=160 +innodb_sync_spin_loops=20 +innodb_flush_method=O_DIRECT +innodb_io_capacity=30000 +innodb_io_capacity_max=40000 +innodb_lru_scan_depth=9000 +innodb_page_cleaners=1 +innodb_spin_wait_pause_multiplier=25 + +#perf special +innodb_flush_neighbors=0 +innodb_write_io_threads=1 +innodb_read_io_threads=1 +innodb_purge_threads=1 + +sql_mode=STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO,STRICT_ALL_TABLES + +#skop_log_bin +ssl=0 + +table_open_cache=15000 -- 2.23.0

1 0

[PATCH lkp-tests 1/4] tests: script to start mysql
by Luan Shengde 21 Apr '21

21 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/mysql-8-0-20 | 73 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 tests/mysql-8-0-20 diff --git a/tests/mysql-8-0-20 b/tests/mysql-8-0-20 new file mode 100755 index 000000000..9b00766f0 --- /dev/null +++ b/tests/mysql-8-0-20 @@ -0,0 +1,73 @@ +#!/bin/sh +# - data_dev +# mysql_user +# mysql_host +# mysql_port +# mysql_db +# mysql_password +. $LKP_SRC/lib/common.sh + +: "${mysql_user:=root}" +: "${mysql_host:=localhost}" +: "${mysql_port:=3306}" +: "${mysql_db:=sysbench_test}" +: "${mysql_password:=mysql_password}" + +create_basedir() +{ + mkdir -p /data + mkfs.xfs -f /dev/"$data_dev" + mount /dev/"$data_dev" /data + + mkdir -p /data/mysql/{data,share,tmp,run,log} + + useradd mysql + touch /data/mysql/log/mysql.log + chown -R mysql:mysql /data + chown -R mysql:mysql /data/mysql/log/mysql.log +} + +configure_mysql() +{ + if [ ! -f $mysqld ] || [ ! -f $mysql ] || [ ! -f $mysql_server ]; then + echo "mysql is not installed!" && exit 1 + fi + + "$mysqld" --user=mysql --basedir=/data/mysql --datadir=/data/mysql/data/ --initialize --port="$mysql_port" + sleep 10 + + "$mysql_server" start + sleep 10 + + "$mysql" -u "$mysql_user" -e "flush privileges; use mysql; alter user 'root'@'localhost' identified by '$mysql_password'; flush privileges;" + sed -i '/^skip-grant-tables/d' /etc/my.cnf + "$mysql_server" restart + sleep 10 +} + +create_database() +{ + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "use mysql; update user set host ='%' where user='root'; flush privileges;" + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $mysql_db;" + "$mysql_server" stop + sleep 10 +} + +start_mysql() +{ + "$mysqld" --defaults-file=/etc/my.cnf --port="$mysql_port" + sleep 20 +} + +main() +{ + local mysql="/usr/local/mysql/bin/mysql" + local mysqld="/usr/local/mysql/bin/mysqld" + local mysql_server="/usr/local/mysql/support-files/mysql.server" + + create_basedir + configure_mysql + create_database + start_mysql +} +main -- 2.23.0

1 0

[PATCH compass-ci] lib/dump_stat.rb: optimize warning info
by Lu Weitao 21 Apr '21

21 Apr '21

[Why] when a key - value is invalid, should show the key and value Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/dump_stat.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/dump_stat.rb b/lib/dump_stat.rb index a497d461..1b56cc35 100644 --- a/lib/dump_stat.rb +++ b/lib/dump_stat.rb @@ -56,7 +56,7 @@ module DumpStat if value.is_a?(String) value = check_string_value(k, value, @monitor) next unless value - return nil unless number?(value, @invalid_records, @record_index, @monitor) + return nil unless number?(k, value, @invalid_records, @record_index, @monitor) value = value.index('.') ? value.to_f : value.to_i elsif value.is_a?(Array) @@ -65,7 +65,7 @@ module DumpStat value[i] = check_string_value(k, value[i], @monitor) next unless value[i] - return nil unless number?(value[i], @invalid_records, @record_index, @monitor) + return nil unless number?(k, value[i], @invalid_records, @record_index, @monitor) value[i] = value[i].index('.') ? value[i].to_f : value[i].to_i valid_stats_verification(k, value[i]) @@ -164,10 +164,10 @@ def check_string_value(key, value, monitor) end # only number is valid -def number?(value, invalid_records, record_index, monitor) +def number?(key, value, invalid_records, record_index, monitor) unless value.numeric? invalid_records.push record_index - warn_stat "invalid stats value: #{value}", monitor + warn_stat "invalid stats key-value: \n key: #{key} \n value: #{value}", monitor return nil end -- 2.23.0

1 0

[PATCH compass-ci 5/5] es_query: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

Everying where "ESQuery.new" is used needs to be adjusted Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/assign-account/start | 4 ++++ container/assist-result/start | 6 ++++++ container/delimiter/start | 6 ++++++ container/mail-robot/start | 5 +++++ lib/commit_status.rb | 2 +- lib/compare.rb | 4 ++-- lib/es_jobs.rb | 2 +- lib/es_query.rb | 6 ++---- sbin/es-find | 2 +- src/lib/web_backend.rb | 2 +- 10 files changed, 29 insertions(+), 10 deletions(-) diff --git a/container/assign-account/start b/container/assign-account/start index b926853..c8c55da 100755 --- a/container/assign-account/start +++ b/container/assign-account/start @@ -4,6 +4,8 @@ . ../defconfig.sh +load_service_authentication + docker_rm assign-account cmd=( @@ -11,6 +13,8 @@ cmd=( --restart=always --name=assign-account -d + -e ES_USER=$ES_USER + -e ES_PASSWORD=$ES_PASSWORD -v /etc:/etc:rw -v /srv/account-info:/opt/account_data/:rw -v /home:/home:rw diff --git a/container/assist-result/start b/container/assist-result/start index af812aa..9998588 100755 --- a/container/assist-result/start +++ b/container/assist-result/start @@ -9,11 +9,17 @@ require_relative '../defconfig.rb' names = Set.new %w[ ES_HOST ES_PORT + ES_USER + ES_PASSWORD ASSIST_RESULT_HOST ASSIST_RESULT_PORT ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' diff --git a/container/delimiter/start b/container/delimiter/start index 9485750..53adc03 100755 --- a/container/delimiter/start +++ b/container/delimiter/start @@ -9,9 +9,15 @@ require_relative '../defconfig.rb' names = Set.new %w[ TASKQUEUE_HOST TASKQUEUE_PORT + ES_USER + ES_PASSWORD ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' diff --git a/container/mail-robot/start b/container/mail-robot/start index ce18498..07aea39 100755 --- a/container/mail-robot/start +++ b/container/mail-robot/start @@ -15,6 +15,8 @@ names = Set.new %w[ SCHED_HTTPS_HOST SCHED_HTTPS_PORT ES_PORT + ES_USER + ES_PASSWORD MAILDIR ACCOUNT_ISSUER lab @@ -22,6 +24,9 @@ names = Set.new %w[ defaults = relevant_defaults(names) +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) cmd = %w[ diff --git a/lib/commit_status.rb b/lib/commit_status.rb index 81aeb29..baae530 100644 --- a/lib/commit_status.rb +++ b/lib/commit_status.rb @@ -26,7 +26,7 @@ def query_latest_good_commit(job, error_id) end def query_jobs_from_es(items) - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new result = es.multi_field_query items jobs_list = result['hits']['hits'] jobs_list.map! { |job| job['_source'] } diff --git a/lib/compare.rb b/lib/compare.rb index 7c182c4..7386cae 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -38,7 +38,7 @@ end def create_matrices_list(conditions) matrices_list = [] suite_list = [] - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new conditions.each do |condition| query_results = es.multi_field_query(condition, desc_keyword: 'start_time') matrix, suites = combine_query_data(query_results) @@ -71,7 +71,7 @@ def compare_group(argv, dimensions, options) end def create_groups_matrices_list(conditions, dims) - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new query_results = es.multi_field_query(conditions, desc_keyword: 'start_time') combine_group_query_data(query_results['hits']['hits'], dims) end diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index 25e54b4..3c29b66 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -23,7 +23,7 @@ class ESJobs end def query_jobs_from_es - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new result = es.multi_field_query @es_query jobs = result['hits']['hits'] jobs.map! { |job| job['_source'] } diff --git a/lib/es_query.rb b/lib/es_query.rb index 22d68a9..dd6d97d 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -7,12 +7,10 @@ require_relative 'constants.rb' # build multiple query request body class ESQuery - HOST = (ENV.key?('ES_HOST') ? ENV['ES_HOST'] : ES_HOST) - PORT = (ENV.key?('ES_PORT') ? ENV['ES_PORT'] : ES_PORT).to_i - def initialize(host = HOST, port = PORT, index: 'jobs') + def initialize(hosts = ES_HOSTS, index: 'jobs') @index = index @scroll_id = '' - @client = Elasticsearch::Client.new url: "http://#{host}:#{port}" + @client = Elasticsearch::Client.new hosts: hosts raise 'Connect Elasticsearch error!' unless @client.ping end diff --git a/sbin/es-find b/sbin/es-find index f548fc1..78e30a8 100755 --- a/sbin/es-find +++ b/sbin/es-find @@ -77,7 +77,7 @@ options = { nr_jobs: 10, desc_keyword: 'start_time' } range = nil count = nil -es = ESQuery.new(ES_HOST, ES_PORT) +es = ESQuery.new(ES_HOSTS) opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: es-find [options] search_key1=val1[,val2..] ..' diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index c4d4edd..91287b4 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -657,7 +657,7 @@ def get_error_objects(filter_items) end def get_job_list(items) - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new query_results = es.multi_field_query(items) extract_jobs_list(query_results['hits']['hits']) -- 2.23.0

1 0

[PATCH compass-ci 4/5] ruby-services: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

Two ruby services: web-backend git_mirror Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/git-mirror/start | 12 +++++++++++- container/web-backend/start | 4 ++++ lib/constants.rb | 22 ++++++++++++++++++++++ lib/git_mirror.rb | 2 +- src/lib/web_backend.rb | 2 +- 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/container/git-mirror/start b/container/git-mirror/start index f01789c..9c9e3e2 100755 --- a/container/git-mirror/start +++ b/container/git-mirror/start @@ -3,8 +3,17 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig.rb' +names = Set.new %w[ + ES_USER + ES_PASSWORD +] + +service_authentication = relevant_service_authentication(names) +env = docker_env(service_authentication) + DOCKER_CCI = '/c/compass-ci' DOCKER_REPO = '/c/git-repos' docker_rm "git-mirror" @@ -12,10 +21,11 @@ docker_rm "git-mirror" lkp = %x(id -u lkp) group_id = %x(cat /etc/group | grep '^committer' | awk -F ':' '{print $3}') -cmd = %W[ +cmd = %w[ docker run --restart=always --name git-mirror +] + env + %W[ -u #{lkp.to_i}:#{group_id.to_i} -d -e CCI_SRC=#{DOCKER_CCI} diff --git a/container/web-backend/start b/container/web-backend/start index cebf949..250c524 100755 --- a/container/web-backend/start +++ b/container/web-backend/start @@ -13,6 +13,8 @@ load_cci_defaults [[ $SRV_HTTP_RESULT_HOST ]] || SRV_HTTP_RESULT_HOST=$(curl ip.sb) [[ $SRV_HTTP_RESULT_PORT ]] || SRV_HTTP_RESULT_PORT=20007 +load_service_authentication + cmd=( docker run --name web-backend @@ -26,6 +28,8 @@ cmd=( -e CCI_SRC=/c/compass-ci -e LKP_SRC=/c/lkp-tests -e UPSTREAM_REPOS_PATH=/c/upstream-repos + -e ES_USER=$ES_USER + -e ES_PASSWORD=$ES_PASSWORD -v $CCI_SRC:/c/compass-ci -v $LKP_SRC:/c/lkp-tests diff --git a/lib/constants.rb b/lib/constants.rb index 11927f9..c71c3de 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -5,9 +5,31 @@ require_relative '../container/defconfig.rb' config = cci_defaults + +names = Set.new %w[ + ES_USER + ES_PASSWORD +] + ES_HOST ||= config['ES_HOST'] || '172.17.0.1' ES_PORT ||= config['ES_PORT'] || 9200 +if ENV['ES_USER'] + ES_USER = ENV['ES_USER'] + ES_PASSWORD = ENV['ES_PASSWORD'] +else + service_authentication = relevant_service_authentication(names) + ES_USER = service_authentication['ES_USER'] + ES_PASSWORD = service_authentication['ES_PASSWORD'] +end + +ES_HOSTS = [{ + host: ES_HOST, + port: ES_PORT, + user: ES_USER, + password: ES_PASSWORD +}] + SEND_MAIL_HOST ||= config['SEND_MAIL_HOST'] || '172.17.0.1' SEND_MAIL_PORT ||= config['SEND_MAIL_PORT'] || 10001 diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 6100d6b..71d6956 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -125,7 +125,7 @@ class MirrorMain @defaults = {} @git_queue = Queue.new @log = JSONLogger.new - @es_client = Elasticsearch::Client.new(url: "http://#{ES_HOST}:#{ES_PORT}") + @es_client = Elasticsearch::Client.new(hosts: ES_HOSTS) clone_upstream_repo load_fork_info connection_init diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 932d362..c4d4edd 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -34,7 +34,7 @@ NOT_SHOW_FIELDS = %w[result_root].freeze ALL_FIELDS = FIELDS + NOT_SHOW_FIELDS NOT_NEED_EXIST_FIELDS = %w[error_ids upstream_repo].freeze PREFIX_SEARCH_FIELDS = ['tbox_group'].freeze -ES_CLIENT = Elasticsearch::Client.new(url: "http://#{ES_HOST}:#{ES_PORT}") +ES_CLIENT = Elasticsearch::Client.new(hosts: ES_HOSTS) COMPARE_RECORDS_NUMBER = 100 def es_query(query) -- 2.23.0

1 0

[PATCH compass-ci 3/5] services: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

services: lifecycle monitoring extract-stats Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/extract-stats/start | 6 ++++++ container/lifecycle/start | 6 ++++++ container/monitoring/start | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/container/extract-stats/start b/container/extract-stats/start index 621099a..7a0377e 100755 --- a/container/extract-stats/start +++ b/container/extract-stats/start @@ -12,6 +12,8 @@ names = Set.new %w[ REDIS_PORT ES_HOST ES_PORT + ES_USER + ES_PASSWORD SEND_MAIL_HOST SEND_MAIL_PORT SRV_HTTP_RESULT_HOST @@ -20,6 +22,10 @@ names = Set.new %w[ ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' diff --git a/container/lifecycle/start b/container/lifecycle/start index 6e9b4fb..484584f 100755 --- a/container/lifecycle/start +++ b/container/lifecycle/start @@ -8,9 +8,15 @@ require_relative '../defconfig.rb' names = Set.new %w[ LIFECYCLE_PORT + ES_USER + ES_PASSWORD ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) LIFECYCLE_PORT = defaults['LIFECYCLE_PORT'] || '11312' diff --git a/container/monitoring/start b/container/monitoring/start index f4214ed..23d5cb4 100755 --- a/container/monitoring/start +++ b/container/monitoring/start @@ -10,9 +10,15 @@ names = Set.new %w[ MONITOR_PORT MQ_PORT MQ_HOST + ES_USER + ES_PASSWORD ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) DEFAULT_CCI = '/c/cci' -- 2.23.0

1 0

[PATCH compass-ci 2/5] service/scheduler: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/scheduler/my-start | 6 ++++++ container/scheduler/start | 6 ++++++ src/scheduler/elasticsearch_client.cr | 8 +++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/container/scheduler/my-start b/container/scheduler/my-start index e5855af..5dd871e 100755 --- a/container/scheduler/my-start +++ b/container/scheduler/my-start @@ -13,6 +13,8 @@ names = Set.new %w[ REDIS_PORT ES_HOST ES_PORT + ES_USER + ES_PASSWORD OS_HTTP_HOST OS_HTTP_PORT INITRD_HTTP_HOST @@ -38,6 +40,10 @@ defaults['SCHED_PORT'] ||= '3000' defaults['SCHED_HOST'] ||= '172.17.0.1' defaults['MASTER_FLUENTD_PORT'] ||= '24224' defaults['SRV_HTTP_CCI_PORT'] ||= '11307' + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) CCI_REPOS = ENV['CCI_REPOS'] || '/c' diff --git a/container/scheduler/start b/container/scheduler/start index 598b263..029dda8 100755 --- a/container/scheduler/start +++ b/container/scheduler/start @@ -14,6 +14,8 @@ names = Set.new %w[ REDIS_PORT ES_HOST ES_PORT + ES_USER + ES_PASSWORD OS_HTTP_HOST OS_HTTP_PORT INITRD_HTTP_HOST @@ -44,6 +46,10 @@ SCHED_PORT = defaults['SCHED_PORT'] sched_nodes.each do |node| docker_rm "s001-alpine-#{node['port']}" end + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) CCI_REPOS = ENV['CCI_REPOS'] || '/c' diff --git a/src/scheduler/elasticsearch_client.cr b/src/scheduler/elasticsearch_client.cr index 0e83ff1..1ef79a6 100644 --- a/src/scheduler/elasticsearch_client.cr +++ b/src/scheduler/elasticsearch_client.cr @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +require "uri" require "yaml" require "json" require "any_merge" @@ -30,7 +31,12 @@ class Elasticsearch::Client HOST = (ENV.has_key?("ES_HOST") ? ENV["ES_HOST"] : JOB_ES_HOST) PORT = (ENV.has_key?("ES_PORT") ? ENV["ES_PORT"] : JOB_ES_PORT).to_i32 - def initialize(host = HOST, port = PORT) + def initialize(host = HOST, port = PORT, auth = true) + if auth + user = ENV["ES_USER"]? + password = ENV["ES_PASSWORD"]? + host = "http://#{user}:#{URI.encode_www_form(password)}@#{host}" if user && password + end @client = Elasticsearch::API::Client.new({:host => host, :port => port}) end -- 2.23.0

1 0

[PATCH compass-ci 1/5] service/es: configuring authentication
by Wu Zhende 21 Apr '21

21 Apr '21

To improve security, need to enter the account and password to access the kibana web page. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/defconfig.sh | 19 +++++++++++++++++++ container/es/Dockerfile | 7 +++++++ container/es/build | 9 ++++++++- container/es/start | 2 +- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/container/defconfig.sh b/container/defconfig.sh index 50446d8..64a434f 100755 --- a/container/defconfig.sh +++ b/container/defconfig.sh @@ -20,6 +20,12 @@ load_cci_defaults() done } +load_service_authentication() +{ + shopt -s nullglob + create_yaml_variables '/etc/compass-ci/passwd.yaml' +} + docker_rm() { container=$1 @@ -28,6 +34,19 @@ docker_rm() docker rm -f $container } +check_auth_es_ready() +{ + local port=$1 + load_service_authentication + local i + for i in {1..30} + do + + curl -s localhost:$port -u $ES_USER:$ES_PASSWORD> /dev/null && return + sleep 2 + done +} + check_service_ready() { local port=$1 diff --git a/container/es/Dockerfile b/container/es/Dockerfile index 44d87ee..f29e677 100644 --- a/container/es/Dockerfile +++ b/container/es/Dockerfile @@ -4,11 +4,16 @@ FROM elasticsearch:7.11.1@sha256:d52cda1e73d1b1915ba2d76ca1e426620c7b5d6942d9d2f432259503974ba786 ARG MEMORY +ARG USER +ARG PASSWORD RUN sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a path.data: /srv/es' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a node.name: node-1' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a cluster.initial_master_nodes: ["node-1"]' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.security.enabled: true' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.license.self_generated.type: basic' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.security.transport.ssl.enabled: true' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i "s/-Xms256m/-Xms${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options && \ sed -i "s/-Xmx256m/-Xmx${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options @@ -17,6 +22,8 @@ RUN mkdir /usr/share/elasticsearch/tmp && \ WORKDIR /usr/share/elasticsearch +RUN ./bin/elasticsearch-users useradd ${USER} -p ${PASSWORD} -r superuser + ENV PATH /usr/share/elasticsearch/bin:$PATH ENV ES_TMPDIR /usr/share/elasticsearch/tmp diff --git a/container/es/build b/container/es/build index db5145f..e023227 100755 --- a/container/es/build +++ b/container/es/build @@ -3,10 +3,17 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig.rb' +names = Set.new %w[ + ES_USER + ES_PASSWORD +] + +defaults = relevant_service_authentication(names) docker_skip_rebuild "es:7.11.1" available_memory = get_available_memory -system "docker build -t es:7.11.1 --build-arg MEMORY=#{available_memory} --network=host ." +system "docker build -t es:7.11.1 --build-arg MEMORY=#{available_memory} --build-arg USER=#{defaults['ES_USER']} --build-arg PASSWORD=#{defaults['ES_PASSWORD']} --network=host ." diff --git a/container/es/start b/container/es/start index 3aa9525..cff44ea 100755 --- a/container/es/start +++ b/container/es/start @@ -21,5 +21,5 @@ cmd=( "${cmd[@]}" # set index -check_service_ready 9200 +check_auth_es_ready 9200 find $CCI_SRC/sbin/ -name "es-*-mapping.sh" -exec sh {} \; -- 2.23.0

1 0

[PATCH compass-ci] delimiter: delete es_query in bisect script
by Cao Xueliang 20 Apr '21

20 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/git_bisect.rb | 8 +++----- src/delimiter/find-commit/bisect_run_script.rb | 18 +++++++++++------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/git_bisect.rb b/lib/git_bisect.rb index 6aedda5..c0c0a60 100644 --- a/lib/git_bisect.rb +++ b/lib/git_bisect.rb @@ -3,7 +3,6 @@ # frozen_string_literal: true require 'json' -require_relative 'es_query' require_relative 'error_messages' require_relative 'sched_client' require_relative '../src/delimiter/utils' @@ -12,7 +11,6 @@ require_relative "#{ENV['LKP_SRC']}/lib/monitor" # find the first bad commit class GitBisect def initialize(task) - @es = ESQuery.new @task = task end @@ -54,7 +52,7 @@ class GitBisect puts "upstream_commit: #{@upstream_commit}" raise 'upstream info is null' unless @upstream_repo || @upstream_commit - @upstream_repo_git = "git://#{GIT_MIRROR_HOST}/#{@upstream_repo}" + @upstream_repo_git = "git://#{GIT_MIRROR_HOST}/upstream/#{@upstream_repo}" end def set_upstream_url @@ -66,7 +64,7 @@ class GitBisect def set_work_dir @work_dir = Utils.clone_repo(@upstream_repo_git, @upstream_commit) puts "work_dir: #{@work_dir}" - raise "checkout repo: #{@upstream_repo} to commit: #{@upstream_commit} failed!" unless @work_dir + raise "checkout repo: #{@upstream_repo_git} to commit: #{@upstream_commit} failed!" unless @work_dir end def set_pkgbuild_repo @@ -104,7 +102,7 @@ class GitBisect return nil unless temp[0].include?('is the first bad commit') || temp[-1].include?('bisect run success') first_bad_commit = Utils.parse_first_bad_commit(result) - bisect_error = ErrorMessages.new(@build_pkg_dir).obtain_error_messages_by_error_id(@error_id) + bisect_error = Utils.get_bisect_error(@bad_job_id, @error_id) all_errors = Utils.obt_errors(@work_dir, first_bad_commit) first_bad_commit_result_root = Utils.obt_result_root_by_commit(first_bad_commit) diff --git a/src/delimiter/find-commit/bisect_run_script.rb b/src/delimiter/find-commit/bisect_run_script.rb index 8cdf675..05c529e 100755 --- a/src/delimiter/find-commit/bisect_run_script.rb +++ b/src/delimiter/find-commit/bisect_run_script.rb @@ -5,13 +5,11 @@ # frozen_string_literal: true require 'json' -require_relative "#{ENV['CCI_SRC']}/lib/es_query" require_relative "#{ENV['CCI_SRC']}/src/delimiter/utils" # git bisect run class GitBisectRun def initialize(job_id, error_id, work_dir) - @es = ESQuery.new @job_id = job_id @error_id = error_id @work_dir = work_dir @@ -27,12 +25,17 @@ class GitBisectRun private def get_bisect_status(job) - status = Utils.get_job_status(job, @error_id) - exit 125 unless status + begin + status = Utils.get_job_status(job, @error_id) + exit 125 unless status - exit 1 if status.eql?('bad') + exit 1 if status.eql?('bad') - exit 0 + exit 0 + rescue StandardError => e + puts e.backtrace.inspect + exit -1 + end end end @@ -43,6 +46,7 @@ begin run = GitBisectRun.new job_id, error_id, work_dir run.git_bisect -rescue +rescue StandardError => e + puts e.backtrace.inspect exit -1 end -- 2.23.0

1 0

[PATCH compass-ci] container/assist-result: add new function in assist-result
by Cao Xueliang 20 Apr '21

20 Apr '21

add get_compare_result and get_error_messages inassist-result, use them in delimiter utils. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assist-result/routes.rb | 23 +++++++++++++++++++ .../assist-result/views/get_compare_result.rb | 10 ++++++++ .../assist-result/views/get_error_messages.rb | 9 ++++++++ lib/assist_result_client.rb | 20 ++++++++++++++++ src/delimiter/utils.rb | 15 ++++++++---- 5 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 container/assist-result/views/get_compare_result.rb create mode 100644 container/assist-result/views/get_error_messages.rb diff --git a/container/assist-result/routes.rb b/container/assist-result/routes.rb index 287db25..c0712a9 100755 --- a/container/assist-result/routes.rb +++ b/container/assist-result/routes.rb @@ -9,6 +9,8 @@ require 'sinatra' require_relative './views/get_job_yaml' require_relative './views/get_job_content' require_relative './views/check_job_credible' +require_relative './views/get_compare_result' +require_relative './views/get_error_messages' configure do set :bind, '0.0.0.0' @@ -45,3 +47,24 @@ get '/get_job_content/:job_id' do return [200, result.to_json] end + +get '/get_compare_errors/*,*' do |pre_id, cur_id| + begin + result = get_compare_errors(pre_id, cur_id) + rescue StandardError => e + return [400, e.backtrace.inspect] + end + + return [200, result.to_json] +end + +post '/get_error_messages' do + begin + data = JSON.parse(Base64.decode64(request.body.read)) + result = get_error_messages(data['job_id'], data['error_id']) + rescue StandardError => e + return [400, e.backtrace.inspect] + end + + return [200, result.to_json] +end diff --git a/container/assist-result/views/get_compare_result.rb b/container/assist-result/views/get_compare_result.rb new file mode 100644 index 0000000..cf9e1de --- /dev/null +++ b/container/assist-result/views/get_compare_result.rb @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative "#{ENV['CCI_SRC']}/lib/compare_error_messages" + +def get_compare_errors(pre_id, cur_id) + _, errors = CEM.get_compare_result(pre_id, cur_id) + return errors +end diff --git a/container/assist-result/views/get_error_messages.rb b/container/assist-result/views/get_error_messages.rb new file mode 100644 index 0000000..e429dfe --- /dev/null +++ b/container/assist-result/views/get_error_messages.rb @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative "#{ENV['CCI_SRC']}/lib/compare_error_messages" + +def get_error_messages(job_id, error_id) + return CEM.get_error_messages(job_id, error_id) +end diff --git a/lib/assist_result_client.rb b/lib/assist_result_client.rb index 9c42b97..592c841 100644 --- a/lib/assist_result_client.rb +++ b/lib/assist_result_client.rb @@ -42,4 +42,24 @@ class AssistResult return response.body end + + def get_compare_errors(pre_id, cur_id) + resource = RestClient::Resource.new("http://#{@host}:#{@port}/get_compare_errors/#{pre_id},#{cur_id}") + response = resource.get() + return nil unless response.code == 200 + + return response.body + end + + def get_error_messages(job_id, error_id) + data = { + 'job_id' => job_id, + 'error_id' => error_id + } + resource = RestClient::Resource.new("http://#{@host}:#{@port}/get_error_messages") + response = resource.post(Base64.encode64(data.to_json)) + return nil unless response.code == 200 + + return response.body + end end diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index f960c7e..c782ea2 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -67,7 +67,7 @@ module Utils def submit_job(job) save_job_to_yaml(job, PROCESS_JOB_YAML) - response = %x(#{LKP_SRC}/sbin/submit #{PROCESS_JOB_YAML}) + response = %x(#{ENV['LKP_SRC']}/sbin/submit #{PROCESS_JOB_YAML}) puts "submit job response: #{response}" return nil if response =~ /job id=0/ return $1 if response =~ /job id=(.*)/ @@ -87,6 +87,7 @@ module Utils query = { 'job_id': new_job_id, 'job_state': 'extract_finished' } extract_finished = monitor_run_stop(query) return nil unless extract_finished.zero? + puts "#{bad_job_id}, #{new_job_id}, #{error_id}" check_result = AssistResult.new.check_job_credible(bad_job_id, new_job_id, error_id) raise "check job credible failed: #{bad_job_id}, #{new_job_id}, #{error_id}" if check_result == nil @@ -185,13 +186,17 @@ module Utils def obt_errors_by_commits(cur_commit, pre_commit) cur_id = obt_id_by_commit(cur_commit) pre_id = obt_id_by_commit(pre_commit) - _, errors = get_compare_result(pre_id, cur_id) - return errors + return AssistResult.new.get_compare_errors(pre_id, cur_id) end def obt_result_root_by_commit(commit) - id = obt_id_by_commit(commit) - ESQuery.new.query_by_id(id)['result_root'] + job_id = obt_id_by_commit(commit) + job_content = AssistResult.new.get_job_content(job_id) + return JSON.parse(job_content)['result_root'] + end + + def get_bisect_error(job_id, error_id) + return AssistResult.new.get_error_messages(job_id, error_id) end end end -- 2.23.0

1 0

[PATCH v3 lkp-tests] sbin/submit: support -n
by Li Yuanchao 20 Apr '21

20 Apr '21

If -n $number, and $number is a integer and $number > 1, submit $number times. And 'nr_run' would be added in the job.yaml. Else, submit once. [usage] submit xxx xxx -n 3 xxx.yaml Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/submit | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/sbin/submit b/sbin/submit index edb207cc0..a35ca0dac 100755 --- a/sbin/submit +++ b/sbin/submit @@ -19,6 +19,7 @@ opt_monitor_query = {} opt_my_queue = false actions = ['output', 'stop'] result_roots = [] +nr_run = 1 options = OptionParser.new do |opts| opts.banner = 'Usage: submit [options] job1.yaml job2.yaml ...' @@ -57,6 +58,11 @@ options = OptionParser.new do |opts| actions << 'mirror_result' end + opts.on('-n job_number', '--number job_number', 'number to submit job') do |number| + nr_run = number.to_i + nr_run = 1 if nr_run < 1 + end + opts.on('-m', '--monitor', "monitor job status: use -m 'KEY: VALUE' to add rule") do opt_monitor = true k, v = ARGV[0].sub(' ', '').split(':', 2) if ARGV[0] @@ -195,16 +201,19 @@ ARGV.each do |jobfile| scheduler_client = SchedulerClient.new(job['SCHED_HOST'], job['SCHED_PORT']) # submit job + job_hash['nr_run'] = nr_run job_json = job_hash.to_json - messages = scheduler_client.submit_job(job_json) - JSON.parse(messages).each do |msg| - if msg['message'].empty? - result_roots << msg['result_root'] - job_ids << msg['job_id'].to_s - puts("submit #{jobfile}, got job id=#{msg['job_id']}") - else - opt_monitor = false - puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + nr_run.times do + messages = scheduler_client.submit_job(job_json) + JSON.parse(messages).each do |msg| + if msg['message'].empty? + result_roots << msg['result_root'] + job_ids << msg['job_id'].to_s + puts("submit #{jobfile}, got job id=#{msg['job_id']}") + else + opt_monitor = false + puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + end end end end -- 2.23.0

1 0

[PATCH compass-ci] lib: use module as namespace in compare_error_messages.rb
by Cao Xueliang 20 Apr '21

20 Apr '21

1. use module CEM. 2. add get_error_messages function. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- .../assist-result/views/check_job_credible.rb | 2 +- lib/compare_error_messages.rb | 138 ++++++++++-------- 2 files changed, 75 insertions(+), 65 deletions(-) diff --git a/container/assist-result/views/check_job_credible.rb b/container/assist-result/views/check_job_credible.rb index a8d387f..c5fc453 100644 --- a/container/assist-result/views/check_job_credible.rb +++ b/container/assist-result/views/check_job_credible.rb @@ -5,5 +5,5 @@ require_relative "#{ENV['CCI_SRC']}/lib/compare_error_messages" def check_job_credible(pre_job_id, cur_job_id, error_id) - return credible?(pre_job_id, cur_job_id, error_id) + return CEM.credible?(pre_job_id, cur_job_id, error_id) end diff --git a/lib/compare_error_messages.rb b/lib/compare_error_messages.rb index 9c864a3..93b38ed 100644 --- a/lib/compare_error_messages.rb +++ b/lib/compare_error_messages.rb @@ -8,91 +8,101 @@ require_relative 'es_query' require_relative 'error_messages' require_relative "#{ENV['LKP_SRC']}/lib/common" -# previous_job_id's error_ids include error_id -# If later_job_id's output include error filename that extract from error_id, -# the later_job_id is credible. -def credible?(previous_job_id, later_job_id, error_id) - es = ESQuery.new - later_es_result = es.query_by_id(later_job_id) +module CEM + extend self - return true if later_es_result['job_state'] == 'finished' + # previous_job_id's error_ids include error_id + # If later_job_id's output include error filename that extract from error_id, + # the later_job_id is credible. + def credible?(previous_job_id, later_job_id, error_id) + es = ESQuery.new + later_es_result = es.query_by_id(later_job_id) - previous_result_file = File.join('/srv', es.query_by_id(previous_job_id)['result_root'], 'build-pkg') - later_result_file = File.join('/srv', later_es_result['result_root'], 'build-pkg') + return true if later_es_result['job_state'] == 'finished' - filenames_check = filenames_check(previous_result_file, later_result_file, error_id) + previous_result_file = File.join('/srv', es.query_by_id(previous_job_id)['result_root'], 'build-pkg') + later_result_file = File.join('/srv', later_es_result['result_root'], 'build-pkg') - return false if filenames_check.empty? || filenames_check.value?(false) + filenames_check = filenames_check(previous_result_file, later_result_file, error_id) - return true -end - -def filenames_check(previous_result_file, later_result_file, error_id) - filenames = Set.new - filenames_check = Hash.new { |h, k| h[k] = false } + return false if filenames_check.empty? || filenames_check.value?(false) - error_lines = ErrorMessages.new(previous_result_file).obtain_error_messages_by_error_id(error_id, true) - error_lines.each do |error_line| - # "src/ssl_sock.c:1454:104: warning: unused parameter 'al' [-Wunused-parameter]" => "src/ssl_sock" - filenames << $1.chomp(File.extname($1)) if error_line =~ /(.*)(:\d+){2}: (error|warning):/ + return true end - File.open(later_result_file).each_line do |line| - filenames.each do |filename| - filenames_check[filename] - filenames_check[filename] = true if line.include?(filename) + def filenames_check(previous_result_file, later_result_file, error_id) + filenames = Set.new + filenames_check = Hash.new { |h, k| h[k] = false } + + error_lines = ErrorMessages.new(previous_result_file).obtain_error_messages_by_error_id(error_id, true) + error_lines.each do |error_line| + # "src/ssl_sock.c:1454:104: warning: unused parameter 'al' [-Wunused-parameter]" => "src/ssl_sock" + filenames << $1.chomp(File.extname($1)) if error_line =~ /(.*)(:\d+){2}: (error|warning):/ + end + + File.open(later_result_file).each_line do |line| + filenames.each do |filename| + filenames_check[filename] + filenames_check[filename] = true if line.include?(filename) + end end + + return filenames_check end - return filenames_check -end + def get_compare_result(previous_job_id, later_job_id) + es = ESQuery.new -def get_compare_result(previous_job_id, later_job_id) - es = ESQuery.new + previous_error_ids = es.query_by_id(previous_job_id)['error_ids'] + later_es_result = es.query_by_id(later_job_id) + later_error_ids = later_es_result['error_ids'] + later_result_file = File.join('/srv', later_es_result['result_root'], 'build-pkg') - previous_error_ids = es.query_by_id(previous_job_id)['error_ids'] - later_es_result = es.query_by_id(later_job_id) - later_error_ids = later_es_result['error_ids'] - later_result_file = File.join('/srv', later_es_result['result_root'], 'build-pkg') + new_error_ids = get_new_error_ids(later_error_ids, previous_error_ids) + error_messages = ErrorMessages.new(later_result_file).obtain_error_messages + new_error_number, formatted_error_messages = format_error_messages(error_messages, new_error_ids) - new_error_ids = get_new_error_ids(later_error_ids, previous_error_ids) - error_messages = ErrorMessages.new(later_result_file).obtain_error_messages - new_error_number, formatted_error_messages = format_error_messages(error_messages, new_error_ids) + # new error use ">>" as an identifier in the error header + return new_error_number, formatted_error_messages + end - # new error use ">>" as an identifier in the error header - return new_error_number, formatted_error_messages -end + def get_error_messages(job_id, error_id) + es_result = ESQuery.new.query_by_id(job_id) + result_file = File.join('/srv', es_result['result_root'], 'build-pkg') + return ErrorMessages.new(result_file).obtain_error_messages_by_error_id(error_id) + end -def get_new_error_ids(later_error_ids, previous_error_ids) - return nil_to_empty_array(later_error_ids) - nil_to_empty_array(previous_error_ids) -end + def get_new_error_ids(later_error_ids, previous_error_ids) + return nil_to_empty_array(later_error_ids) - nil_to_empty_array(previous_error_ids) + end -def nil_to_empty_array(array) - array = [] if array.nil? - return array -end + def nil_to_empty_array(array) + array = [] if array.nil? + return array + end -def format_error_messages(error_messages, new_error_ids) - formatted_error_messages = Terminal::Table.new - formatted_error_messages.style = { border_x: '', border_y: '', border_i: '', padding_left: 0 } - new_error_number = 0 - - error_messages.each do |k, v| - if new_error_ids.include?("build-pkg.#{build_pkg_error_id(k)}") - new_error_number += 1 - formatted_error_messages = add_sign(formatted_error_messages, '>>', v) - else - formatted_error_messages = add_sign(formatted_error_messages, ' ', v) + def format_error_messages(error_messages, new_error_ids) + formatted_error_messages = Terminal::Table.new + formatted_error_messages.style = { border_x: '', border_y: '', border_i: '', padding_left: 0 } + new_error_number = 0 + + error_messages.each do |k, v| + if new_error_ids.include?("build-pkg.#{build_pkg_error_id(k)}") + new_error_number += 1 + formatted_error_messages = add_sign(formatted_error_messages, '>>', v) + else + formatted_error_messages = add_sign(formatted_error_messages, ' ', v) + end end + + return new_error_number, formatted_error_messages.to_s end - return new_error_number, formatted_error_messages.to_s -end + def add_sign(formatted_error_messages, sign, set) + set.each do |value| + formatted_error_messages.add_row([sign, value]) + end -def add_sign(formatted_error_messages, sign, set) - set.each do |value| - formatted_error_messages.add_row([sign, value]) + return formatted_error_messages end - - return formatted_error_messages end -- 2.23.0

1 0

[PATCH compass-ci] sbin/auto_submit: support argvs for competition
by Li Yuanchao 20 Apr '21

20 Apr '21

Get true author_email and committer_email for PR. And get base_commit for branch, which is the commit when branch is created. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/sbin/auto_submit b/sbin/auto_submit index 4adce89..28e3715 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -51,17 +51,35 @@ class AutoSubmit real_argvs = Array.new(submit_argv) real_argvs.push("upstream_branch=#{branch}") - real_argvs.push("upstream_commit=#{commit_id}") real_argvs.push("upstream_dir=#{belong}") tag = %x(git -C #{mirror_dir} tag --points-at #{commit_id}) real_argvs.push("upstream_tag=#{tag}") unless tag.empty? - author_email = %x(git -C #{mirror_dir} log --format=%ae -1) - committer_email = %x(git -C #{mirror_dir} log --format=%ce -1) - real_argvs.push("author_email=#{author_email} committer_email=#{committer_email}") + if git_repo.include?('my-sql_with_boostkit_patch') + competi_argvs = get_argvs_for_competition(mirror_dir, branch, commit_id) + real_argvs.push(competi_argvs) + end + real_argvs.push("upstream_commit=#{commit_id}") system(real_argvs.join(' ')) end + def get_argvs_for_competition(real_argvs, mirror_dir, branch, commit_id) + argvs_for_base = Array.new(real_argvs) + author_email = %x(git -C #{mirror_dir} log --format=%ae -1 #{commit_id}).chomp + committer_email = %x(git -C #{mirror_dir} log --format=%ce -1 #{commit_id}).chomp + + if committer_email.include?('noreply') + emails = %x(git -C #{mirror_dir} log --format=%ae -2 #{commit_id}) + committer_email = emails.lines[0].chomp + author_email = emails.lines[1].chomp + end + + base_commit = %x(git -C #{mirror_dir} merge-base master #{branch}).chomp + argvs_for_base.push("upstream_commit=#{base_commit}") + system(argvs_for_base.join(' ')) + return "author_email=#{author_email} committer_email=#{committer_email} base_commit=#{base_commit}" + end + def submit_specific_branch(submit_argv, newrefs_info, branches) branches.each do |branch_name| branch = "refs/heads/#{branch_name}" -- 2.23.0

1 0

[PATCH compass-ci] extract-stats: fix expect ":" but got "="
by Cao Xueliang 20 Apr '21

20 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/stats_worker.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index 47c3a59..f385fee 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -99,7 +99,7 @@ class StatsWorker sample_error_id = new_error_ids.sample @log.info("send a delimiter task: job_id is #{job_id}") queue = "#{DELIMITER_TASK_QUEUE}/#{job_id}" - value = {"job_id" => job_id, "error_id" => sample_error_id} + value = %({"job_id": "#{job_id}", "error_id": "#{sample_error_id}"}) @etcd.put(queue, value) msg = %({"job_id": "#{job_id}", "new_error_id": "#{sample_error_id}"}) -- 2.23.0

1 0

[PATCH compass-ci] container/scheduler-http: fix unfriendly error
by Liu Yinsi 20 Apr '21

20 Apr '21

before: /c/compass-ci/container/scheduler-https# ./start Traceback (most recent call last): ./start:14:in `<main>': CA certificate is missing. Ignore if the HTTPS scheduler is not required (RuntimeError) after: /c/compass-ci/container/scheduler-https# ./start CA certificate is missing. Ignore if the HTTPS scheduler is not required Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/scheduler-https/start | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/container/scheduler-https/start b/container/scheduler-https/start index 4ec84a28..374300c0 100755 --- a/container/scheduler-https/start +++ b/container/scheduler-https/start @@ -11,7 +11,10 @@ crt_path = "#{ca_path}/server.crt" key_path = "#{ca_path}/server.key" warn_msg = 'CA certificate is missing. Ignore if the HTTPS scheduler is not required' -raise warn_msg unless File.exist?(crt_path) && File.exist?(key_path) +unless File.exist?(crt_path) && File.exist?(key_path) + puts warn_msg + exit +end names = Set.new %w[ SCHED_HOST -- 2.23.0

1 0

[PATCH v2 compass-ci] lib/rpm-repo.rb: do not move to pub directory until test completed
by Li Ping 20 Apr '21

20 Apr '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/rpm-repo.rb | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/rpm-repo.rb b/lib/rpm-repo.rb index 658a5270..c8d6b087 100755 --- a/lib/rpm-repo.rb +++ b/lib/rpm-repo.rb @@ -6,6 +6,7 @@ require 'fileutils' require 'json' require './mq_client' +require_relative 'json_logger.rb' MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || '172.17.0.1' MQ_PORT = ENV['MQ_PORT'] || 5672 @@ -29,31 +30,47 @@ MQ_PORT = ENV['MQ_PORT'] || 5672 # change /srv/rpm/pub/**/repodate # @mq.ack(info) class HandleRepo + @@upload_dir_prefix = "/srv/rpm/upload/" def initialize @mq = MQClient.new(MQ_HOST, MQ_PORT) - @update = [] + @log = JSONLogger.new end def handle_new_rpm queue = @mq.queue("update_repo") queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| - rpm_info = JSON.parse(msg) - rpm_info["upload_rpms"].each do |rpm| - rpm_path = File.dirname(rpm).sub("upload", "testing") - FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) + begin + rpm_info = JSON.parse(msg) + check_upload_rpms(rpm_info) + rpm_info["upload_rpms"].each do |rpm| + rpm_path = File.dirname(rpm).sub("upload", "testing") + FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) - dest = File.join(rpm_path.to_s, File.basename(rpm)) - @update << dest - FileUtils.mv(rpm, dest) - system("createrepo --update $(dirname #{rpm_path})") + dest = File.join(rpm_path.to_s, File.basename(rpm)) + FileUtils.mv(rpm, dest) + system("createrepo --update $(dirname #{rpm_path})") + end + @mq.ack(info) + rescue StandardError => e + @log.warn({ + "error message": e.message + }.to_json) + @mq.ack(info) end - update_pub_dir - @mq.ack(info) end end - def update_pub_dir - @update.each do |rpm| + def check_upload_rpms(data) + raise JSON.dump({ "errcode" => "200", "errmsg" => "no upload_rpms params" }) unless data.key?("upload_rpms") + raise JSON.dump({ "errcode" => "200", "errmsg" => "upload_rpms params type error" }) if data["upload_rpms"].class != Array + data["upload_rpms"].each do |rpm| + raise JSON.dump({ "errcode" => "200", "errmsg" => "#{rpm} not exist" }) unless File.exist?(rpm) + raise JSON.dump({ "errcode" => "200", "errmsg" => "the upload directory is incorrect" }) unless File.dirname(rpm).start_with?(@@upload_dir_prefix) + end + end + + def update_pub_dir(update) + update.each do |rpm| pub_path = File.dirname(rpm).sub("testing", "pub") FileUtils.mkdir_p(pub_path) unless File.directory?(pub_path) -- 2.23.0

1 0

[PATCH compass-ci 3/3] service/kibana-logging: configuring ES authentication information
by Wu Zhende 20 Apr '21

20 Apr '21

logging-es set up the certification. kibana-logging needs to configure the connection information to connect to the logging-es. When logging to the kibana web page, need to enter the account and password to enhance security. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana-logging/Dockerfile | 7 +++++-- container/kibana-logging/build | 10 +++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/container/kibana-logging/Dockerfile b/container/kibana-logging/Dockerfile index 970eb5a..7085618 100644 --- a/container/kibana-logging/Dockerfile +++ b/container/kibana-logging/Dockerfile @@ -5,8 +5,11 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -# docker image borrowed from hub.docker.com/r/gagara/kibana-oss-arm64 +ARG USER +ARG PASSWORD MAINTAINER Wu Zhende <wuzhende666(a)163.com> -RUN sed -i 's/server.host: "0"/server.host: "0.0.0.0"/' config/kibana.yml +RUN sed -i 's/server.host: "0"/server.host: "0.0.0.0"/' config/kibana.yml && \ + sed -i "6i\elasticsearch.username: ${USER}" config/kibana.yml && \ + sed -i "6i\elasticsearch.password: ${PASSWORD}" config/kibana.yml diff --git a/container/kibana-logging/build b/container/kibana-logging/build index 9c75aeb..238a88d 100755 --- a/container/kibana-logging/build +++ b/container/kibana-logging/build @@ -3,8 +3,16 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig' +names = Set.new %w[ + LOGGING_ES_USER + LOGGING_ES_PASSWORD +] + +defaults = relevant_service_account(names) + docker_skip_rebuild "kibana:7.11.1" BASE_IMAGE_DICT = { @@ -13,4 +21,4 @@ BASE_IMAGE_DICT = { BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] -system "docker build -t kibana:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} ." +system "docker build -t kibana:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg USER=#{defaults['LOGGING_ES_USER']} --build-arg PASSWORD=#{defaults['LOGGING_ES_PASSWORD']} ." -- 2.23.0

1 0

[PATCH compass-ci 2/3] service/master-fluentd: configuring ES authentication information
by Wu Zhende 20 Apr '21

20 Apr '21

logging-es set up the certification. fluentd needs to configure the connection information to connect to the logging-es. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/master-fluentd/docker-fluentd.conf | 2 ++ container/master-fluentd/start | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/container/master-fluentd/docker-fluentd.conf b/container/master-fluentd/docker-fluentd.conf index 410f2eb..73d7103 100644 --- a/container/master-fluentd/docker-fluentd.conf +++ b/container/master-fluentd/docker-fluentd.conf @@ -42,6 +42,8 @@ @type elasticsearch host "#{ENV['LOGGING_ES_HOST']}" port "#{ENV['LOGGING_ES_PORT']}" + user "#{ENV['LOGGING_ES_USER']}" + password "#{ENV['LOGGING_ES_PASSWORD']}" suppress_type_name true flush_interval 1s num_threads 10 diff --git a/container/master-fluentd/start b/container/master-fluentd/start index a05881c..27aece9 100755 --- a/container/master-fluentd/start +++ b/container/master-fluentd/start @@ -9,11 +9,17 @@ require_relative '../defconfig.rb' names = Set.new %w[ LOGGING_ES_HOST LOGGING_ES_PORT + LOGGING_ES_USER + LOGGING_ES_PASSWORD ] +account_info = relevant_service_account(names) + defaults = relevant_defaults(names) defaults['LOGGING_ES_HOST'] ||= '172.17.0.1' defaults['LOGGING_ES_PORT'] ||= '9202' +defaults.merge!(account_info) + env = docker_env(defaults) docker_rm 'master-fluentd' -- 2.23.0

1 0

[PATCH lab-z9] add new info for a125
by Bai Jing 20 Apr '21

20 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- hosts/taishan200-2280-2s64p-256g--a125 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/hosts/taishan200-2280-2s64p-256g--a125 b/hosts/taishan200-2280-2s64p-256g--a125 index 5c5862e..f2cb708 100644 --- a/hosts/taishan200-2280-2s64p-256g--a125 +++ b/hosts/taishan200-2280-2s64p-256g--a125 @@ -2,11 +2,7 @@ memory: 256G nr_hdd_partitions: 2 hdd_partitions: - /dev/disk/by-id/scsi-35000039988707ca1 - - scsi-35000039988707ca10 - - scsi-35000039988707ca1-part* - - /dev/disk/by-id/scsi-35000039988707d25-part1 - - /dev/disk/by-id/scsi-35000039988707d25-part2 - - /dev/disk/by-id/scsi-35000039988707d25-part3 + - /dev/disk/by-id/scsi-35000039988707d25 mac_addr: - 44:67:47:85:f4:65 - 44:67:47:85:f4:66 -- 2.23.0

1 0

[PATCH v2 lkp-tests] sbin/submit: support nr_run
by Li Yuanchao 20 Apr '21

20 Apr '21

If nr_run is a integer and nr_run > 1, submit nr_run times. Else, submit once. [usage] submit xxx xxx nr_run=3 xxx.yaml Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/submit | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/sbin/submit b/sbin/submit index edb207cc0..bcc155f3e 100755 --- a/sbin/submit +++ b/sbin/submit @@ -195,16 +195,21 @@ ARGV.each do |jobfile| scheduler_client = SchedulerClient.new(job['SCHED_HOST'], job['SCHED_PORT']) # submit job + nr_run = job_hash['nr_run'] || 1 + nr_run = nr_run.to_i + nr_run = 1 if nr_run < 1 job_json = job_hash.to_json - messages = scheduler_client.submit_job(job_json) - JSON.parse(messages).each do |msg| - if msg['message'].empty? - result_roots << msg['result_root'] - job_ids << msg['job_id'].to_s - puts("submit #{jobfile}, got job id=#{msg['job_id']}") - else - opt_monitor = false - puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + nr_run.times do + messages = scheduler_client.submit_job(job_json) + JSON.parse(messages).each do |msg| + if msg['message'].empty? + result_roots << msg['result_root'] + job_ids << msg['job_id'].to_s + puts("submit #{jobfile}, got job id=#{msg['job_id']}") + else + opt_monitor = false + puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + end end end end -- 2.23.0

1 0

[PATCH compass-ci] sparrow/3-code/dev-env: set default value
by Liu Yinsi 20 Apr '21

20 Apr '21

[error] submit_id=e628930d-f8ce-4ea1-b323-f8593c6f60cf 2021-04-19 20:59:08 +0800 WARN -- skip non-executable /c/lkp-tests/tests/self-test submit /c/lkp-tests/jobs/host-info.yaml failed, got job id=0, error: The /srv/initrd/lkp/latest/lkp-aarch64.cgz does not exist. [why] when selftest, if not to define $lkp_initrd_user in /etc/compass-ci/defaults/*.yaml, when submit job, it will not to find /srv/initrd/lkp/$lkp_initrd_user/lkp-$arch.cgz. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 969889ed..e02de355 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -55,12 +55,14 @@ server_name=$(hostname | cut -f1 -d.) : ${MAILDIR:=/srv/cci/Maildir/.compass-ci} : ${SENT_MAILDIR:=/srv/cci/Maildir/.sent} : ${NETDATA_PORT:=20013} +: ${lkp_initrd_user:=latest} mkdir -p /etc/compass-ci/defaults cat > /etc/compass-ci/defaults/$server_name.yaml <<EOF SCHED_HOST: $sched_host SCHED_PORT: $sched_port lab: $lab +lkp_initrd_user: $lkp_initrd_user EOF [[ -f /etc/compass-ci/defaults/upstream-config ]] || { -- 2.23.0

1 0

[PATCH compass-ci] sparrow/1-storage/permission: set lkp owner and group
by Liu Yinsi 20 Apr '21

20 Apr '21

/srv/initrd/deps and /srv/initrd/pkg permission must be lkp:lkp, or build pkg and deps are unable to transfer to the directories. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/1-storage/permission | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sparrow/1-storage/permission b/sparrow/1-storage/permission index db33a93..3c229c5 100755 --- a/sparrow/1-storage/permission +++ b/sparrow/1-storage/permission @@ -13,6 +13,9 @@ chmod_dir=( /srv/rabbitmq /c/git-repos /srv/tmp/public + /srv/initrd + /srv/initrd/pkg + /srv/initrd/deps ) chmod 775 "${chmod_dir[@]}" @@ -20,6 +23,8 @@ chmod 775 "${chmod_dir[@]}" chown_dir=( /srv/result /srv/initrd + /srv/initrd/pkg + /srv/initrd/deps /srv/es /srv/cache/netdata_cache /srv/cache/netdata_lib -- 2.23.0

1 0

[PATCH lkp-tests] sbin/submit: support nr_run
by Li Yuanchao 20 Apr '21

20 Apr '21

If nr_run is a integer and nr_run > 1, submit nr_run times. Else, submit once. [usage] submit xxx xxx nr_run=3 xxx.yaml Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/submit | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/sbin/submit b/sbin/submit index edb207cc0..2e6ad8960 100755 --- a/sbin/submit +++ b/sbin/submit @@ -195,16 +195,21 @@ ARGV.each do |jobfile| scheduler_client = SchedulerClient.new(job['SCHED_HOST'], job['SCHED_PORT']) # submit job - job_json = job_hash.to_json - messages = scheduler_client.submit_job(job_json) - JSON.parse(messages).each do |msg| - if msg['message'].empty? - result_roots << msg['result_root'] - job_ids << msg['job_id'].to_s - puts("submit #{jobfile}, got job id=#{msg['job_id']}") - else - opt_monitor = false - puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + nr_run = job_hash['nr_run'] || 1 + nr_run = nr_run.to_i + nr_run = 1 if nr_run < 1 + nr_run.times do + job_json = job_hash.to_json + messages = scheduler_client.submit_job(job_json) + JSON.parse(messages).each do |msg| + if msg['message'].empty? + result_roots << msg['result_root'] + job_ids << msg['job_id'].to_s + puts("submit #{jobfile}, got job id=#{msg['job_id']}") + else + opt_monitor = false + puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + end end end end -- 2.23.0

3 4

[PATCH lkp-tests 2/2] install-iso-firstboot: modify how job's IPXE response is retrieved
by Yu Chuan 19 Apr '21

19 Apr '21

[Why] Vmlinuz, initrd, reboot.cgz is all that is required for firstboot after ISO is installed. Therefore, the original os_mount=local is changed to custom_ipxe to obtain the ipxe response. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/install-iso-firstboot.yaml | 9 +-------- tests/install-iso-pre | 14 +++++++++----- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/jobs/install-iso-firstboot.yaml b/jobs/install-iso-firstboot.yaml index 7be9eaaab184..c170b8a07f8a 100644 --- a/jobs/install-iso-firstboot.yaml +++ b/jobs/install-iso-firstboot.yaml @@ -1,10 +1,3 @@ suite: install-iso-firstboot -sleep: 1 - -# os_mount must be equal to 'local' -os_mount: local - -# kernel_custom_params must have 'boot_lv_suffix' and 'src_lv_suffix', -# and they must be the same. so can boot from the specify logical volume -kernel_custom_params: +custom_ipxe: diff --git a/tests/install-iso-pre b/tests/install-iso-pre index 768ffb2cca45..c89895a994cc 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -225,14 +225,18 @@ submit_install_iso_firstboot_yaml() local firstboot_yaml="${LKP_SRC}/jobs/install-iso-firstboot.yaml" [ -f "${firstboot_yaml}" ] || die "cannot find ${firstboot_yaml}" + custom_ipxe="#!ipxe\\\\n" + custom_ipxe="${custom_ipxe}\\\\ndhcp" + custom_ipxe="${custom_ipxe}\\\\ninitrd http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/${iso_os}/${iso_arch}/${iso_version}/initrd.lkp" + custom_ipxe="${custom_ipxe}\\\\ninitrd http://${INITRD_HTTP_HOST}:${INITRD_HTTP_PORT}/initrd/custom_bootstrap/reboot.cgz" + custom_ipxe="${custom_ipxe}\\\\nkernel http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/${iso_os}/${iso_arch}/${iso_version}/boot/vmlinuz ip=dhcp root=/dev/mapper/os-${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX} initrd=initrd.lkp initrd=reboot.cgz local rw" + custom_ipxe="${custom_ipxe}\\\\nboot\\\\n" + + sed -i "/custom_ipxe:/ccustom_ipxe: \"${custom_ipxe}\"" $firstboot_yaml + ${LKP_SRC}/sbin/submit \ ${firstboot_yaml} \ suite=install-iso-firstboot-${ROOTFS_SUFFIX} \ - os=${iso_os} \ - os_arch=${iso_arch} \ - os_version=${iso_version} \ - os_mount=local \ - kernel_custom_params="src_lv_suffix=${ROOTFS_SUFFIX} boot_lv_suffix=${ROOTFS_SUFFIX}" \ testbox=$testbox || die "submit ${firstboot_yaml} failed" } -- 2.23.0

1 0

[PATCH lkp-tests 1/2] install-iso-pre: fix some bugs
by Yu Chuan 19 Apr '21

19 Apr '21

[What Bugs] 1. fix the use of secrets fields. 2. add mkfs.ext4 step after create logical volume. 3. vmlinux => vmlinuz 4. ks upload dir: - before: /srv/os/kickstart/ks/${iso_os}_${iso_arch}_${iso_version}.cfg - after: /srv/os/install/ks/${iso_os}_${iso_arch}_${iso_version}.ks 5. second-level hash problem when submit. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index ef4a0659fe02..768ffb2cca45 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -21,7 +21,7 @@ check_params() "HTTP_SERVER_HOST" "HTTP_SERVER_PORT" "HTTP_ROOT_DIR" - "ROOTFS_ROOT_PASSWD" + "secrets_ROOTFS_ROOT_PASSWD" "rootfs_disk" "RSYNC_SERVER_HOST" @@ -31,7 +31,7 @@ check_params() "NFS_ROOT_DIR" "LAB_Z9_URL" - "MY_TOKEN" + "secrets_MY_TOKEN" ) for i in "${required_vars[@]}" @@ -65,6 +65,8 @@ create_lv() log_info "start create lv:$LV_NAME" lvcreate -y -L 10G -n ${LV_NAME} ${VG_NAME} || die "create logical volume failed." + + mkfs.ext4 /dev/mapper/${VG_NAME}-${LV_NAME} } prepare_lv() @@ -111,7 +113,7 @@ generate_ks() logvol / --vgname=${VG_NAME} --name=${LV_NAME} --useexisting # Root password - rootpw --iscrypted ${ROOTFS_ROOT_PASSWD} + rootpw --iscrypted ${secrets_ROOTFS_ROOT_PASSWD} # Reboot after installation reboot --eject @@ -180,7 +182,7 @@ prepare_submit() SCHED_PORT: ${SCHED_PORT} my_name: ${my_name} my_email: ${my_email} - my_token: ${MY_TOKEN} + my_token: ${secrets_MY_TOKEN} lab: z9 EOF } @@ -195,7 +197,7 @@ prepare_install_iso_yaml() custom_ipxe="#!ipxe\\\\n" custom_ipxe="${custom_ipxe}\\\\ndhcp" custom_ipxe="${custom_ipxe}\\\\ninitrd ${tftp_root}/initrd.img" - custom_ipxe="${custom_ipxe}\\\\nkernel ${tftp_root}/vmlinux ro inst.geoloc=0 console=ttyS0 smmu.bypassdev=0x1000:0x17 smmu.bypassde1000:0x15 viff video=VGA-1:640x480-32@60me initrd=initrd.img ip=dhcp inst.ks=http://${HTTP_SERVER_HOST}:${HTTP_SERVER_PORT}/os/kickstart/ks/${iso_os}_${iso_arch}_${iso_version}.cfg" + custom_ipxe="${custom_ipxe}\\\\nkernel ${tftp_root}/vmlinuz ro inst.geoloc=0 console=ttyS0 smmu.bypassdev=0x1000:0x17 smmu.bypassde1000:0x15 viff video=VGA-1:640x480-32@60me initrd=initrd.img ip=dhcp inst.ks=http://${HTTP_SERVER_HOST}:${HTTP_SERVER_PORT}/os/install/ks/${iso_os}_${iso_arch}_${iso_version}.ks" custom_ipxe="${custom_ipxe}\\\\nboot\\\\n" log_info "custom_ipxe: $custom_ipxe" @@ -244,13 +246,13 @@ submit_install_iso_sync_rootfs_yaml() ${LKP_SRC}/sbin/submit \ ${sync_rootfs_yaml} \ suite=install-iso-sync-rootfs-${ROOTFS_SUFFIX} \ - iso_os=${iso_os} \ - iso_arch=${iso_arch} \ - iso_version=${iso_version} \ - NFS_SERVER_HOST=${NFS_SERVER_HOST} \ - NFS_ROOT_DIR=${NFS_ROOT_DIR} \ - ROOTFS_SUFFIX=${ROOTFS_SUFFIX} \ - VG_NAME=${VG_NAME} \ + install-iso-sync-rootfs.iso_os=${iso_os} \ + install-iso-sync-rootfs.iso_arch=${iso_arch} \ + install-iso-sync-rootfs.iso_version=${iso_version} \ + install-iso-sync-rootfs.NFS_SERVER_HOST=${NFS_SERVER_HOST} \ + install-iso-sync-rootfs.NFS_ROOT_DIR=${NFS_ROOT_DIR} \ + install-iso-sync-rootfs.ROOTFS_SUFFIX=${ROOTFS_SUFFIX} \ + install-iso-sync-rootfs.VG_NAME=${VG_NAME} \ testbox=$testbox || die "submit ${sync_rootfs_yaml} failed" } -- 2.23.0

1 0

[PATCH compass-ci] lib/rpm-repo.rb: do not move to pub directory until test completed
by Li Ping 19 Apr '21

19 Apr '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/rpm-repo.rb | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/lib/rpm-repo.rb b/lib/rpm-repo.rb index 658a5270..f2344dcc 100755 --- a/lib/rpm-repo.rb +++ b/lib/rpm-repo.rb @@ -31,29 +31,41 @@ MQ_PORT = ENV['MQ_PORT'] || 5672 class HandleRepo def initialize @mq = MQClient.new(MQ_HOST, MQ_PORT) - @update = [] end def handle_new_rpm queue = @mq.queue("update_repo") queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| - rpm_info = JSON.parse(msg) - rpm_info["upload_rpms"].each do |rpm| - rpm_path = File.dirname(rpm).sub("upload", "testing") - FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) + begin + rpm_info = JSON.parse(msg) + check_upload_rpms(rpm_info) + rpm_info["upload_rpms"].each do |rpm| + rpm_path = File.dirname(rpm).sub("upload", "testing") + FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) - dest = File.join(rpm_path.to_s, File.basename(rpm)) - @update << dest - FileUtils.mv(rpm, dest) - system("createrepo --update $(dirname #{rpm_path})") + dest = File.join(rpm_path.to_s, File.basename(rpm)) + FileUtils.mv(rpm, dest) + system("createrepo --update $(dirname #{rpm_path})") + end + @mq.ack(info) + rescue StandardError => e + puts "error message: ", e.message + @mq.ack(info) end - update_pub_dir - @mq.ack(info) end end - def update_pub_dir - @update.each do |rpm| + def check_upload_rpms(data) + raise JSON.dump({ 'errcode' => '200', 'errmsg' => 'no upload_rpms params' }) unless data.key?('upload_rpms') + raise JSON.dump({ 'errcode' => '200', 'errmsg' => "upload_rpms params type error" }) if data["upload_rpms"].class != Array + data["upload_rpms"].each do |rpm| + raise JSON.dump({ "errcode" => "200", "errmsg" => "#{rpm} not exist" }) unless File.exist?(rpm) + raise JSON.dump({ "errcode" => "200", "errmsg" => "the upload directory is incorrect" }) unless File.dirname(rpm).start_with?("/srv/rpm/upload/") + end + end + + def update_pub_dir(update) + update.each do |rpm| pub_path = File.dirname(rpm).sub("testing", "pub") FileUtils.mkdir_p(pub_path) unless File.directory?(pub_path) -- 2.23.0

1 0

[PATCH compass-ci] container/webhook: change the port which listen to
by Li Yuanchao 19 Apr '21

19 Apr '21

Change from 11301 to 20005 as planned Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/webhook/root/webhook.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/webhook/root/webhook.js b/container/webhook/root/webhook.js index 4639e14..6ca836e 100755 --- a/container/webhook/root/webhook.js +++ b/container/webhook/root/webhook.js @@ -17,4 +17,4 @@ http.createServer(function(req, res){ res.statusCode = 404 res.end('no such location') }) -}).listen(11301) +}).listen(20005) -- 2.23.0

1 0

[PATCH lkp-tests] pkg/mysql/PKGBUILD: fix md5sums
by Lin Jiaxin 19 Apr '21

19 Apr '21

reference: commit bc02d3fe0c13cf85d561a26fdef8dd04c8ede4aa Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- pkg/mysql/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mysql/PKGBUILD b/pkg/mysql/PKGBUILD index 304f6a4a6..c13f07d67 100644 --- a/pkg/mysql/PKGBUILD +++ b/pkg/mysql/PKGBUILD @@ -9,7 +9,7 @@ source=("https://cdn.mysql.com/archives/mysql-8.0/mysql-boost-$pkgver.tar.gz" "https://mirrors.huaweicloud.com/kunpeng/archive/openEuler/patch/database/my…" "https://github.com/thkukuk/rpcsvc-proto/releases/download/v1.4/rpcsvc-proto…" "my.cnf") -md5sums=('7472a25d91973cbba5d8a8f176a3080b' '6e853850efbd6f887ac3e13b1bf99cc4' 'f13eb11de53616017f7428cc06d4e242' 'df1022ccc5d0aff30b8e505735b4a98b' '1390631a1ea1832f9c27d07fcc1e536e') +md5sums=('7472a25d91973cbba5d8a8f176a3080b' '6e853850efbd6f887ac3e13b1bf99cc4' 'f13eb11de53616017f7428cc06d4e242' 'df1022ccc5d0aff30b8e505735b4a98b' 'd90ba2c701507eeac43b06fcfa49c2c1') prepare() { -- 2.23.0

1 0

[PATCH compass-ci] es_query.rb: fix _scroll_id change
by Cui Yili 19 Apr '21

19 Apr '21

[why] _scroll_id will change every time query, so you need to save the last _scroll_id. Signed-off-by: Cui Yili <2268260388(a)qq.com> --- lib/es_query.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 22d68a9..6bd39bf 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -47,11 +47,12 @@ class ESQuery }, size: size } result = @client.search index: @index, scroll: '10m', body: query - @scroll_id = result['_scroll_id'] - return result else - @client.scroll scroll: '10m', scroll_id: @scroll_id + result = @client.scroll scroll: '10m', scroll_id: @scroll_id end + + @scroll_id = result['_scroll_id'] unless result.empty? + return result end def query_by_id(id) -- 2.23.0

2 1

[PATCH compass-ci] sbin/auto_submit: add author_email and committer_email
by Li Yuanchao 19 Apr '21

19 Apr '21

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbin/auto_submit b/sbin/auto_submit index faebb2f..4adce89 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -55,6 +55,9 @@ class AutoSubmit real_argvs.push("upstream_dir=#{belong}") tag = %x(git -C #{mirror_dir} tag --points-at #{commit_id}) real_argvs.push("upstream_tag=#{tag}") unless tag.empty? + author_email = %x(git -C #{mirror_dir} log --format=%ae -1) + committer_email = %x(git -C #{mirror_dir} log --format=%ce -1) + real_argvs.push("author_email=#{author_email} committer_email=#{committer_email}") system(real_argvs.join(' ')) end -- 2.23.0

1 0