- Compass-ci - mailweb.openeuler.org

[PATCH lkp-tests] sbin/makepkg: cpio add --quiet to do not print the number of blocks copied
by Lin Jiaxin 18 Mar '21

18 Mar '21

avoid unnecessary stderr like this: ==> /tmp/stderr <== 43 blocks Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- sbin/makepkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/makepkg b/sbin/makepkg index cb051641..9352557a 100755 --- a/sbin/makepkg +++ b/sbin/makepkg @@ -2273,7 +2273,7 @@ extend_pkgver_cgz_name() { create_cpio() { find . | grep -Fxv -e ./bin -e ./sbin -e ./lib -e ./lib64 -e ./etc/init.d | - cpio -o -H newc --owner=root.root | + cpio --quiet -o -H newc --owner=root.root | gzip -n -9 } -- 2.23.0

1 0

[PATCH compass-ci 3/3] providers/qemu/pxe.sh: added QEMU multiple network card functionality
by Wang Chenglong 18 Mar '21

18 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- providers/qemu/pxe.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/qemu/pxe.sh b/providers/qemu/pxe.sh index 3ab3c2b..b6c4d6d 100755 --- a/providers/qemu/pxe.sh +++ b/providers/qemu/pxe.sh @@ -31,7 +31,7 @@ kvm=( -device virtio-gpu-pci -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd -rtc base=localtime - -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} + -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac[0]} -k en-us -no-reboot -nographic -- 2.23.0

1 1

[PATCH compass-ci 2/3] providers/qemu/kvm.sh: added QEMU multiple network card functionality
by Wang Chenglong 18 Mar '21

18 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- providers/qemu/kvm.sh | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index dcab81a..3b220ef 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -23,8 +23,9 @@ check_logfile() write_logfile() { ipxe_script=ipxe_script - curl http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/boot.ipxe/mac/${mac} > $ipxe_script + curl http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/boot.ipxe/mac/${mac_arr[0]} > $ipxe_script cat $ipxe_script >> ${log_file} + cat $ipxe_script } parse_ipxe_script() @@ -121,12 +122,17 @@ add_disk() set_nic() { - nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" +# nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" + + for i in $(seq 0 $nr_nic) + do + nic[$i]="-nic tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac_arr[$i]}" + done } set_device() { - device="virtio-net-device,netdev=net0,mac=${mac}" + device="virtio-net-device,netdev=net0,mac=${mac[0]}" } set_netdev() @@ -190,7 +196,7 @@ individual_option() -machine virt-4.0,accel=kvm,gic-version=3 -cpu Kunpeng-920 -bios $bios - -nic $nic + ${nic[@]} ) ;; qemu-kvm) @@ -198,17 +204,17 @@ individual_option() -machine virt-4.0,accel=kvm,gic-version=3 -cpu Kunpeng-920 -bios $bios - -nic $nic + ${nic[@]} ) [ "$(arch)" == "x86_64" ] && arch_option=( -bios $bios - -nic $nic + ${nic[@]} ) ;; qemu-system-x86_64) arch_option=( -bios $bios - -nic $nic + ${nic[@]} ) ;; qemu-system-riscv64) -- 2.23.0

1 1

[PATCH compass-ci 1/3] providers/qemu.sh: added QEMU multiple network card functionality
by Wang Chenglong 18 Mar '21

18 Mar '21

[How] Use the 'nr_nic' to set the number of network cards in yaml. nr_nic: 2 root@vm-2p8g ~# ifconfig enp0s1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet 172.18.115.162 netmask 255.255.0.0 broadcast 172.18.255.255 inet6 fe80::835:ffff:fedf:384e prefixlen 64 scopeid 0x20<link> ether 0a:35:ff:df:38:4e txqueuelen 1000 (Ethernet) RX packets 586 bytes 180087 (175.8 KiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 80 bytes 12579 (12.2 KiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 enp0s2: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500 inet6 fe80::868:97ff:fe92:59bb prefixlen 64 scopeid 0x20<link> ether 0a:68:97:92:59:bb txqueuelen 1000 (Ethernet) RX packets 527 bytes 173034 (168.9 KiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 20 bytes 4760 (4.6 KiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- providers/qemu.sh | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/providers/qemu.sh b/providers/qemu.sh index 2fa5771..24211bc 100755 --- a/providers/qemu.sh +++ b/providers/qemu.sh @@ -2,6 +2,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # - hostname +# - nr_nic . $LKP_SRC/lib/yaml.sh . $CCI_SRC/container/defconfig.sh @@ -10,26 +11,40 @@ load_cci_defaults : ${hostname:="vm-1p1g-1"} : ${queues:="vm-1p1g.$(arch)"} +: ${nr_nic:=1} # unicast prefix: x2, x6, xA, xE -export mac=$(echo $hostname | md5sum | sed 's/^$..$$..$$..$$..$$..$.*$/0a-\1-\2-\3-\4-\5/') + +set_mac() +{ + export nr_nic=$(expr $nr_nic - 1) + for i in $(seq 0 $nr_nic) + do + mac=$(echo $hostname$i | md5sum | sed 's/^$..$$..$$..$$..$$..$.*$/0a-\1-\2-\3-\4-\5/') + mac_arr[$i]=$mac + done +} +set_mac + + echo hostname: $hostname -echo mac: $mac -echo $mac > mac -echo "arp -n | grep ${mac//-/:}" > ip.sh +echo mac: ${mac_arr[*]} +echo ${mac_arr[*]} > mac +echo "arp -n | grep ${mac_arr[0]//-/:}" > ip.sh chmod +x ip.sh + set_host_info() { # use "," replace " " local api_queues=$(echo $queues | sed -r 's/ +/,/g') - curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/set_host_mac?hostname=${hostname}&mac=${mac}" + curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/set_host_mac?hostname=${hostname}&mac=${mac_arr[0]}" curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/set_host2queues?host=${hostname}&queues=${api_queues}" } set_host_info del_host_info() { - curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/del_host_mac?mac=${mac}" > /dev/null 2>&1 + curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/del_host_mac?mac=${mac_arr[0]}" > /dev/null 2>&1 curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/del_host2queues?host=${hostname}" > /dev/null 2>&1 } -- 2.23.0

1 1

[PATCH compass-ci] container/mail-robot: update new port
by Luan Shengde 18 Mar '21

18 Mar '21

assigned new port for service: assign-account: 29999 => 10000 send-internet-mail: 49000 => 10001 Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/apply-account.rb | 2 +- container/mail-robot/lib/apply-jumper-account.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/container/mail-robot/lib/apply-account.rb b/container/mail-robot/lib/apply-account.rb index 17705ec..1de250b 100755 --- a/container/mail-robot/lib/apply-account.rb +++ b/container/mail-robot/lib/apply-account.rb @@ -12,7 +12,7 @@ require_relative 'assign-account-fail-eamil' require_relative 'apply-jumper-account' require_relative 'parse-apply-account-email' -SEND_MAIL_PORT = ENV['SEND_MAIL_PORT'] || 49000 +SEND_MAIL_PORT = ENV['SEND_MAIL_PORT'] || 10001 # assign uuid/account for user # when mail-robot listened new email, and the email's subject diff --git a/container/mail-robot/lib/apply-jumper-account.rb b/container/mail-robot/lib/apply-jumper-account.rb index 73d2946..9043309 100755 --- a/container/mail-robot/lib/apply-jumper-account.rb +++ b/container/mail-robot/lib/apply-jumper-account.rb @@ -6,7 +6,7 @@ require 'json' JUMPER_HOST = ENV['JUMPER_HOST'] || 'api.compass-ci.openeuler.org' -JUMPER_PORT = ENV['JUMPER_PORT'] || 29999 +JUMPER_PORT = ENV['JUMPER_PORT'] || 10000 # used to apply account # be called after AssignAccount successfully parsed my_commit_url and my_ssh_pubkey -- 2.23.0

2 1

[PATCH compass-ci] sparrow: add /srv/rabbitmq for rabbitmq container
by Cao Xueliang 18 Mar '21

18 Mar '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- sparrow/1-storage/permission | 2 ++ sparrow/1-storage/tiny | 1 + 2 files changed, 3 insertions(+) diff --git a/sparrow/1-storage/permission b/sparrow/1-storage/permission index 0713d7e..26086d5 100755 --- a/sparrow/1-storage/permission +++ b/sparrow/1-storage/permission @@ -10,6 +10,7 @@ chmod_dir=( /srv/cci/libvirt-xml /srv/cci/serial/fluentd-pos /srv/cci/Maildir + /srv/rabbitmq ) chmod 775 "${chmod_dir[@]}" @@ -22,6 +23,7 @@ chown_dir=( /srv/cache/netdata_lib /srv/cci/serial/fluentd-pos /srv/cci/libvirt-xml + /srv/rabbitmq ) chown lkp:lkp "${chown_dir[@]}" diff --git a/sparrow/1-storage/tiny b/sparrow/1-storage/tiny index 8fd3652..9939d93 100755 --- a/sparrow/1-storage/tiny +++ b/sparrow/1-storage/tiny @@ -24,6 +24,7 @@ dirs=( /srv/cci/Maildir/.compass-ci/new /srv/cci/Maildir/.compass-ci/cur /srv/cci/Maildir/.compass-ci/sent + /srv/rabbitmq ) mkdir -p "${dirs[@]}" -- 2.23.0

1 0

[PATCH compass-ci] sparrow/4-docker/buildall: fix spelling error
by Liu Yinsi 18 Mar '21

18 Mar '21

before: initrd-lkp after: lkp-initrd Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/4-docker/buildall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparrow/4-docker/buildall b/sparrow/4-docker/buildall index a7ace65..3cd5719 100755 --- a/sparrow/4-docker/buildall +++ b/sparrow/4-docker/buildall @@ -62,7 +62,7 @@ do_one_run() } [ -x first-run ] && ./first-run [ -x start ] && ./start - [ "$container_name" == 'initrd-lkp' ] && ./run + [ "$container_name" == 'lkp-initrd' ] && ./run ) } -- 2.23.0

1 0

[PATCH v5 lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 18 Mar '21

18 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 79 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100755 tests/ansible_test diff --git a/tests/ansible_test b/tests/ansible_test new file mode 100755 index 000000000..56926bbed --- /dev/null +++ b/tests/ansible_test @@ -0,0 +1,79 @@ +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] || { + echo "Ansible failed to install." + exit 1 + } + } +} + +download_ansible_role() +{ + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git + if [ $? -eq 0 ]; then + echo "$benchmark is successfully cloned" + else + echo "Please check whether your ansible_repo is correct!" + exit 1 + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: localhost + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook is running failed!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

1 0

[PATCH v2 lkp-tests] tests/openeuler_docker: fix to build openeuler image error
by Liu Shaofei 18 Mar '21

18 Mar '21

^[[91m+ rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 ^[[0m^[[91merror: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7: import read failed(2). ^[[0m Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- tests/openeuler_docker | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/openeuler_docker b/tests/openeuler_docker index b0b5336a3..483f40135 100755 --- a/tests/openeuler_docker +++ b/tests/openeuler_docker @@ -78,6 +78,11 @@ handle_epelrpm() } } +handle_rpm_gpg() +{ + sed -i "s|RPM-GPG-KEY-CentOS-7|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" +} + add_base_commands() { # grep -qw "groupadd*" "$file" && { @@ -95,6 +100,7 @@ handle_dockerfile() is_openeuler_build_fail=1 handle_FROM handle_epel + handle_rpm_gpg add_base_commands if docker_build "-openeuler"; then docker_run "-openeuler" -- 2.23.0

1 0

[PATCH lkp-tests] tests/openeuler_docker: fix to build openeuler image error
by Liu Shaofei 18 Mar '21

18 Mar '21

^[[91m+ rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 ^[[0m^[[91merror: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7: import read failed(2). ^[[0m Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- tests/openeuler_docker | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/openeuler_docker b/tests/openeuler_docker index b0b5336a3..139b5eb56 100755 --- a/tests/openeuler_docker +++ b/tests/openeuler_docker @@ -78,6 +78,13 @@ handle_epelrpm() } } +handle_rpm_gpg() +{ + grep -qw "RPM-GPG-KEY-CentOS-7" "$dockerfile_name" && { + sed -i "s|RPM-GPG-KEY-CentOS-7|PM-GPG-KEY-openEuler|g" "$dockerfile_name" + } +} + add_base_commands() { # grep -qw "groupadd*" "$file" && { @@ -95,6 +102,7 @@ handle_dockerfile() is_openeuler_build_fail=1 handle_FROM handle_epel + handle_rpm_gpg add_base_commands if docker_build "-openeuler"; then docker_run "-openeuler" -- 2.23.0

3 4

[PATCH compass-ci] fix network problem: add domestic resource url for Debian packages
by Luan Shengde 18 Mar '21

18 Mar '21

[why] the default url for Debian package resources may lead to download timeout problems: E: Release file for http://security.debian.org/debian-security/dists/buster/updates/InRelease is expired (invalid since 9h 8min 10s). Updates for this repository will not be applied. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/Dockerfile | 2 ++ container/mail-robot/conf/sources.list | 9 +++++++++ container/send-internet-mail/Dockerfile | 2 ++ container/send-internet-mail/conf/sources.list | 9 +++++++++ container/send-mail/Dockerfile | 2 ++ container/send-mail/conf/sources.list | 9 +++++++++ 6 files changed, 33 insertions(+) create mode 100644 container/mail-robot/conf/sources.list create mode 100644 container/send-internet-mail/conf/sources.list create mode 100644 container/send-mail/conf/sources.list diff --git a/container/mail-robot/Dockerfile b/container/mail-robot/Dockerfile index 4402543..e4b3e7c 100644 --- a/container/mail-robot/Dockerfile +++ b/container/mail-robot/Dockerfile @@ -7,6 +7,8 @@ MAINTAINER Luan Shengde <shdluan(a)163.com> ENV DEBIAN_FRONTEND noninteractive +COPY conf/sources.list* /etc/apt/ + RUN apt-get update && \ apt-get install -y git uuid-runtime ruby-listen ruby-json ruby-mail ruby-rest-client diff --git a/container/mail-robot/conf/sources.list b/container/mail-robot/conf/sources.list new file mode 100644 index 0000000..8179e01 --- /dev/null +++ b/container/mail-robot/conf/sources.list @@ -0,0 +1,9 @@ +deb [trusted=yes] http://mirrors.163.com/debian/ buster main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian/ buster-updates main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian/ buster-backports main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian-security/ buster/updates main non-free contrib + +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster-updates main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster-backports main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian-security/ buster/updates main non-free contrib diff --git a/container/send-internet-mail/Dockerfile b/container/send-internet-mail/Dockerfile index 29f1bc6..fa9ec2c 100644 --- a/container/send-internet-mail/Dockerfile +++ b/container/send-internet-mail/Dockerfile @@ -5,5 +5,7 @@ FROM debian MAINTAINER Luan Shengde <shdluan(a)163.com> ENV DEBIAN_FRONTEND noninteractive +COPY conf/sources.list* /etc/apt/ + RUN apt-get update && \ apt-get install -y ruby-mail ruby-json ruby-sinatra ruby-redis ruby-rufus-scheduler diff --git a/container/send-internet-mail/conf/sources.list b/container/send-internet-mail/conf/sources.list new file mode 100644 index 0000000..8179e01 --- /dev/null +++ b/container/send-internet-mail/conf/sources.list @@ -0,0 +1,9 @@ +deb [trusted=yes] http://mirrors.163.com/debian/ buster main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian/ buster-updates main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian/ buster-backports main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian-security/ buster/updates main non-free contrib + +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster-updates main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster-backports main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian-security/ buster/updates main non-free contrib diff --git a/container/send-mail/Dockerfile b/container/send-mail/Dockerfile index 29f1bc6..fa9ec2c 100644 --- a/container/send-mail/Dockerfile +++ b/container/send-mail/Dockerfile @@ -5,5 +5,7 @@ FROM debian MAINTAINER Luan Shengde <shdluan(a)163.com> ENV DEBIAN_FRONTEND noninteractive +COPY conf/sources.list* /etc/apt/ + RUN apt-get update && \ apt-get install -y ruby-mail ruby-json ruby-sinatra ruby-redis ruby-rufus-scheduler diff --git a/container/send-mail/conf/sources.list b/container/send-mail/conf/sources.list new file mode 100644 index 0000000..8179e01 --- /dev/null +++ b/container/send-mail/conf/sources.list @@ -0,0 +1,9 @@ +deb [trusted=yes] http://mirrors.163.com/debian/ buster main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian/ buster-updates main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian/ buster-backports main non-free contrib +deb [trusted=yes] http://mirrors.163.com/debian-security/ buster/updates main non-free contrib + +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster-updates main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian/ buster-backports main non-free contrib +deb-src [trusted=yes] http://mirrors.163.com/debian-security/ buster/updates main non-free contrib -- 2.23.0

1 0

[PATCH compass-ci] dracut-initrd: add xz-utils
by Wang Chenglong 18 Mar '21

18 Mar '21

[Why] wcl@crystal ~/compass-ci/container/dracut-initrd% ./run /srv/os/centos/x86_64/6.10/lib/modules/3.10.0-957.1.3.el7.x86_64 dracut: Executing: /usr/bin/dracut -o iscsi --omit-drivers iscsi --force --kver 3.10.0-957.1.3.el7.x86_64 -k /lib/modules/3.10.0-957.1.3.el7.x86_64 /boot/initramfs.lkp-3.10.0-957.1.3.el7.x86_64.img dracut: Kernel modules are compressed with xz, but xz is not available. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 616183f..b8eb6f5 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -6,7 +6,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y --fix-missing --no-install-recommends -qq apt-utils \ -nfs-common netbase cifs-utils kmod rsync dracut dracut-network \ +nfs-common netbase cifs-utils kmod rsync dracut dracut-network xz-utils \ dracut-config-generic apt-get clean -- 2.23.0

2 1

[PATCH v3 lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 18 Mar '21

18 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 47 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..d6acad1ab 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,40 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + elsif error_msg.is_a? String + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/[^\w]/, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line.gsub!(/^-|-$/, '') + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

1 0

[PATCH compass-ci] container/crystal-shards: replace https:// with git:// for building image
by Cao Xueliang 17 Mar '21

17 Mar '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/crystal-shards/Dockerfile | 7 +++++-- container/crystal-shards/git | 7 +++++++ 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100755 container/crystal-shards/git diff --git a/container/crystal-shards/Dockerfile b/container/crystal-shards/Dockerfile index 2311554..15fe523 100644 --- a/container/crystal-shards/Dockerfile +++ b/container/crystal-shards/Dockerfile @@ -5,7 +5,10 @@ FROM alpine:crystal-base MAINTAINER chief <taxcom(a)tom.com> -copy shard.yml /usr/share/crystal/app/shard.yml +COPY git /usr/local/sbin/git +RUN echo -e "[url \"git://\"]\n\tinsteadOf = https://" >> /etc/gitconfig + +COPY shard.yml /usr/share/crystal/app/shard.yml WORKDIR /usr/share/crystal/app RUN shards @@ -14,7 +17,7 @@ sed -i '99s/arguments\[:id]/arguments\[:id]?/' /usr/share/crystal/app/lib/elast sed -i 's/, Utils.__listify(arguments\[:q].as(String))/ /' /usr/share/crystal/app/lib/elasticsearch-crystal/src/elasticsearch/api/actions/search.cr;\ sed -i '205a \ params.clear' /usr/share/crystal/app/lib/elasticsearch-crystal/src/elasticsearch/api/actions/search.cr; -copy shard-amqp.yml /usr/share/crystal/app/shard.yml +COPY shard-amqp.yml /usr/share/crystal/app/shard.yml RUN shards CMD ["bash"] diff --git a/container/crystal-shards/git b/container/crystal-shards/git new file mode 100755 index 0000000..7bb0b85 --- /dev/null +++ b/container/crystal-shards/git @@ -0,0 +1,7 @@ +#!/bin/bash + +if [[ "$*" = "ls-remote --get-url origin" ]]; then + /usr/bin/git "$@" | sed 's|git://|https://|g' +else + /usr/bin/git "$@" +fi -- 2.23.0

1 0

[PATCH compass-ci] container/crystal-shards: delete gitcache host and port
by Cao Xueliang 17 Mar '21

17 Mar '21

The gitcache service cannot be used to maintain branches when build crystal-shards images. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/crystal-shards/Dockerfile | 4 ---- container/crystal-shards/build | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/container/crystal-shards/Dockerfile b/container/crystal-shards/Dockerfile index bdc5b66..2311554 100644 --- a/container/crystal-shards/Dockerfile +++ b/container/crystal-shards/Dockerfile @@ -5,10 +5,6 @@ FROM alpine:crystal-base MAINTAINER chief <taxcom(a)tom.com> -ARG GITCACHE_HOST -ARG GITCACHE_PORT -RUN [ -z "$GITCACHE_HOST" ] || echo -e "[url \"http://$GITCACHE_HOST:${GITCACHE_PORT:-5000}/\"]\n\tinsteadOf = https://" >> /etc/gitconfig - copy shard.yml /usr/share/crystal/app/shard.yml WORKDIR /usr/share/crystal/app diff --git a/container/crystal-shards/build b/container/crystal-shards/build index 7c2a712..00153ab 100755 --- a/container/crystal-shards/build +++ b/container/crystal-shards/build @@ -11,4 +11,4 @@ docker_skip_rebuild "alpine:crystal-shards" load_cci_defaults -docker build --build-arg GITCACHE_HOST=$GITCACHE_HOST --build-arg GITCACHE_PORT=$GITCACHE_PORT -t alpine:crystal-shards . +docker build -t alpine:crystal-shards . -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] src/lib/web_backend.rb: implement for group jobs stats count
by Li Yuanchao 17 Mar '21

17 Mar '21

Query jobs by conditions such as group id or suite, and count the number of passed cases and failed cases. output is like: { "kezhiming" : { "all" : $all, "nr_pass" : $nr_pass, "nr_fail" : $nr_fali }, "chenqun" : { "all" : $all, "nr_pass" : $nr_pass, "nr_fail" : $nr_fali }, ... } Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 66 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 89c0f3c..4a5a51f 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -548,3 +548,69 @@ def new_refs_statistics(params) end [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] end + +def single_count(stats) + fail_count = 0 + pass_count = 0 + single_nr_fail = 0 + single_nr_pass = 0 + stats.each do |stat, value| + fail_count += 1 if stat.match(/\.fail$/i) + pass_count += 1 if stat.match(/\.pass$/i) + single_nr_fail = value if stat.match(/\.nr_fail$/i) + single_nr_pass = value if stat.match(/\.nr_pass$/i) + end + fail_count = single_nr_fail.zero? ? fail_count : single_nr_fail + pass_count = single_nr_pass.zero? ? pass_count : single_nr_pass + [fail_count, pass_count, fail_count + pass_count] +end + +def count_stats(job_list) + nr_fail = 0 + nr_pass = 0 + all = 0 + job_list.each do |job| + next unless job['_source']['stats'] + + fail_count, pass_count, all_count = single_count(job['_source']['stats']) + nr_fail += fail_count + nr_pass += pass_count + all += all_count + end + { 'nr_fail' => nr_fail, 'nr_pass' => nr_pass, 'all' => all } +end + +def get_jobs_stats_count(dimension, must, size, from) + dimension_list = get_dimension_list(dimension) + stats_count = {} + dimension_list.each do |dim| + job_list = query_dimension(dimension[0], dim, must, size, from) + stats_count[dim] = count_stats(job_list) + end + stats_count.to_json +end + +def get_stats_by_dimension(conditions, dimension, must, size, from) + must += build_multi_field_subquery_body(conditions) + count_query = { query: { bool: { must: must } } } + total = es_count(count_query) + return {} if total < 1 + + get_jobs_stats_count(dimension, must, size, from) +end + +def get_jobs_stats(params) + dimension, conditions = get_dimension_conditions(params) + must = get_es_must(params) + get_stats_by_dimension(conditions, dimension, must, 1000, 0) +end + +def group_jobs_stats(params) + begin + body = get_jobs_stats(params) + rescue StandardError => e + warn e.message + return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'group jobs table error'] + end + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] +end -- 2.23.0

1 0

[PATCH compass-ci 2/2] src/lib/web_backend.rb: implement for group jobs stats count
by Li Yuanchao 17 Mar '21

17 Mar '21

Query jobs by conditions such as group id or suite, and count the number of passed cases and failed cases. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 66 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 89c0f3c..ea45d59 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -548,3 +548,69 @@ def new_refs_statistics(params) end [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] end + +def single_count(stats) + fail_count = 0 + pass_count = 0 + single_nr_fail = 0 + single_nr_pass = 0 + stats.each do |stat, value| + fail_count += 1 if stat.match(/\.fail$/i) + pass_count += 1 if stat.match(/\.pass$/i) + single_nr_fail = value if stat.match(/\.nr_fail$/i) + single_nr_pass = value if stat.match(/\.nr_pass$/i) + end + fail_count = single_nr_fail.zero? ? fail_count : single_nr_fail + pass_count = single_nr_pass.zero? ? pass_count : single_nr_pass + [fail_count, pass_count, fail_count + pass_count] +end + +def count_stats(job_list) + nr_fail = 0 + nr_pass = 0 + all = 0 + job_list.each do |job| + next unless job['_source']['stats'] + + fail_count, pass_count, all_count = single_count(job['_source']['stats']) + nr_fail += fail_count + nr_pass += pass_count + all += all_count + end + { 'nr_fail' => nr_fail, 'nr_pass' => nr_pass, 'all' => all } +end + +def get_jobs_stats_count(dimension, must, size, from) + dimension_list = get_dimension_list(dimension) + stats_count = {} + dimension_list.each do |dim| + job_list = query_dimension(dimension[0], dim, must, size, from) + stats_count[dim] = count_stats(job_list) + end + stats_count.to_json +end + +def get_stats_by_dimension(conditions, dimension, must, size, from) + must += build_multi_field_subquery_body(conditions) + count_query = { query: { bool: { must: must } } } + total = es_count(count_query) + return {} if total < 1 + + get_jobs_stats_count(dimension, must, size, from) +end + +def get_jobs_stats(params) + dimension, conditions = get_dimension_conditions(params) + must = get_es_must(params) + get_stats_by_dimension(conditions, dimension, must, 1000, 0) +end + +def group_jobs_table(params) + begin + body = get_jobs_stats(params) + rescue StandardError => e + warn e.message + return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'group jobs table error'] + end + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] +end -- 2.23.0

2 2

[PATCH compass-ci 1/2] container/web-backend: add api for group jobs stats count
by Li Yuanchao 17 Mar '21

17 Mar '21

Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/web-backend/web-backend | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 33a3fae..131cff0 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -146,3 +146,28 @@ end get '/get_repo_statistics' do new_refs_statistics(params) end + +# GET /get_group_jobs_table?suite=iperf&dimension=my_name +# must: +# - query_conditions +# - suite / group_id / ... +# - dimension +# - group_id / my_email / my_name +# +# Response like: +# - { +# "kezhiming" : { +# "all" : $all, +# "nr_pass" : $nr_pass, +# "nr_fail" : $nr_fali +# }, +# "chenqun" : { +# "all" : $all, +# "nr_pass" : $nr_pass, +# "nr_fail" : $nr_fali +# }, +# ... +# } +get '/get_group_jobs_table' do + group_jobs_table(params) +end -- 2.23.0

2 2

[PATCH compass-ci] sparrow/3-code/dev-env: set $lab default value
by Liu Yinsi 17 Mar '21

17 Mar '21

before: $lab always is nolab after: if env $lab already exists, use env value. it can make code more flexible. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 6072ce8..e182b65 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -4,6 +4,7 @@ [ -z $server_ip ] && server_ip=$(ip route get 1.2.3.4 | awk '{print $7; exit}') server_name=$(hostname | cut -f1 -d.) +: ${lab:=nolab} : ${sched_host:=$server_ip} : ${sched_port:=3000} : ${ES_HOST:=$server_ip} @@ -38,7 +39,7 @@ mkdir -p /etc/compass-ci/defaults cat > /etc/compass-ci/defaults/$server_name.yaml <<EOF SCHED_HOST: $sched_host SCHED_PORT: $sched_port -lab: nolab +lab: $lab EOF mkdir -p /etc/compass-ci/service -- 2.23.0

1 0

[PATCH lkp-tests] tests/deploy-ci: delete env GITCACHE_HOST and GITCACHE_PORT
by Liu Yinsi 17 Mar '21

17 Mar '21

[why] gitcache service cannot be used to maintain branches when build compass-ci/container/crystal-shards images, so delete. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- tests/deploy-cci | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/deploy-cci b/tests/deploy-cci index 8e6acbac4..6f97d7151 100755 --- a/tests/deploy-cci +++ b/tests/deploy-cci @@ -6,8 +6,6 @@ export_env() { export initrd_http_host=$INITRD_HTTP_HOST export initrd_http_port=$INITRD_HTTP_PORT - export gitcache_host=$GITCACHE_HOST - export gitcache_port=$GITCACHE_PORT } deploy() -- 2.23.0

1 0

[PATCH v2 compass-ci] result-webdav-webdav/nginx.conf: do not overwrite the uploaded config with the same name
by Lin Jiaxin 17 Mar '21

17 Mar '21

Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- container/result-webdav/nginx.conf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/container/result-webdav/nginx.conf b/container/result-webdav/nginx.conf index cc21926..e0e0ee2 100644 --- a/container/result-webdav/nginx.conf +++ b/container/result-webdav/nginx.conf @@ -27,6 +27,22 @@ http { dav_methods PUT MKCOL; dav_access user:rw group:rw all:rw; + set $config_upload_condition "false"; + if ( $uri ~ "^/cci/build-config.*" ) + { + set $config_upload_condition "true"; + } + + if ( -e $request_filename ) + { + set $config_upload_condition "${config_upload_condition}+true"; + } + + if ( $config_upload_condition = "true+true" ) + { + return 200; + } + access_by_lua_block { function split(str, reps) local resultstrlist = {} -- 2.23.0

1 0

[PATCH v2 lkp-tests] rpmbuild for openeuler
by Li Ping 17 Mar '21

17 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- jobs/rpmbuild-pkg.yaml | 2 +- tests/rpmbuild-pkg | 47 +++++++++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/jobs/rpmbuild-pkg.yaml b/jobs/rpmbuild-pkg.yaml index 53e145b3b..a70590afe 100644 --- a/jobs/rpmbuild-pkg.yaml +++ b/jobs/rpmbuild-pkg.yaml @@ -1,5 +1,5 @@ suite: rpmbuild-pkg category: functional -rpm_repo: +upstream_repo: rpmbuild-pkg: diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index b641e97d1..86b775334 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -1,12 +1,16 @@ #!/bin/bash -# - rpm_repo +# - upstream_repo +# - compat_os . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh -[ -n "$rpm_repo" ] || die "rpm_repo is empty" -package_name=${rpm_repo##*/} -sync_dest="/initrd/rpmbuild-pkg/${os}/${os_arch}/${os_version}/${package_name}" +: "${compat_os:=bud-openeuler}" + +[ -n "$upstream_repo" ] || die "upstream_repo is empty" +package_name=${upstream_repo##*/} +rpm_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compat_os}/${os_arch}/Packages" +src_rpm_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compat_os}/source/Packages" init_workspace() { @@ -14,21 +18,16 @@ init_workspace() rpmdev-setuptree } -get_pkgfile() +download_upstream_repo() { - curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-show", "HEAD:'$1'"]}' -o "${2}" -} - -download_rpm_repo() -{ - local filelist=$(curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-ls-files", "."]}') - for pkgfile in "${filelist[@]}" + git clone "git://$GIT_SERVER/openeuler/${upstream_repo}" || die "clone git repo ${package_name} failed: git://$GIT_SERVER/openeuler/${upstream_repo}" + cd $package_name || exit + filelist=$(git ls-files) + for pkgfile in ${filelist[@]} do local dir="SOURCES" echo $pkgfile | egrep "\.spec$" && dir="SPECS" - get_pkgfile "$pkgfile" "${HOME}/rpmbuild/${dir}/$pkgfile" + mv "$pkgfile" "${HOME}/rpmbuild/${dir}/" done } @@ -50,19 +49,29 @@ build_rpm() show_rpm_files() { find ${HOME}/rpmbuild/RPMS -type f -name "*.rpm" +} + +show_src_rpm_files() +{ find ${HOME}/rpmbuild/SRPMS -type f -name "*.rpm" } upload_rpm_pkg() { - local file - for file in $(show_rpm_files) + local rpm_file + for rpm_file in $(show_rpm_files) + do + upload_one_curl ${rpm_file} ${rpm_dest} + done + + local src_rpm_file + for src_rpm_file in $(show_src_rpm_files) do - upload_one_curl ${file} ${sync_dest} + upload_one_curl ${src_rpm_file} ${src_rpm_dest} done } init_workspace -download_rpm_repo +download_upstream_repo build_rpm upload_rpm_pkg -- 2.23.0

1 1

[PATCH v3 lkp-tests] rpmbuild for openeuler
by Li Ping 17 Mar '21

17 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- jobs/rpmbuild-pkg.yaml | 2 +- tests/rpmbuild-pkg | 47 +++++++++++++++++++++++++----------------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/jobs/rpmbuild-pkg.yaml b/jobs/rpmbuild-pkg.yaml index 53e145b3b..a70590afe 100644 --- a/jobs/rpmbuild-pkg.yaml +++ b/jobs/rpmbuild-pkg.yaml @@ -1,5 +1,5 @@ suite: rpmbuild-pkg category: functional -rpm_repo: +upstream_repo: rpmbuild-pkg: diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index b641e97d1..256e836bf 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -1,12 +1,16 @@ #!/bin/bash -# - rpm_repo +# - upstream_repo +# - compat_os . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh -[ -n "$rpm_repo" ] || die "rpm_repo is empty" -package_name=${rpm_repo##*/} -sync_dest="/initrd/rpmbuild-pkg/${os}/${os_arch}/${os_version}/${package_name}" +: "${compat_os:=budding-openeuler}" + +[ -n "$upstream_repo" ] || die "upstream_repo is empty" +package_name=${upstream_repo##*/} +rpm_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compat_os}/${os_arch}/Packages" +src_rpm_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compat_os}/source/Packages" init_workspace() { @@ -14,21 +18,16 @@ init_workspace() rpmdev-setuptree } -get_pkgfile() +download_upstream_repo() { - curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-show", "HEAD:'$1'"]}' -o "${2}" -} - -download_rpm_repo() -{ - local filelist=$(curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-ls-files", "."]}') - for pkgfile in "${filelist[@]}" + git clone "git://$GIT_SERVER/openeuler/${upstream_repo}" || die "clone git repo ${package_name} failed: git://$GIT_SERVER/openeuler/${upstream_repo}" + cd $package_name || exit + filelist=$(git ls-files) + for pkgfile in ${filelist[@]} do local dir="SOURCES" echo $pkgfile | egrep "\.spec$" && dir="SPECS" - get_pkgfile "$pkgfile" "${HOME}/rpmbuild/${dir}/$pkgfile" + mv "$pkgfile" "${HOME}/rpmbuild/${dir}/" done } @@ -50,19 +49,29 @@ build_rpm() show_rpm_files() { find ${HOME}/rpmbuild/RPMS -type f -name "*.rpm" +} + +show_src_rpm_files() +{ find ${HOME}/rpmbuild/SRPMS -type f -name "*.rpm" } upload_rpm_pkg() { - local file - for file in $(show_rpm_files) + local rpm_file + for rpm_file in $(show_rpm_files) + do + upload_one_curl ${rpm_file} ${rpm_dest} + done + + local src_rpm_file + for src_rpm_file in $(show_src_rpm_files) do - upload_one_curl ${file} ${sync_dest} + upload_one_curl ${src_rpm_file} ${src_rpm_dest} done } init_workspace -download_rpm_repo +download_upstream_repo build_rpm upload_rpm_pkg -- 2.23.0

1 0

[PATCH lkp-tests] add new error type
by Bai Jing 17 Mar '21

17 Mar '21

[input] Error: Module or Group 'convert' does not exist [output] error.nor-exit-module-or-group.convert: 1 Signed-off-by: Bai Jing <799286817(a)qq.com> --- stats/openeuler_docker | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stats/openeuler_docker b/stats/openeuler_docker index 4c65c6bb3..97cf24d7f 100755 --- a/stats/openeuler_docker +++ b/stats/openeuler_docker @@ -18,6 +18,11 @@ while (line = STDIN.gets) puts "unknown-repo.#{repo}: 1" puts "unknown-repo.#{repo}.message: #{line}" + # Error: Module or Group 'convert' does not exist. + when /Error: Module or Group ('[^\s]+')/ + repo = $1.delete!("'") + puts "error.not-exist-module-or-group.#{repo}: 1" + puts "error.not-exist-module-or-group.#{repo}.message: #{line}" # /bin/sh: passwd: command not found when /\/bin\/sh: (.+): command not found/ puts "sh.command-not-found.#{$1}: 1" -- 2.23.0

1 0

[PATCH compass-ci] container/logging-es: remove redundant code
by Liu Yinsi 17 Mar '21

17 Mar '21

[why] these deleted code optimize the bulk write performance, but affect the real-time performance of data. At now, our fluentd architecture already been optimized, the bulk write is no longer a bottleneck, so delete codes. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/logging-es/start | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/container/logging-es/start b/container/logging-es/start index efd89de..9be7a5b 100755 --- a/container/logging-es/start +++ b/container/logging-es/start @@ -19,14 +19,3 @@ cmd=( ) "${cmd[@]}" - -# set index -check_service_ready 9202 -curl -sSH "Content-Type: application/json" -XPUT "localhost:9202/_settings" -d ' -{ - "index": { - "refresh_interval": "120s", - "translog.durability": "async", - "translog.sync_interval": "120s", - "translog.flush_threshold_size": "1024mb"} -}' -- 2.23.0

1 0

[PATCH compass-ci] dracut-initrd: add xz-utils
by Wang Chenglong 17 Mar '21

17 Mar '21

[Why] wcl@crystal ~/compass-ci/container/dracut-initrd% ./run /srv/os/centos/x86_64/6.10/lib/modules/3.10.0-957.1.3.el7.x86_64 dracut: Executing: /usr/bin/dracut -o iscsi --omit-drivers iscsi --force --kver 3.10.0-957.1.3.el7.x86_64 -k /lib/modules/3.10.0-957.1.3.el7.x86_64 /boot/initramfs.lkp-3.10.0-957.1.3.el7.x86_64.img dracut: Kernel modules are compressed with xz, but xz is not available. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 616183f..b8eb6f5 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -6,7 +6,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get update apt-get install -y --fix-missing --no-install-recommends -qq apt-utils \ -nfs-common netbase cifs-utils kmod rsync dracut dracut-network \ +nfs-common netbase cifs-utils kmod rsync dracut dracut-network xz-utils \ dracut-config-generic apt-get clean -- 2.23.0

1 0

[PATCH v2 lkp-tests] jobs: add secrets_info.yaml
by Wei Jihui 17 Mar '21

17 Mar '21

secrets_info.yaml is for user use secret field Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- jobs/secrets_info.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 jobs/secrets_info.yaml diff --git a/jobs/secrets_info.yaml b/jobs/secrets_info.yaml new file mode 100644 index 00000000..4026b3b7 --- /dev/null +++ b/jobs/secrets_info.yaml @@ -0,0 +1,6 @@ +# This file is for user add secret field, so that it's not visible from outside(job.yaml), but can be used at testbox. +# Usage: submit job.yaml -i secrets_info.yaml +# Use at testbox: testbox has env with: "secrets_" + secret field, such as $secrets_my_token + +secrets: + my_token: {{my_token}} -- 2.23.0

1 0

[PATCH lkp-tests] jobs: add secrets_info.yaml
by Wei Jihui 17 Mar '21

17 Mar '21

secrets_info.yaml is for user use secret field Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- jobs/secrets_info.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 jobs/secrets_info.yaml diff --git a/jobs/secrets_info.yaml b/jobs/secrets_info.yaml new file mode 100644 index 00000000..196cf5ed --- /dev/null +++ b/jobs/secrets_info.yaml @@ -0,0 +1,6 @@ +# This file is for user add secret field, so that it;s not visible from outside, but can be use at testbox. +# Usage: submit job.yaml -i secrets_info.yaml +# Use at testbox: testbox has env with: "secrets_" + secret field, such as $secrets_my_token + +secrets: + my_token: {{my_token}} -- 2.23.0

2 1

[PATCH compass-ci] sbin/es-jobs: add -e to get fail result
by Wu Zhende 17 Mar '21

17 Mar '21

es-jobs -e group_id=xxxx error.result.path=/srv/result/openeuler_docker/xxxx: ["Error: unable to find a match"] Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/es_jobs.rb | 24 +++++++++++++++++++++++- sbin/es-jobs | 7 ++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index b0d0fb2..e4e9530 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -12,13 +12,14 @@ require_relative './es_query' # deal jobs search from es class ESJobs - def initialize(es_query, my_refine = [], fields = [], stats_filter = []) + def initialize(es_query, my_refine = [], fields = [], stats_filter = [], error_result = false) @es_query = es_query @jobs = query_jobs_from_es @refine = my_refine @fields = fields @stats_filter = stats_filter @refine_jobs = [] + @error_result = error_result set_jobs_summary end @@ -98,6 +99,7 @@ class ESJobs 'max.stats' => {}, 'min.stats' => {} } + result['error.result'] = {} if @error_result metrics.each { |metric| result['raw.stats'][metric] = [] } result end @@ -119,7 +121,27 @@ class ESJobs result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 result['raw.id']["[#{keyword}=#{job[keyword]}]"] ||= [] result['raw.id']["[#{keyword}=#{job[keyword]}]"] << job['id'] + + next unless @error_result && job[keyword] == "failed" + + get_failed_result(result, job) + end + end + + def get_failed_result(result, job) + path = "/srv#{job['result_root']}/#{job['suite']}" + res = find_error_lines(path) + result['error.result']["path=#{path}"] = res + end + + def find_error_lines(path) + res = [] + return res unless File.exist?(path) + + File.open(path).each do |line| + res << line.to_s if line.match(/error:/i) end + res end def stats_count(result) diff --git a/sbin/es-jobs b/sbin/es-jobs index ff0d540..3804dc9 100755 --- a/sbin/es-jobs +++ b/sbin/es-jobs @@ -22,6 +22,7 @@ end opt_refine = [-1] opt_fields = [] opt_stats_filter = [] +opt_error_result = false opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: es-jobs [options] search_key1=val1[,val2..] ..' @@ -45,11 +46,15 @@ opt_parser = OptionParser.new do |opts| opts.on('-s fields', '--stats-filter fields', 'return data contains fields in stats') do |fields| opt_stats_filter = fields.split(',') end + + opts.on('-e', '--error.result', 'obtain the error information about job failure') do + opt_error_result = true + end end opt_parser.parse!(ARGV) items = parse_argv raise 'Please enter a query' if items.empty? -es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter) +es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter, opt_error_result) es_jobs.output -- 2.23.0

1 0

[PATCH compass-ci] container/git-mirror: change mount dir
by Li Yuanchao 17 Mar '21

17 Mar '21

As several repos are supported as upstream-repos, to avoid confusion, change the dir name of /c/upstream-repos to /c/git-repos. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/git-mirror/start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/git-mirror/start b/container/git-mirror/start index ec42595..f01789c 100755 --- a/container/git-mirror/start +++ b/container/git-mirror/start @@ -6,7 +6,7 @@ require_relative '../defconfig.rb' DOCKER_CCI = '/c/compass-ci' -DOCKER_REPO = '/c/upstream-repos' +DOCKER_REPO = '/c/git-repos' docker_rm "git-mirror" lkp = %x(id -u lkp) -- 2.23.0

1 0

[PATCH compass-ci] container/web-backend|-nginx: update SRV_HTTP_RESULT_PORT
by Lu Weitao 17 Mar '21

17 Mar '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend-nginx/nginx.conf | 2 +- container/web-backend/start | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/container/web-backend-nginx/nginx.conf b/container/web-backend-nginx/nginx.conf index 381888a..a3313fb 100644 --- a/container/web-backend-nginx/nginx.conf +++ b/container/web-backend-nginx/nginx.conf @@ -29,7 +29,7 @@ http { proxy_set_header X-Forward-For $proxy_add_x_forwarded_for; # srv-http server - proxy_pass http://172.17.0.1:11300; + proxy_pass http://172.17.0.1:20007; } } } diff --git a/container/web-backend/start b/container/web-backend/start index e680255..cebf949 100755 --- a/container/web-backend/start +++ b/container/web-backend/start @@ -11,7 +11,7 @@ docker_rm web-backend load_cci_defaults [[ $SRV_HTTP_RESULT_HOST ]] || SRV_HTTP_RESULT_HOST=$(curl ip.sb) -[[ $SRV_HTTP_RESULT_PORT ]] || SRV_HTTP_RESULT_PORT=11300 +[[ $SRV_HTTP_RESULT_PORT ]] || SRV_HTTP_RESULT_PORT=20007 cmd=( docker run -- 2.23.0

1 0

[PATCH v2 compass-ci] container/web-backend: add API /get_table_error
by Lu Weitao 17 Mar '21

17 Mar '21

[Example] curl -X GET 'localhost:30000/get_job_error?group_id=wcl_ansible-openeuler-03-10' return: { "filter":{"group_id":"wcl_ansible-openeuler-03-10"}, "table_fields":["job_id","error_id","error_message","link to result"], "data_table":[ [ "crystal.1354921", "ansible_test.error.Unable-to-start-service-httpd-Job-for-httpdservice-xxx.fail", "{\"changed\": false, \"msg\": \"Unable to start service httpd: Job for httpd.service failed because a timeout was exceeded.}", "http://172.17.0.1:11300/result/ansible_test/2021-03-10/vm-2p16g--wcl1/opene… ate/crystal.1354921" ], [ "crystal.1354929", ... ] ... ] } Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend/web-backend | 19 +++++++++ src/lib/web_backend.rb | 68 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 74eda85..fa2a53d 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -146,3 +146,22 @@ end get '/get_repo_statistics' do new_refs_statistics(params) end + +# GET /get_job_error?suite=virttest&tbox_group=vm-2p8g +# Response +# { +# "filter":{"group_id":"wcl_ansible-openeuler-03-10"}, +# "table_fields":["job_id","error_id","error_message","link to result"], +# "data_table":[ +# [ +# "crystal.1354921", +# "ansible_test.error.Unable-to-start-service-httpd-Job-for-httpdservice-failed.message", +# "{\"changed\": false, \"msg\": \"Unable to start service httpd: Job for httpd.service failed because a timeout was exceeded.} +# "http://172.17.0.1:11300/result/ansible_test/2021-03-10/vm-2p16g--wcl1/opene…" +# ], +# ... +# ] +# } +get '/get_job_error' do + get_job_error(params) +end diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 89c0f3c..48b7ef2 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -12,6 +12,7 @@ require "#{CCI_SRC}/lib/compare.rb" require "#{CCI_SRC}/lib/constants.rb" require "#{CCI_SRC}/lib/es_query.rb" require "#{CCI_SRC}/lib/matrix2.rb" +require "#{CCI_SRC}/lib/params_group.rb" require "#{CCI_SRC}/lib/compare_data_format.rb" UPSTREAM_REPOS_PATH = ENV['UPSTREAM_REPOS_PATH'] || '/c/upstream-repos' @@ -548,3 +549,70 @@ def new_refs_statistics(params) end [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] end + + +# ------------------------------------------------------------------------------------------- +# job error table like: +# job_id error_id error_message link to result +# ------------------------------------------------------------------------------------- +# crystal.630608 "stderr.xxx" "messag:xxxx" https://$host:$port/$result_root +# ... +# ------------------------------------------------------------------------------------------- + +def get_job_error(params) + begin + body = job_error_body(params) + rescue StandardError => e + warn e.message + return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'get error table error'] + end + + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] +end + +def job_error_body(params) + error_table = get_error_table(params) + { + filter: params, + table_fields: ["job_id", "error_id", "error_message", "link to result"], + data_table: error_table, + }.to_json +end + +def get_error_table(filter_items) + error_table = [] + + job_list = get_job_list(filter_items) + job_list.each do |job| + error_table += get_error_from_job(job) + end + + error_table +end + +def get_job_list(items) + es = ESQuery.new(ES_HOST, ES_PORT) + query_results = es.multi_field_query(items) + + extract_jobs_list(query_results['hits']['hits']) +end + +# get all error_id from one job +def get_error_from_job(job) + job_error = [] + job['stats'].each do |metric, value| + next unless metric.end_with?('.message') + + result_host = ENV['SRV_HTTP_RESULT_HOST'] || SRV_HTTP_RESULT_HOST + result_port = ENV['SRV_HTTP_RESULT_PORT'] || SRV_HTTP_RESULT_PORT + error_id = metric.sub('.message', '.fail') + job_error << [ + job['id'], + error_id, + value, + "http://#{result_host}:#{result_port}#{job['result_root']}" + ] + end + + job_error +end -- 2.23.0

2 2

[PATCH v4 lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 17 Mar '21

17 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 81 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 tests/ansible_test diff --git a/tests/ansible_test b/tests/ansible_test new file mode 100755 index 000000000..bf75b19b7 --- /dev/null +++ b/tests/ansible_test @@ -0,0 +1,81 @@ +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + ansible_host=localhost + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] || { + echo "Ansible failed to install." + exit 1 + } + } +} + +download_ansible_role() +{ + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git + if [ $? -eq 0 ]; then + echo "$benchmark is successfully cloned" + else + echo "Please check whether your ansible_repo is correct!" + exit 1 + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: $ansible_host + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook is running failed!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

1 0

[PATCH lkp-tests] rpmbuild for openeuler
by Li Ping 16 Mar '21

16 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- jobs/rpmbuild-pkg.yaml | 2 +- tests/rpmbuild-pkg | 46 +++++++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/jobs/rpmbuild-pkg.yaml b/jobs/rpmbuild-pkg.yaml index 53e145b3b..a70590afe 100644 --- a/jobs/rpmbuild-pkg.yaml +++ b/jobs/rpmbuild-pkg.yaml @@ -1,5 +1,5 @@ suite: rpmbuild-pkg category: functional -rpm_repo: +upstream_repo: rpmbuild-pkg: diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index b641e97d1..034e4f3e1 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -1,12 +1,16 @@ #!/bin/bash -# - rpm_repo +# - upstream_repo +# - compatible_os . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh -[ -n "$rpm_repo" ] || die "rpm_repo is empty" -package_name=${rpm_repo##*/} -sync_dest="/initrd/rpmbuild-pkg/${os}/${os_arch}/${os_version}/${package_name}" +: "${compatible_os:=compat-openeuler}" + +[ -n "$upstream_repo" ] || die "upstream_repo is empty" +package_name=${upstream_repo##*/} +rpm_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compatible_os}/${os_arch}/Packages" +source_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compatible_os}/source/Packages" init_workspace() { @@ -14,21 +18,16 @@ init_workspace() rpmdev-setuptree } -get_pkgfile() +download_upstream_repo() { - curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-show", "HEAD:'$1'"]}' -o "${2}" -} - -download_rpm_repo() -{ - local filelist=$(curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-ls-files", "."]}') - for pkgfile in "${filelist[@]}" + git clone "git://$GIT_SERVER/openeuler/${upstream_repo}" || die "clone git repo ${package_name} failed: git://$GIT_SERVER/openeuler/${upstream_repo}" + cd $package_name || exit + filelist=$(git ls-files) + for pkgfile in ${filelist[@]} do local dir="SOURCES" echo $pkgfile | egrep "\.spec$" && dir="SPECS" - get_pkgfile "$pkgfile" "${HOME}/rpmbuild/${dir}/$pkgfile" + mv "$pkgfile" "${HOME}/rpmbuild/${dir}/" done } @@ -50,19 +49,28 @@ build_rpm() show_rpm_files() { find ${HOME}/rpmbuild/RPMS -type f -name "*.rpm" +} + +show_src_rpm_files() +{ find ${HOME}/rpmbuild/SRPMS -type f -name "*.rpm" } upload_rpm_pkg() { - local file - for file in $(show_rpm_files) + local rpm_file + for rpm_file in $(show_rpm_files) + do + upload_one_curl ${rpm_file} ${rpm_dest} + done + local src_rpm_file + for src_rpm_file in $(show_src_rpm_files) do - upload_one_curl ${file} ${sync_dest} + upload_one_curl ${src_rpm_file} ${source_dest} done } init_workspace -download_rpm_repo +download_upstream_repo build_rpm upload_rpm_pkg -- 2.23.0

1 0

[PATCH compass-ci] src/scheduler/find_job_boot.cr: merge 2 similar functions
by Xu Xijian 16 Mar '21

16 Mar '21

ipxe_msg() and grub_msg() have the same function, they just diff at boot_type in ret msg. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/scheduler/find_job_boot.cr | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 416f8f4..78b10ad 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -126,16 +126,8 @@ class Sched return boot_content(job, boot_type) end - private def ipxe_msg(msg) - "#!ipxe - echo ... - echo #{msg} - echo ... - reboot" - end - - private def grub_msg(msg) - "#!grub + private def boot_msg(boot_type, msg) + "#!#{boot_type} echo ... echo #{msg} echo ... @@ -220,9 +212,9 @@ class Sched case boot_type when "ipxe" - return job ? get_boot_ipxe(job) : ipxe_msg("No job now") + return job ? get_boot_ipxe(job) : boot_msg(boot_type, "No job now") when "grub" - return job ? get_boot_grub(job) : grub_msg("No job now") + return job ? get_boot_grub(job) : boot_msg(boot_type, "No job now") when "container" return job ? get_boot_container(job) : Hash(String, String).new.to_json when "libvirt" -- 2.23.0

1 0

[PATCH lkp-tests 2/2] tests/monitor-job: refactor monitor-job submit job
by Wei Jihui 16 Mar '21

16 Mar '21

add my-token for submit job, so del $CCI_SRC/sbin/build-my-info Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- tests/monitor-job | 59 ++++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/tests/monitor-job b/tests/monitor-job index 5da992fa..8f8ac682 100755 --- a/tests/monitor-job +++ b/tests/monitor-job @@ -2,60 +2,67 @@ # - monitor_repo # - job_yaml # - job_testbox +# lab +# my_name +# my_email +# my_token . $LKP_SRC/lib/log.sh -set_scheduler_config() +set_submit_config() { - mkdir -p /etc/compass-ci/defaults - cat > /etc/compass-ci/defaults/$lab.yaml <<-EOF + mkdir -p ~/.config/compass-ci/defaults/ + cat > ~/.config/compass-ci/defaults/"$lab".yaml <<-EOF SCHED_HOST: $SCHED_HOST SCHED_PORT: $SCHED_PORT lab: $lab + EOF + + cat > ~/.config/compass-ci/defaults/account.yaml <<-EOF + my_name: $my_name my_email: $my_email EOF -} -install_gem_pkgs() -{ - gem sources -r http://rubygems.org -a https://gems.ruby-china.com/ - gem install git activesupport rest-client faye-websocket + my_token=${my_token:-$secrets_my_token} + mkdir -p ~/.config/compass-ci/include/lab/ + cat > ~/.config/compass-ci/include/lab/"$lab".yaml <<-EOF + my_token: $my_token + EOF } -build_my_info() +install_lkp() { - git clone git://$GIT_SERVER/gitee.com/wu_fengguang/compass-ci.git "$CCI_SRC" - ruby "$CCI_SRC"/sbin/build-my-info.rb -e "$my_email" -n "$my_name" -l "$lab" - git clone git://$GIT_SERVER/gitee.com/wu_fengguang/lkp-tests.git "$LKP_SRC" + cd "$LKP_SRC" && make install } download_monitor_repo() { - if [ -n "$monitor_repo" ]; then - git clone git://"$LKP_SERVER"/"$monitor_repo" - else - log_error "monitor_repo is not specified" && exit 1 - fi - - test_code=${monitor_repo##*/} - cp -r "$test_code"/* $LKP_SRC/ + git clone git://"$GIT_SERVER"/"$monitor_repo" "$repo_path" + cp -r "$repo_path"/* "$LKP_SRC"/ } submit_job() { - yaml=${job_yaml##*/} - "$LKP_SRC"/sbin/submit -a "$yaml" testbox="$job_testbox" + if [ -f "$LKP_SRC"/"$job_yaml" ]; then + job_yaml="$LKP_SRC"/"$job_yaml" + else + job_yaml=${job_yaml##*/} + fi + + "$LKP_SRC"/sbin/submit -a "$job_yaml" testbox="$job_testbox" } main() { - local CCI_SRC=/c/compass-ci + [ -z "$monitor_repo" ] && log_error "monitor_repo is not specified" && exit 1 + + local repo=${monitor_repo##*/} + local repo_path=/c/"$repo" local LKP_SRC=/c/lkp-tests - install_gem_pkgs - set_scheduler_config - build_my_info + set_submit_config + install_lkp download_monitor_repo submit_job } -- 2.23.0

1 0

[PATCH lkp-tests 1/2] distro/depends/monitor-job: add sudo
by Wei Jihui 16 Mar '21

16 Mar '21

monitor-job task need install lkp-tests, so sudo cmd is needed. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- distro/depends/monitor-job | 1 + 1 file changed, 1 insertion(+) diff --git a/distro/depends/monitor-job b/distro/depends/monitor-job index 5bfafa87..cb01a8c8 100644 --- a/distro/depends/monitor-job +++ b/distro/depends/monitor-job @@ -1,3 +1,4 @@ git +sudo ruby-dev build-essential -- 2.23.0

1 0

[PATCH lkp-tests] change service ports
by Wu Zhende 16 Mar '21

16 Mar '21

monitoring: 11310 to 20001 scheduler-https: 11306 to 20002 sshr: 50000-51999 to 21000-23999 Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- daemon/sshd | 2 +- include/lab/z9 | 2 +- lib/monitor.rb | 4 ++-- sbin/monitor | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/daemon/sshd b/daemon/sshd index 7ba471a88..50a7677ab 100755 --- a/daemon/sshd +++ b/daemon/sshd @@ -15,7 +15,7 @@ PROJECT_DIR=$(dirname $SCRIPT_DIR) sshr_ip=$LKP_SERVER [ -n "$sshr_port" ] || sshr_port=5050 [ -n "$sshr_port_len" ] || sshr_port_len=2000 -[ -n "$sshr_port_base" ] || sshr_port_base=50000 +[ -n "$sshr_port_base" ] || sshr_port_base=21000 generate_key_file() { diff --git a/include/lab/z9 b/include/lab/z9 index 0592fdb6a..68c7aafc1 100644 --- a/include/lab/z9 +++ b/include/lab/z9 @@ -1,2 +1,2 @@ SCHED_HOST: api.compass-ci.openeuler.org -SCHED_PORT: 11306 +SCHED_PORT: 20002 diff --git a/lib/monitor.rb b/lib/monitor.rb index 24191fa8d..b08fdf4c9 100755 --- a/lib/monitor.rb +++ b/lib/monitor.rb @@ -34,7 +34,7 @@ class Monitor return unless @monitor_url == '' if host = @job['SCHED_HOST'] - return @monitor_url = "ws://#{host}:11310/filter" + return @monitor_url = "ws://#{host}:20001/filter" end Dir.glob(['/etc/compass-ci/monitor/*.yaml', @@ -49,7 +49,7 @@ class Monitor revise_hash(@defaults, defaults, true) end - @monitor_url = @defaults['monitor_url'] || 'ws://localhost:11310/filter' + @monitor_url = @defaults['monitor_url'] || 'ws://localhost:20001/filter' end def merge_overrides diff --git a/sbin/monitor b/sbin/monitor index a2e677fc0..f3ec73469 100755 --- a/sbin/monitor +++ b/sbin/monitor @@ -26,7 +26,7 @@ options = OptionParser.new do |opts| opts.separator 'options:' opts.on('-u URL', '--url URL', - 'set Message Server API address, default is ws://localhost:11310/filter') do |monitor_url| + 'set Message Server API address, default is ws://localhost:20001/filter') do |monitor_url| opt_monitor_url = monitor_url end -- 2.23.0

1 0

[PATCH compass-ci] service: change port
by Wu Zhende 16 Mar '21

16 Mar '21

kibana: 11309 to 20000 monitoring: 11310 to 20001 scheduler-https: 11306 to 20002 sshr: 50000-51999 to 21000-23999 Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana/start | 2 +- container/monitoring/build | 2 +- container/monitoring/start | 2 +- container/scheduler-https/start | 4 ++-- container/ssh-r/Dockerfile | 2 +- container/ssh-r/start | 2 +- doc/manual/submit-job.en.md | 4 ++-- doc/manual/submit-job.zh.md | 4 ++-- src/delimiter/constants.rb | 2 +- src/monitoring/constants.cr | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/container/kibana/start b/container/kibana/start index dbb3c56..26a4944 100755 --- a/container/kibana/start +++ b/container/kibana/start @@ -24,7 +24,7 @@ cmd = %W[ -v /etc/localtime:/etc/localtime:ro -d -e ELASTICSEARCH_HOSTS=http://#{LOGGING_ES_HOST}:#{LOGGING_ES_PORT} - -p 11309:5601 + -p 20000:5601 kibana:7.6.2 ] diff --git a/container/monitoring/build b/container/monitoring/build index 121ede0..4865c70 100755 --- a/container/monitoring/build +++ b/container/monitoring/build @@ -9,7 +9,7 @@ require_relative '../defconfig.rb' names = Set.new ['MONITOR_PORT'] defaults = relevant_defaults(names) -MONITOR_PORT = (defaults['MONITOR_PORT'] || '11310') +MONITOR_PORT = (defaults['MONITOR_PORT'] || '20001') docker_skip_rebuild "monitoring-#{MONITOR_PORT}" diff --git a/container/monitoring/start b/container/monitoring/start index 2e2946c..f4214ed 100755 --- a/container/monitoring/start +++ b/container/monitoring/start @@ -17,7 +17,7 @@ env = docker_env(defaults) DEFAULT_CCI = '/c/cci' DEFAULT_LKP = '/c/lkp-tests' -MONITOR_PORT = defaults['MONITOR_PORT'] || '11310' +MONITOR_PORT = defaults['MONITOR_PORT'] || '20001' docker_rm "monitoring-#{MONITOR_PORT}" cmd = %W[ diff --git a/container/scheduler-https/start b/container/scheduler-https/start index 98940d7..ee93e79 100755 --- a/container/scheduler-https/start +++ b/container/scheduler-https/start @@ -39,12 +39,12 @@ names = Set.new %w[ defaults = relevant_defaults(names) defaults['SSHR_PORT'] ||= 5050 -defaults['SSHR_PORT_BASE'] ||= 50000 +defaults['SSHR_PORT_BASE'] ||= 21000 defaults['SSHR_PORT_LEN'] ||= 2000 defaults['SCHED_PORT'] ||= '3000' defaults['SCHED_HOST'] ||= '172.17.0.1' defaults['MASTER_FLUENTD_PORT'] ||= '24224' -defaults['SCHED_HTTPS_PORT'] ||= 11306 +defaults['SCHED_HTTPS_PORT'] ||= 20002 defaults['SCHED_HTTPS_HOST'] ||= 'api.compass-ci.openeuler.org' env = docker_env(defaults) diff --git a/container/ssh-r/Dockerfile b/container/ssh-r/Dockerfile index 65a13aa..416e0ef 100644 --- a/container/ssh-r/Dockerfile +++ b/container/ssh-r/Dockerfile @@ -21,7 +21,7 @@ RUN chown -R sshr:sshr /home/sshr/.ssh/authorized_keys \ && chmod 600 /home/sshr/.ssh/authorized_keys \ && passwd -u sshr -EXPOSE 50000-51999 +EXPOSE 21000-23999 EXPOSE 22 COPY entry.sh /entry.sh diff --git a/container/ssh-r/start b/container/ssh-r/start index 5c800f3..6c7a373 100755 --- a/container/ssh-r/start +++ b/container/ssh-r/start @@ -15,7 +15,7 @@ cmd=( -e GATEWAY_PORTS=true -e TCP_FORWARDING=true -d - -p 50000-51999:50000-51999 + -p 21000-23999:21000-23999 -p 5050:22 -v /etc/localtime:/etc/localtime:ro -v /srv/pub/sshr/keys/:/etc/ssh/keys diff --git a/doc/manual/submit-job.en.md b/doc/manual/submit-job.en.md index de73f8f..5106bd9 100644 --- a/doc/manual/submit-job.en.md +++ b/doc/manual/submit-job.en.md @@ -141,7 +141,7 @@ options: hi8109@account-vm ~% submit -m iperf.yaml submit iperf.yaml, got job_id=z9.173923 query=>{"job_id":["z9.173923"]} - connect to ws://172.168.131.2:11310/filter + connect to ws://172.168.131.2:20001/filter {"job_id":"z9.173923","message":"","job_state":"submit","result_root":"/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch6 {"job_id": "z9.173923", "result_root": "/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/tcp-30/z9.173923", "job_state {"job_id": "z9.173923", "job_state": "boot"} @@ -175,7 +175,7 @@ options: hi8109@account-vm ~% submit -m -c borrow-1h.yaml submit borrow-1h.yaml, got job_id=z9.173925 query=>{"job_id":["z9.173925"]} - connect to ws://172.168.131.2:11310/filter + connect to ws://172.168.131.2:20001/filter {"job_id":"z9.173925","message":"","job_state":"submit","result_root":"/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch {"job_id": "z9.173925", "result_root": "/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/3600/z9.173925", "job_state" {"job_id": "z9.173925", "job_state": "boot"} diff --git a/doc/manual/submit-job.zh.md b/doc/manual/submit-job.zh.md index 77e9566..5f1838b 100644 --- a/doc/manual/submit-job.zh.md +++ b/doc/manual/submit-job.zh.md @@ -140,7 +140,7 @@ options: hi8109@account-vm ~% submit -m iperf.yaml testbox=vm-2p8g submit iperf.yaml, got job_id=z9.173923 query=>{"job_id":["z9.173923"]} - connect to ws://172.168.131.2:11310/filter + connect to ws://172.168.131.2:20001/filter {"job_id":"z9.173923","message":"","job_state":"submit","result_root":"/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch6 {"job_id": "z9.173923", "result_root": "/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/tcp-30/z9.173923", "job_state {"job_id": "z9.173923", "job_state": "boot"} @@ -173,7 +173,7 @@ options: hi8109@account-vm ~% submit -m -c borrow-1h.yaml testbox=vm-2p8g submit borrow-1h.yaml, got job_id=z9.173925 query=>{"job_id":["z9.173925"]} - connect to ws://172.168.131.2:11310/filter + connect to ws://172.168.131.2:20001/filter {"job_id":"z9.173925","message":"","job_state":"submit","result_root":"/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch {"job_id": "z9.173925", "result_root": "/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/3600/z9.173925", "job_state" {"job_id": "z9.173925", "job_state": "boot"} diff --git a/src/delimiter/constants.rb b/src/delimiter/constants.rb index a790736..da3f075 100644 --- a/src/delimiter/constants.rb +++ b/src/delimiter/constants.rb @@ -4,7 +4,7 @@ LKP_SRC ||= ENV['LKP_SRC'] || '/c/lkp-tests' MONITOR_HOST ||= ENV['MONITOR_HOST'] || '172.17.0.1' -MONITOR_PORT ||= ENV['MONITOR_PORT'] || '11310' +MONITOR_PORT ||= ENV['MONITOR_PORT'] || '20001' GIT_MIRROR_HOST ||= ENV['GIT_MIRROR_HOST'] || '172.17.0.1' DELIMITER_EMAIL ||= ENV['DELIMITER_EMAIL'] || 'delimiter@localhost' BISECT_RUN_SCRIPT = "#{ENV['CCI_SRC']}/src/delimiter/find-commit/bisect_run_script.rb" diff --git a/src/monitoring/constants.cr b/src/monitoring/constants.cr index 154c536..9e080c8 100644 --- a/src/monitoring/constants.cr +++ b/src/monitoring/constants.cr @@ -1,4 +1,4 @@ MQ_HOST = (ENV.has_key?("MQ_HOST") ? ENV["MQ_HOST"] : "172.17.0.1") MQ_PORT = (ENV.has_key?("MQ_PORT") ? ENV["MQ_PORT"] : 5672).to_i32 -MONITOR_PORT = (ENV.has_key?("MONITOR_PORT") ? ENV["MONITOR_PORT"] : 11310).to_i32 +MONITOR_PORT = (ENV.has_key?("MONITOR_PORT") ? ENV["MONITOR_PORT"] : 20001).to_i32 -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] container/dracut-initrd: add e2fsck and basename into dracut module 00bash
by Xu Xijian 16 Mar '21

16 Mar '21

1. mke2fs is depended by e2fsck. 2. We've add basename into 98dracut module, but it's too late, so add it into an earlier module 00bash. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 494588a..616183f 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -17,7 +17,8 @@ cp -a /usr/local/bin/cifs-lib.sh /usr/lib/dracut/modules.d/95cifs/ cat overlay-lkp.sh >> /usr/lib/dracut/modules.d/90overlay-root/overlay-mount.sh sed -i "/install() {/a\ inst /usr/bin/awk" /usr/lib/dracut/modules.d/40network/module-setup.sh -sed -i "/install() {/a\ inst /sbin/mke2fs /usr/bin/basename" /usr/lib/dracut/modules.d/98dracut-systemd/module-setup.sh +sed -i "/install() {/a\ inst /sbin/e2fsck /sbin/mke2fs /usr/bin/basename" /usr/lib/dracut/modules.d/00bash/module-setup.sh + pre_mount_file="/usr/lib/dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh" [ "$(sed -n '$p' $pre_mount_file)" = "exit 0" ] && sed -i '$d' "$pre_mount_file" cat set-local-sysroot.sh >> "$pre_mount_file" -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] src/lib(scheduer): change function name "*pushlish*" to "*publish*" for mq
by Xu Xijian 16 Mar '21

16 Mar '21

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/lifecycle.cr | 4 ++-- src/lib/mq.cr | 4 ++-- src/lib/sched.cr | 4 ++-- src/scheduler/close_job.cr | 2 +- src/scheduler/find_job_boot.cr | 2 +- src/scheduler/update_job_parameter.cr | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 1eb6264..4bd2ac0 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -79,7 +79,7 @@ class Lifecycle "job_state" => "abnormal", "testbox" => testbox } - @mq.pushlish_confirm("job_mq", msg.to_json) + @mq.publish_confirm("job_mq", msg.to_json) @match[testbox].delete(id) end end @@ -313,7 +313,7 @@ class Lifecycle mq_queue = get_machine_reboot_queue(testbox) machine.as_h.delete("history") machine.as_h["testbox"] = JSON::Any.new(testbox) - @mq.pushlish_confirm(mq_queue, machine.to_json, durable: true) + @mq.publish_confirm(mq_queue, machine.to_json, durable: true) machine["state"] = "rebooting_queue" machine["time"] = Time.local.to_s("%Y-%m-%dT%H:%M:%S+0800") diff --git a/src/lib/mq.cr b/src/lib/mq.cr index 980585d..1c7f4b7 100644 --- a/src/lib/mq.cr +++ b/src/lib/mq.cr @@ -37,7 +37,7 @@ class MQClient Singleton::Of(self).instance end - def pushlish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish_confirm(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) @@ -46,7 +46,7 @@ class MQClient end end - def pushlish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 9528f54..ac677de 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -219,9 +219,9 @@ class Sched return nil end - def mq_pushlish_confirm(queue, msg) + def mq_publish_confirm(queue, msg) 3.times do - @mq.pushlish_confirm(queue, msg) + @mq.publish_confirm(queue, msg) break rescue e res = @mq.reconnect diff --git a/src/scheduler/close_job.cr b/src/scheduler/close_job.cr index e014361..a57b233 100644 --- a/src/scheduler/close_job.cr +++ b/src/scheduler/close_job.cr @@ -45,7 +45,7 @@ class Sched "job_state" => "close", "time" => get_time } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end end diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 4a8ea8a..416f8f4 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -32,7 +32,7 @@ class Sched "time" => get_time, "job_state" => "boot" } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end # auto submit a job to collect the host information. diff --git a/src/scheduler/update_job_parameter.cr b/src/scheduler/update_job_parameter.cr index d81f28b..92b905e 100644 --- a/src/scheduler/update_job_parameter.cr +++ b/src/scheduler/update_job_parameter.cr @@ -44,6 +44,6 @@ class Sched "job_state" => (@env.get?("job_state") || "update").to_s, "time" => @env.get?("time").to_s } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end -- 2.23.0

1 0

[PATCH compass-ci] container/dracut-initrd: add e2fsck and basename into dracut module 00bash
by Xu Xijian 16 Mar '21

16 Mar '21

1. mke2fs is depended by e2fsck. 2. We've add basename into 98dracut module, but it's too late, so add it into an earlier module 00bash. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 494588a..48e8bcc 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -18,6 +18,7 @@ cp -a /usr/local/bin/cifs-lib.sh /usr/lib/dracut/modules.d/95cifs/ cat overlay-lkp.sh >> /usr/lib/dracut/modules.d/90overlay-root/overlay-mount.sh sed -i "/install() {/a\ inst /usr/bin/awk" /usr/lib/dracut/modules.d/40network/module-setup.sh sed -i "/install() {/a\ inst /sbin/mke2fs /usr/bin/basename" /usr/lib/dracut/modules.d/98dracut-systemd/module-setup.sh +sed -i "/install() {/a\ inst /sbin/e2fsck /sbin/mke2fs /usr/bin/basename" /usr/lib/dracut/modules.d/00bash/module-setup.sh pre_mount_file="/usr/lib/dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh" [ "$(sed -n '$p' $pre_mount_file)" = "exit 0" ] && sed -i '$d' "$pre_mount_file" cat set-local-sysroot.sh >> "$pre_mount_file" -- 2.23.0

2 2

[PATCH v3 lkp-tests] add test script for dockerfile os migrating
by Bai Jing 16 Mar '21

16 Mar '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 151 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100755 tests/openeuler_docker diff --git a/tests/openeuler_docker b/tests/openeuler_docker new file mode 100755 index 000000000..b0b5336a3 --- /dev/null +++ b/tests/openeuler_docker @@ -0,0 +1,151 @@ +#!/bin/sh +# - dockerfile_url + +# exit 1 if openEuler failed +# exit 0 if +# - no centos support +# - centos failed + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/log.sh + +[ -n "$dockerfile_url" ] || die "dockerfile_url is empty" + +declare -i nr_openeuler_fail=0 +declare -i nr_openeuler_success=0 +declare -i is_openeuler_build_fail=0 +declare -i dockerfile_count=0 + +fit_repo() +{ + yum reinstall -y git + git clone "$dockerfile_url" "$BENCHMARK_ROOT/src/" + dockerfile_path="$(find "$BENCHMARK_ROOT/src/" -type f -name "*Dockerfile*" | xargs grep -HE '^FROM centos:(centos|)[78]')" + dockerfile_count=$(( $(echo $dockerfile_path | awk '{print NF}') / 2 )) + [ -n "$dockerfile_path" ] || { + echo "current-repo-not-support-centos: 1" + exit 0 + } +} + +openeuler_ready() +{ + docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar +} + +centos_docker() +{ + is_openeuler_build_fail=0 + index=1 + for file in $dockerfile_path + do + [[ $file =~ ^centos ]] && continue + file=$(echo "$file" | awk -F: '{print $1}') + dockerfile_name="${file##*/}" + cd "${file%/*}" || continue + image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') + [ -n "$image" ] && image=${image}_$index + if [ -f "build.sh" ] && [ -x "build.sh" ]; then + ./build.sh > /dev/null 2> /dev/null + else + docker_build > /dev/null 2> /dev/null + fi + + [ $? == 0 ] && { + handle_dockerfile + } + + let index++ + done +} + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" +} + +handle_epel() +{ + grep -qw "epel-release" "$dockerfile_name" && { + sed -i "s/epel-release/bash/g" "$dockerfile_name" + } +} + +handle_epelrpm() +{ + grep -qw "https://.*epel-release.*rpm" "$dockerfile_name" && { + sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" + } +} + +add_base_commands() +{ + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" +} + +handle_epel_repo() +{ + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" +} + +handle_dockerfile() +{ + is_openeuler_build_fail=1 + handle_FROM + handle_epel + add_base_commands + if docker_build "-openeuler"; then + docker_run "-openeuler" + let nr_openeuler_success++ + else + let nr_openeuler_fail++ + fi +} + +docker_build() +{ + docker build -f "$dockerfile_name" -t "${image}$1" . + [ $? != 0 ] && { + [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.fail: 1" + return 1 + } + [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.success: 1" + is_openeuler_build_fail=0 +} + +docker_run() +{ + docker run -d --name "${image}" "${image}$1" + docker_stats=$(docker inspect -f '{{.State.Running}}' ${image}) + [ $docker_stats == false ] && { + echo "${image}.run.fail: 1" + return 1 + } + + echo "${image}.run.success: 1" +} + +handle_build_result() +{ + if [ "$nr_openeuler_fail" != 0 ]; then + echo "openeuler build failed times: $nr_openeuler_fail" + echo "openeuler_valid_url: $dockerfile_url" + exit 1 + # centos all success, openeuler all success + elif [ "$nr_openeuler_success" == "$dockerfile_count" ]; then + echo "openeuler_valid_url: $dockerfile_url" + exit 0 + fi +} + +docker_test() +{ + fit_repo + openeuler_ready + centos_docker + handle_build_result +} + +docker_test -- 2.23.0

1 0

[PATCH lkp-tests] add test script for dockerfile os migrating
by Bai Jing 16 Mar '21

16 Mar '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 152 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100755 tests/openeuler_docker diff --git a/tests/openeuler_docker b/tests/openeuler_docker new file mode 100755 index 000000000..74087ab62 --- /dev/null +++ b/tests/openeuler_docker @@ -0,0 +1,152 @@ +#!/bin/sh +# - dockerfile_url + +# exit 1 if openEuler failed +# exit 0 if +# - no centos support +# - centos failed + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/log.sh + +[ -n "$dockerfile_url" ] || die "dockerfile_url is empty" + +declare -i nr_openeuler_fail=0 +declare -i nr_openeuler_success=0 +declare -i is_openeuler_build_fail=0 +declare -i dockerfile_count=0 + +fit_repo() +{ + yum reinstall -y git + git clone "$dockerfile_url" "$BENCHMARK_ROOT/src/" + dockerfile_path="$(find "$BENCHMARK_ROOT/src/" -type f -name "*Dockerfile*" | xargs grep -HE '^FROM centos:(centos|)[78]')" + dockerfile_count=$(( $(echo $dockerfile_path | awk '{print NF}') / 2 )) + [ -n "$dockerfile_path" ] || { + echo "current-repo-not-support-centos: 1" + exit 0 + } +} + +openeuler_ready() +{ + docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar +} + +centos_docker() +{ + is_openeuler_build_fail=0 + index=1 + for file in $dockerfile_path + do + [[ $file =~ ^centos ]] && continue + file=$(echo "$file" | awk -F: '{print $1}') + dockerfile_name="${file##*/}" + cd "${file%/*}" || continue + image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') + [ -n "$image" ] && image=${image}_$index + if [ -f "build.sh" ] && [ -x "build.sh" ]; then + ./build.sh > /dev/null 2> /dev/null + else + docker_build > /dev/null 2> /dev/null + fi + + [ $? == 0 ] && { + handle_dockerfile + } + + let index++ + done +} + + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" +} + +handle_epel() +{ + grep -qw "epel-release" "$dockerfile_name" && { + sed -i "s/epel-release/bash/g" "$dockerfile_name" + } +} + +handle_epelrpm() +{ + grep -qw "https://.*epel-release.*rpm" "$dockerfile_name" && { + sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" + } +} + +add_base_commands() +{ + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" +} + +handle_epel_repo() +{ + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" +} + +handle_dockerfile() +{ + is_openeuler_build_fail=1 + handle_FROM + handle_epel + add_base_commands + if docker_build "-openeuler"; then + docker_run "-openeuler" + let nr_openeuler_success++ + else + let nr_openeuler_fail++ + fi +} + +docker_build() +{ + docker build -f "$dockerfile_name" -t "${image}$1" . + [ $? != 0 ] && { + [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.fail: 1" + return 1 + } + [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.success: 1" + is_openeuler_build_fail=0 +} + +docker_run() +{ + docker run -d --name "${image}" "${image}$1" + docker_stats=$(docker inspect -f '{{.State.Running}}' ${image}) + [ $docker_stats == false ] && { + echo "${image}.run.fail: 1" + return 1 + } + + echo "${image}.run.success: 1" +} + +handle_build_result() +{ + if [ "$nr_openeuler_fail" != 0 ]; then + echo "openeuler build failed times: $nr_openeuler_fail" + echo "openeuler_valid_url: $dockerfile_url" + exit 1 + # centos all success, openeuler all success + elif [ "$nr_openeuler_success" == "$dockerfile_count" ]; then + echo "openeuler_valid_url: $dockerfile_url" + exit 0 + fi +} + +docker_test() +{ + fit_repo + openeuler_ready + centos_docker + handle_build_result +} + +docker_test -- 2.23.0

2 2

[PATCH compass-ci] sparrow: change service default ports
by Cao Xueliang 16 Mar '21

16 Mar '21

Reference: 6354ba631e0fd5f795a1dc28b65e2002bd3de668 Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/extract-stats/start | 2 +- sparrow/3-code/dev-env | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/container/extract-stats/start b/container/extract-stats/start index 5033754..621099a 100755 --- a/container/extract-stats/start +++ b/container/extract-stats/start @@ -25,7 +25,7 @@ env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' DEFAULT_CCI = '/c/compass-ci' SRV_HTTP_RESULT_HOST = defaults['SRV_HTTP_RESULT_HOST'] || Open3.capture3('curl ip.sb')[0].chomp -SRV_HTTP_RESULT_PORT = defaults['SRV_HTTP_RESULT_PORT'] || '11300' +SRV_HTTP_RESULT_PORT = defaults['SRV_HTTP_RESULT_PORT'] || '20007' docker_rm "extract-stats" cmd = %w[ diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index b740a15..65061d8 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -15,10 +15,10 @@ server_name=$(hostname | cut -f1 -d.) : ${SRV_HTTP_OS_HOST:=$server_ip} : ${SRV_HTTP_GIT_HOST:=$server_ip} : ${SRV_HTTP_CCI_HOST:=$server_ip} -: ${SRV_HTTP_RESULT_PORT:=11300} -: ${SRV_HTTP_OS_PORT:=11304} -: ${SRV_HTTP_GIT_PORT:=11305} -: ${SRV_HTTP_CCI_PORT:=11307} +: ${SRV_HTTP_RESULT_PORT:=20007} +: ${SRV_HTTP_OS_PORT:=20009} +: ${SRV_HTTP_GIT_PORT:=20010} +: ${SRV_HTTP_CCI_PORT:=20011} : ${LOGGING_ES_HOST:=$server_ip} : ${LOGGING_ES_PORT:=9202} : ${INITRD_HTTP_HOST:=$server_ip} -- 2.23.0

1 0

[PATCH compass-ci] src/lib(scheduer): change function name "*pushlish*" to "*publish*" for mq
by Xu Xijian 16 Mar '21

16 Mar '21

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/lifecycle.cr | 4 ++-- src/lib/mq.cr | 4 ++-- src/lib/sched.cr | 4 ++-- src/scheduler/close_job.cr | 2 +- src/scheduler/find_job_boot.cr | 2 +- src/scheduler/update_job_parameter.cr | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 1eb6264..4bd2ac0 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -79,7 +79,7 @@ class Lifecycle "job_state" => "abnormal", "testbox" => testbox } - @mq.pushlish_confirm("job_mq", msg.to_json) + @mq.publish_confirm("job_mq", msg.to_json) @match[testbox].delete(id) end end @@ -313,7 +313,7 @@ class Lifecycle mq_queue = get_machine_reboot_queue(testbox) machine.as_h.delete("history") machine.as_h["testbox"] = JSON::Any.new(testbox) - @mq.pushlish_confirm(mq_queue, machine.to_json, durable: true) + @mq.publish_confirm(mq_queue, machine.to_json, durable: true) machine["state"] = "rebooting_queue" machine["time"] = Time.local.to_s("%Y-%m-%dT%H:%M:%S+0800") diff --git a/src/lib/mq.cr b/src/lib/mq.cr index 980585d..1c7f4b7 100644 --- a/src/lib/mq.cr +++ b/src/lib/mq.cr @@ -37,7 +37,7 @@ class MQClient Singleton::Of(self).instance end - def pushlish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish_confirm(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) @@ -46,7 +46,7 @@ class MQClient end end - def pushlish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 9528f54..ac677de 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -219,9 +219,9 @@ class Sched return nil end - def mq_pushlish_confirm(queue, msg) + def mq_publish_confirm(queue, msg) 3.times do - @mq.pushlish_confirm(queue, msg) + @mq.publish_confirm(queue, msg) break rescue e res = @mq.reconnect diff --git a/src/scheduler/close_job.cr b/src/scheduler/close_job.cr index e014361..a57b233 100644 --- a/src/scheduler/close_job.cr +++ b/src/scheduler/close_job.cr @@ -45,7 +45,7 @@ class Sched "job_state" => "close", "time" => get_time } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end end diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 4a8ea8a..416f8f4 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -32,7 +32,7 @@ class Sched "time" => get_time, "job_state" => "boot" } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end # auto submit a job to collect the host information. diff --git a/src/scheduler/update_job_parameter.cr b/src/scheduler/update_job_parameter.cr index d81f28b..92b905e 100644 --- a/src/scheduler/update_job_parameter.cr +++ b/src/scheduler/update_job_parameter.cr @@ -44,6 +44,6 @@ class Sched "job_state" => (@env.get?("job_state") || "update").to_s, "time" => @env.get?("time").to_s } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end -- 2.23.0

1 0

[PATCH v3 lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 16 Mar '21

16 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 47 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..d6acad1ab 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,40 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + elsif error_msg.is_a? String + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/[^\w]/, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line.gsub!(/^-|-$/, '') + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

1 0

[PATCH compass-ci] doc: update document
by Luan Shengde 16 Mar '21

16 Mar '21

update document for service new port: send-internet-mail: 49000 => 10001 Signed-off-by: Luan Shengde <shdluan(a)163.com> --- doc/send-mail.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/send-mail.md b/doc/send-mail.md index 5af8d17..ffa5651 100644 --- a/doc/send-mail.md +++ b/doc/send-mail.md @@ -8,7 +8,7 @@ The service is used to send mails with formatted mail data ## send mail port - send intranet mail: 11311 -- send internet mail: 49000 +- send internet mail: 10001 ## send mail host - host the service running on @@ -58,5 +58,5 @@ The service is used to send mails with formatted mail data test msg 1010 " - %x(curl -XPOST 'localhost:49000/send_mail_text' -d "#{data}") + %x(curl -XPOST 'localhost:10001/send_mail_text' -d "#{data}") ``` -- 2.23.0

1 0

[PATCH v3 lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 16 Mar '21

16 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 47 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..9e9e15fab 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,40 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + elsif error_msg.is_a? String + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/[^\w]/, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line.gsub!(/-$/, '') + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

1 1

[PATCH v2 lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 16 Mar '21

16 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 47 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..119dc1d2a 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,40 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + else + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/[^\w]/, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line.gsub!(/-$/, '') + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

2 2