- Compass-ci - mailweb.openeuler.org

[PATCH v6 compass-ci] providers/qemu: adapt for x86_64 and riscv64
by Liu Yinsi 08 Jan '21

08 Jan '21

before: support kvm for aarch64 after: support kvm for aarch64, x86_64, riscv64 test: /srv/result/host-info/2021-01-08/vm-8p64g/debian-sid-riscv64/z9.289591 /srv/result/host-info/2021-01-08/vm-8p64g/debian-sid-x86_64/z9.289594 /srv/result/host-info/2021-01-08/vm-8p64g/debian-sid-aarch64/z9.289571 [how] put public options like $kernel, $initrd, $smp together, set individual options like $machine, $bios, $nic according to different qemu command, then merge all options to run qemu. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 116 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 101 insertions(+), 15 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index c926d3d..363cf4c 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -18,12 +18,6 @@ check_logfile() } } -qemu_command() -{ - qemu=qemu-system-aarch64 - command -v $qemu >/dev/null || qemu=qemu-kvm -} - write_logfile() { ipxe_script=ipxe_script @@ -66,12 +60,57 @@ check_option_value() [ -n "$initrds" ] || exit } -get_initrd() +set_initrd() { initrd=initrd cat $initrds > $initrd } +set_bios() +{ + bios=/usr/share/qemu-efi-aarch64/QEMU_EFI.fd + [ -f "$bios" ] || bios=/usr/share/ovmf/OVMF.fd +} + +set_helper() +{ + helper=/usr/libexec/qemu-bridge-helper + [ -f "$helper" ] || helper=/usr/lib/qemu/qemu-bridge-helper +} + +set_nic() +{ + nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" +} + +set_device() +{ + device="virtio-net-device,netdev=net0,mac=${mac}" +} + +set_netdev() +{ + netdev="bridge,br=br0,id=net0,helper=${helper}" +} + +set_qemu() +{ + qemu=( + qemu-system-aarch64 + qemu-kvm + qemu-system-x86_64 + qemu-system-riscv64 + ) + + for qemu in "${qemu[@]}" + do + [ -n "$(command -v ${qemu})" ] && break + done + + # debian has both qemu-system-x86_64 and qemu-system-riscv64 command + [[ $kernel =~ 'riscv64' ]] && qemu=qemu-system-riscv64 +} + print_message() { echo $SCHED_PORT @@ -83,36 +122,83 @@ print_message() sleep 5 } -run_qemu() +public_option() { kvm=( $qemu - -machine virt-4.0,accel=kvm,gic-version=3 -kernel $kernel -initrd $initrd -smp $nr_cpu -m $memory - -cpu Kunpeng-920 - -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd - -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} -k en-us -no-reboot -nographic -serial file:${log_file} -monitor null ) +} + +individual_option() +{ + case "$qemu" in + qemu-system-aarch64) + arch_option=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) + ;; + qemu-kvm) + [ "$(arch)" == "aarch64" ] && arch_option=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) + [ "$(arch)" == "x86_64" ] && arch_option=( + -bios $bios + -nic $nic + ) + ;; + qemu-system-x86_64) + arch_option=( + -bios $bios + -nic $nic + ) + ;; + qemu-system-riscv64) + arch_option=( + -machine virt + -device $device + -netdev $netdev + ) + ;; + esac +} - "${kvm[@]}" --append "${append}" +run_qemu() +{ + "${kvm[@]}" "${arch_option[@]}" --append "${append}" } check_logfile write_logfile -qemu_command parse_ipxe_script check_option_value -get_initrd + +set_initrd +set_bios +set_helper +set_nic +set_device +set_netdev +set_qemu print_message +public_option +individual_option + run_qemu -- 2.23.0

1 0

[PATCH v3 compass-ci] 3-code/git: modify the git download script
by Hu Xuejiao 08 Jan '21

08 Jan '21

Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sparrow/3-code/git | 42 ++++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/sparrow/3-code/git b/sparrow/3-code/git index 48741ec..c9b5df4 100755 --- a/sparrow/3-code/git +++ b/sparrow/3-code/git @@ -5,31 +5,21 @@ # fix files permissions cause container service start failed umask 002 -# download git trees - -cd /c || exit - -# git clone https://gitee.com/openeuler/compass-ci.git -# modify and manual run for now: -[ -d "compass-ci/.git" ] || { - git clone https://gitee.com/wu_fengguang/compass-ci.git - - cd /c/compass-ci || exit - [ -n "$compass_commit_id" ] && git reset --hard "$compass_commit_id" - - ln -s compass-ci /c/cci +download_repo() +{ + local repo="$1" + local commit_id="$2" + + [ -d "/c/$repo/.git" ] || { + git clone https://gitee.com/wu_fengguang/"$repo".git /c/"$repo" + + [ "$repo" == "compass-ci" ] && ln -s /c/"$repo" /c/cci + } + + cd /c/"$repo" || exit + [ -n "$commit_id" ] && git reset --hard "$commit_id" } -[ -d "lkp-tests/.git" ] || { - git clone https://gitee.com/wu_fengguang/lkp-tests.git - - cd /c/lkp-tests || exit - [ -n "$lkp_commit_id" ] && git reset --hard "$lkp_commit_id" -} - -[ -d "upstream-repos/.git" ] || { - git clone https://gitee.com/wu_fengguang/upstream-repos.git - - cd /c/upstream-repos || exit - [ -n "$upstream_commit_id" ] && git reset --hard "$upstream_commit_id" -} +download_repo "compass-ci" "$compass_commit_id" +download_repo "lkp-tests" "$lkp_commit_id" +download_repo "upstream-repos" "$upstream_commit_id" -- 2.23.0

1 0

[PATCH compass-ci] 3-code/git: modify the git download script
by Hu Xuejiao 08 Jan '21

08 Jan '21

Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sparrow/3-code/git | 42 ++++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/sparrow/3-code/git b/sparrow/3-code/git index 48741ec..6174916 100755 --- a/sparrow/3-code/git +++ b/sparrow/3-code/git @@ -5,31 +5,21 @@ # fix files permissions cause container service start failed umask 002 -# download git trees - -cd /c || exit - -# git clone https://gitee.com/openeuler/compass-ci.git -# modify and manual run for now: -[ -d "compass-ci/.git" ] || { - git clone https://gitee.com/wu_fengguang/compass-ci.git - - cd /c/compass-ci || exit - [ -n "$compass_commit_id" ] && git reset --hard "$compass_commit_id" - - ln -s compass-ci /c/cci +download_repo() +{ + local repo="$1" + local commit_id="$2" + + [ -d "/c/$repo/.git" ] || { + git clone https://gitee.com/wu_fengguang/"$repo".git /c/"$repo" + + [ "$repo" == "compass-ci" ] && ln -s /c/"$repo" /c/cci + } + + cd /c/"$repo" || exit + [ -n "$commit_id" ] && git reset --hard "$commit_id" } -[ -d "lkp-tests/.git" ] || { - git clone https://gitee.com/wu_fengguang/lkp-tests.git - - cd /c/lkp-tests || exit - [ -n "$lkp_commit_id" ] && git reset --hard "$lkp_commit_id" -} - -[ -d "upstream-repos/.git" ] || { - git clone https://gitee.com/wu_fengguang/upstream-repos.git - - cd /c/upstream-repos || exit - [ -n "$upstream_commit_id" ] && git reset --hard "$upstream_commit_id" -} +download_repo "compass-ci" "compass_commit_id" +download_repo "lkp-tests" "$lkp_commit_id" +download_repo "upstream-repos" "$upstream_commit_id" -- 2.23.0

2 1

[PATCH v2 compass-ci 2/2] assign-account: add 'lab' to default yaml
by Luan Shengde 08 Jan '21

08 Jan '21

the default yaml file will contain: my_email my_name lab lab is used to read the lab yaml according to its value. example: if the value of lab is 'z9', the sched will read token from file: z9.yaml Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/assign-account/get_account_info.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/container/assign-account/get_account_info.rb b/container/assign-account/get_account_info.rb index af08d91..7e0a5a7 100755 --- a/container/assign-account/get_account_info.rb +++ b/container/assign-account/get_account_info.rb @@ -173,6 +173,7 @@ class AccountStorage # they are added along with 'my_ssh_pubkey' when sending assign account request account_yaml['my_email'] = @data['my_email'] account_yaml['my_name'] = @data['my_name'] + account_yaml['lab'] = @data['lab'] File.open(default_yaml, 'w') do |f| f.puts account_yaml.to_yaml -- 2.23.0

2 1

[PATCH lkp-tests 1/2] lib/job.rb: add fault_reproduction
by Wei Jihui 08 Jan '21

08 Jan '21

do fault reproduction when submit a failed job Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/job.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/job.rb b/lib/job.rb index 8346db87..631ebfc9 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -255,6 +255,7 @@ class Job hash.delete('override') end hash.merge!(@overrides) + hash.merge!(fault_reproduction) if hash.has_key?("id") @jobs.concat(multi_args(hash)) # return [hash] or [h1,h2] end rescue StandardError => e @@ -285,6 +286,10 @@ class Job @jobfile = jobfile end + def fault_reproduction + wait = load_yaml("#{lkp_src}/jobs/fault_reproduction.yaml", {}) + end + def multi_args(hash) jobs_array = [] hash.each { |key, value| -- 2.23.0

2 1

[PATCH lkp-tests] bootstrap: don't try to install ntpdate
by Wu Fengguang 08 Jan '21

08 Jan '21

It's not necessary for every boot. Let's not slow down every test run. Signed-off-by: Wu Fengguang <wfg(a)mail.ustc.edu.cn> --- lib/bootstrap.sh | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index d4b41f3b..b04ea3dd 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -237,20 +237,6 @@ run_ntpdate() [ -n "$LKP_SERVER" ] || return is_clearlinux && clearlinux_timesync && return - has_cmd ntpdate || { - detect_system - local DISTRO=$_system_name_lowercase - [ "$DISTRO" == "unknown" ] && { - echo "[WARNING] Current os cannot be recognized by lkp-test, so skip run_ntpdate." - return - } - has_cmd ruby || { - echo "[WARNING] Cannot find ruby in $DISTRO, so skip run_ntpdate." - return - } - . $LKP_SRC/distro/installer/$DISTRO $(echo ntpdate | $LKP_SRC/sbin/adapt-packages $DISTRO) - } - has_cmd ntpdate || return local hour="$(date +%H)" -- 2.23.0

1 0

[PATCH v3 lkp-tests] lib/bootstrap.sh: 'run-ipconfig' step should be before the check step
by Yu Chuan 08 Jan '21

08 Jan '21

[Why] The `run-ipconfig` step is the IP configuration step for initramfs; The `[ -z "$net_devices"]` step is check whether the network is normal. Now the `check normal` step is before 'ip configuration' step, so we encounter the following error: + export NO_NETWORK=1 + echo export NO_NETWORK=1 due to no net devices + return [Debug Msg] + [ ! -e /usr/share/initramfs-tools/scripts/functions ] + get_net_devices + local net_devices + local i + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + [ p0s1/ != /sys/class/net/enp0s1/ ] + basename /sys/class/net/enp0s1/ + net_devices= enp0s1 + [ != /sys/class/net/lo/ ] + continue + echo enp0s1 + local net_devices= enp0s1 + [ -z ] + warn_no_eth0 + [ -f /proc/config.gz ] + return + echo ls /sys/class/net + ls /sys/class/net + is_virt + [ -n ] + [ -n ] + has_cmd virt-what + command -v virt-what + grep -q -w hypervisor /proc/cpuinfo + return 0 + export NO_NETWORK=1 + echo export NO_NETWORK=1 due to no net devices + return Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 7d15bb370fa9..770d550724f1 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -160,6 +160,9 @@ setup_network() return } + $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig + network_ok && return + local net_devices=$(get_net_devices) if [ -z "$net_devices" ]; then @@ -181,9 +184,6 @@ setup_network() fi fi - $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig - network_ok && return - local err_msg='IP-Config: Auto-configuration of network failed' dmesg | grep -q -F "$err_msg" || { # Include $err_msg in the error message so that it matches -- 2.23.0

2 3

[PATCH v3 lkp-tests] lib/bootstrap.sh: fix error when os is not integrated into lkp-tests
by Yu Chuan 08 Jan '21

08 Jan '21

[Why] If the current os has not been integrated into lkp-tests, run method `detect_system`, then var `$_system_name_lowercase` will be 'unknown'. So we should adapt this. [Error] + local DISTRO=unknown ++ echo ntpdate ++ /home/yuchuan/lkp-tests/sbin/adapt-packages unknown Traceback (most recent call last): 3: from /home/yuchuan/lkp-tests/sbin/adapt-packages:11:in `<main>' 2: from /usr/share/ruby/psych.rb:497:in `load_file' 1: from /usr/share/ruby/psych.rb:497:in `open' /usr/share/ruby/psych.rb:497:in `initialize': No such file or directory @ rb_sysopen - /home/yuchuan/lkp-tests/distro/adaptation/unknown (Errno::ENOENT) [How] If the current os has not been integrated into lkp-tests, then return. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 7d15bb370fa9..912a76bb6687 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -240,6 +240,10 @@ run_ntpdate() has_cmd ntpdate || { detect_system local DISTRO=$_system_name_lowercase + [ "$DISTRO" == "unknown" ] && { + echo "[WARNING] Current os cannot be recognized by lkp-test, so skip run_ntpdate." + return + } . $LKP_SRC/distro/installer/$DISTRO $(echo ntpdate | $LKP_SRC/sbin/adapt-packages $DISTRO) } -- 2.23.0

2 3

[PATCH v5 compass-ci] providers/qemu: adapt for x86_64 and riscv64
by Liu Yinsi 08 Jan '21

08 Jan '21

before: support kvm for aarch64 after: support kvm for aarch64, x86_64, riscv64 test: /srv/result/host-info/2021-01-07/vm-8p64g/debian-sid-riscv64/z9.288315 /srv/result/host-info/2021-01-07/vm-8p64g/debian-sid-x86_64/z9.288332 /srv/result/host-info/2021-01-07/vm-8p64g/debian-sid-aarch64/z9.288335 [how] put public options like $kernel, $initrd, $smp together, list different options like $machine, $bios, $nic according to different qemu command, then merge all options. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 110 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 95 insertions(+), 15 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index c926d3d..a8a2b33 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -18,12 +18,6 @@ check_logfile() } } -qemu_command() -{ - qemu=qemu-system-aarch64 - command -v $qemu >/dev/null || qemu=qemu-kvm -} - write_logfile() { ipxe_script=ipxe_script @@ -66,12 +60,56 @@ check_option_value() [ -n "$initrds" ] || exit } -get_initrd() +set_initrd() { initrd=initrd cat $initrds > $initrd } +set_bios() +{ + bios=/usr/share/qemu-efi-aarch64/QEMU_EFI.fd + [ -f "$bios" ] || bios=/usr/share/ovmf/OVMF.fd +} + +set_helper() +{ + helper=/usr/libexec/qemu-bridge-helper + [ -f "$helper" ] || helper=/usr/lib/qemu/qemu-bridge-helper +} + +set_nic() +{ + nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" +} + +set_device() +{ + device="virtio-net-device,netdev=net0,mac=${mac}" +} + +set_netdev() +{ + netdev="bridge,br=br0,id=net0,helper=${helper}" +} + +set_qemu() +{ + qemu=( + qemu-system-aarch64 + qemu-kvm + qemu-system-x86_64 + ) + + for qemu in "${qemu[@]}" + do + [ -n "$(command -v ${qemu})" ] && break + done + + # debian has both qemu-system-x86_64 and qemu-system-riscv64 command + [[ $kernel =~ 'riscv64' ]] && qemu=qemu-system-riscv64 +} + print_message() { echo $SCHED_PORT @@ -83,36 +121,78 @@ print_message() sleep 5 } -run_qemu() +public_option() { kvm=( $qemu - -machine virt-4.0,accel=kvm,gic-version=3 -kernel $kernel -initrd $initrd -smp $nr_cpu -m $memory - -cpu Kunpeng-920 - -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd - -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} -k en-us -no-reboot -nographic -serial file:${log_file} -monitor null ) +} - "${kvm[@]}" --append "${append}" +run_qemu() +{ + case "$qemu" in + qemu-system-aarch64) + arch_option=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) + ;; + qemu-kvm) + [ "$(arch)" == "aarch64" ] && arch_option=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) + [ "$(arch)" == "x86_64" ] && arch_option=( + -bios $bios + -nic $nic + ) + ;; + qemu-system-x86_64) + arch_option=( + -bios $bios + -nic $nic + ) + ;; + qemu-system-riscv64) + arch_option=( + -machine virt + -device virtio-net-device,netdev=net0,mac=${mac} + -netdev bridge,br=br0,id=net0,helper=${helper} + ) + ;; + esac + + "${kvm[@]}" "${arch_option[@]}" --append "${append}" } check_logfile write_logfile -qemu_command parse_ipxe_script check_option_value -get_initrd + +set_initrd +set_bios +set_helper +set_nic +set_device +set_netdev +set_qemu print_message +public_option run_qemu -- 2.23.0

3 4

[PATCH lkp-tests] lib/bootstrap.sh: avoid the problem caused by the lack of ruby pkg
by Yu Chuan 08 Jan '21

08 Jan '21

[Why] We use $LKP_SRC/sbin/adapt-packages to find the ntpdate pkg suitable for different systems. But we are not sure that ruby is pre-installed in each system. So add a check, if ruby is not pre-installed, return. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 30e4af828bcd..d4b41f3bd0d5 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -244,6 +244,10 @@ run_ntpdate() echo "[WARNING] Current os cannot be recognized by lkp-test, so skip run_ntpdate." return } + has_cmd ruby || { + echo "[WARNING] Cannot find ruby in $DISTRO, so skip run_ntpdate." + return + } . $LKP_SRC/distro/installer/$DISTRO $(echo ntpdate | $LKP_SRC/sbin/adapt-packages $DISTRO) } -- 2.23.0

2 1