March 2021 - Compass-ci - mailweb.openeuler.org

[PATCH lkp-tests 06/11] install-iso-pre: add step: prepare_lv
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- distro/depends/install-iso-pre | 1 + tests/install-iso-pre | 43 +++++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 distro/depends/install-iso-pre diff --git a/distro/depends/install-iso-pre b/distro/depends/install-iso-pre new file mode 100644 index 000000000000..6f88eb919896 --- /dev/null +++ b/distro/depends/install-iso-pre @@ -0,0 +1 @@ +lvm2 diff --git a/tests/install-iso-pre b/tests/install-iso-pre index 4e8cbe22fcd4..85e2a5fd42ec 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -40,9 +40,50 @@ check_params() done } +create_vg() +{ + log_info "start create vg:$VG_NAME" + + local disk + + for disk in "${rootfs_disk[@]}" + do + log_info "start handle disk: $disk" + pvdisplay ${disk} > /dev/null || pvcreate ${disk} || die "create pv failed: $disk" + + if vgdisplay ${VG_NAME} > /dev/null; then + vgextend ${VG_NAME} ${disk} + else + vgcreate ${VG_NAME} ${disk} + fi + done + +} + +create_lv() +{ + log_info "start create lv:$LV_NAME" + + lvcreate -y -L 10G -n ${LV_NAME} ${VG_NAME} || die "create logical volume failed." +} + prepare_lv() { - : + log_info "start prepare logical volume" + + # VG_NAME defines in job.yaml + export VG_NAME + export LV_NAME="${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX}" + + # lv existed + lvdisplay /dev/mapper/${VG_NAME}-${LV_NAME} > /dev/null && { + log_info "iso already install to this testbox, so rootfs maybe already generated ok. please check it." + exit 0 + } + + # lv not existed + vgdisplay ${VG_NAME} > /dev/null || create_vg + create_lv } prepare_ks() -- 2.23.0

2 2

[PATCH lkp-tests 04/11] install-iso-release: add step: release_rootfs
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- distro/depends/install-iso-release | 2 ++ tests/install-iso-release | 53 +++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 distro/depends/install-iso-release diff --git a/distro/depends/install-iso-release b/distro/depends/install-iso-release new file mode 100644 index 000000000000..2abb6b833291 --- /dev/null +++ b/distro/depends/install-iso-release @@ -0,0 +1,2 @@ +nfs-common +lvm2 diff --git a/tests/install-iso-release b/tests/install-iso-release index 23e8fbff8e4f..8c925ee2bcd8 100755 --- a/tests/install-iso-release +++ b/tests/install-iso-release @@ -7,6 +7,20 @@ set -e . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/log.sh +SKIP_RELEASE_DIRS=( + "srv" + "opt" + "mnt" + "media" + "home" + "lost+found" + "dev" + "proc" + "run" + "sys" + "var/log" +) + check_params() { log_info "start check params" @@ -29,9 +43,46 @@ check_params() done } +prepare_exclude_file() +{ + local exclude_file=$1 + + rm -rf $exclude_file + for i in "${SKIP_RELEASE_DIRS[@]}" + do + echo $i >> $exclude_file + done +} + release_rootfs() { - : + # mount nfs rootfs + local nfs_rootfs_mount_point=/tmp/nfs_rootfs_${ROOTFS_SUFFIX} + + mkdir -p ${nfs_rootfs_mount_point} + mount -t nfs ${NFS_SERVER_HOST}:${NFS_ROOT_DIR}/${os}/${os_arch}/${os_version}-iso-${ROOTFS_SUFFIX} ${nfs_rootfs_mount_point} || die "mount nfs rootfs failed" + + # activate the lvm + vgchange -ay + + # mount local logical volume + local lv_rootfs_mount_point=/tmp/lv_rootfs_${ROOTFS_SUFFIX} + + mkdir -p ${lv_rootfs_mount_point} + mount /dev/mapper/${VG_NAME}-${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX} ${lv_rootfs_mount_point} || die "mount lv rootfs failed" + + # release rootfs + local exclude_file=/tmp/exclude_file_${ROOTFS_SUFFIX} + + prepare_exclude_file "$exclude_file" + rsync -az ${lv_rootfs_mount_point}/. ${nfs_rootfs_mount_point} --exclude-from=$exclude_file || die "copy out rootfs failed" + cd ${nfs_rootfs_mount_point} && mkdir -p "${SKIP_RELEASE_DIRS[@]}" + + # umount nfs rootfs + umount $nfs_rootfs_mount_point || log_warn "umount failed: nfs_rootfs_mount_point" + + # umount local logical volume + umount $lv_rootfs_mount_point || log_warn "umount failed: $lv_rootfs_mount_point" } main() -- 2.23.0

2 2

[PATCH compass-ci] lib/lifecycle.cr: optimize the restart queue
by Wu Zhende 10 Mar '21

10 Mar '21

1.delete machine's history don't need this info. 2.add testbox this field is used to find the corresponding host when a phssical machine is restarted. 3.modify mq queue testbox: vm-2p8g.wuzhende-1 before queue: wuzhende now queue: vm-2p8g.wuzhende Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/lifecycle.cr | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 3dab6e8..1eb6264 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -110,7 +110,7 @@ class Lifecycle def mq_event_loop puts "deal job events" - q = @mq.ch.queue("job_mq") + q = @mq.ch.queue("job_mq", durable: false) q.subscribe(no_ack: false) do |msg| event = JSON.parse(msg.body_io.to_s) job_state = event["job_state"]? @@ -311,6 +311,8 @@ class Lifecycle return if Time.local < deadline mq_queue = get_machine_reboot_queue(testbox) + machine.as_h.delete("history") + machine.as_h["testbox"] = JSON::Any.new(testbox) @mq.pushlish_confirm(mq_queue, machine.to_json, durable: true) machine["state"] = "rebooting_queue" @@ -320,7 +322,7 @@ class Lifecycle def get_machine_reboot_queue(testbox) if testbox.includes?(".") - testbox =~ /.*\.(.*)-\d+$/ + testbox =~ /(.*)-\d+$/ else testbox =~ /(.*)--.*/ end -- 2.23.0

1 0

[PATCH lkp-tests 03/11] feat: step 3 of iso2rootfs: copy out rootfs in second boot stage
by Yu Chuan 10 Mar '21

10 Mar '21

Framework of install-iso-release. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/install-iso-release.yaml | 26 +++++++++++++++++++ tests/install-iso-release | 47 +++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 jobs/install-iso-release.yaml create mode 100755 tests/install-iso-release diff --git a/jobs/install-iso-release.yaml b/jobs/install-iso-release.yaml new file mode 100644 index 000000000000..fabb2cd50902 --- /dev/null +++ b/jobs/install-iso-release.yaml @@ -0,0 +1,26 @@ +suite: install-iso-release +category: benchmark +install-iso-release: + ################# + # iso info + ################# + iso_os: + iso_arch: + iso_version: + + ################# + # nfs server info, used for store rootfs + ################# + NFS_SERVER_HOST: + NFS_ROOT_DIR: + + ################# + # result rootfs suffix, format: $(date "+%Y%m%d%H%M%S") + ################# + ROOTFS_SUFFIX: + + ################# + # name of volume group which stores the logical volume of rootfs + ################# + VG_NAME: + diff --git a/tests/install-iso-release b/tests/install-iso-release new file mode 100755 index 000000000000..23e8fbff8e4f --- /dev/null +++ b/tests/install-iso-release @@ -0,0 +1,47 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +set -e + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/log.sh + +check_params() +{ + log_info "start check params" + + local required_vars=( + "iso_os" + "iso_arch" + "iso_version" + + "NFS_SERVER_HOST" + "NFS_ROOT_DIR" + + "ROOTFS_SUFFIX" + "VG_NAME" + ) + + for i in "${required_vars[@]}" + do + [ -n "$(eval echo \$$i)" ] || die "cannot get value of var: $i" + done +} + +release_rootfs() +{ + : +} + +main() +{ + check_params + + release_rootfs + + lvchange --permission r /dev/mapper/${VG_NAME}-${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX} || + die "change logical volume permission to readonly failed:/dev/mapper/${VG_NAME}-${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX}" +} + +main -- 2.23.0

2 2

[PATCH compass-ci] docker/run.sh: change hostname
by Wang Yong 10 Mar '21

10 Mar '21

[before] Build Host : dc-8g [after] Build Host : dc-8g.compass-ci.net this will change Build Host in RPM build Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- providers/docker/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/docker/run.sh b/providers/docker/run.sh index 2810641..2dfe36d 100755 --- a/providers/docker/run.sh +++ b/providers/docker/run.sh @@ -23,7 +23,7 @@ busybox_path=$(command -v busybox) cmd=( docker run --rm - --hostname $host + --hostname $host.compass-ci.net -m $memory --tmpfs /tmp:rw,exec,nosuid,nodev -e CCI_SRC=/c/compass-ci -- 2.23.0

1 0

[PATCH lkp-tests 02/11] feat: step 2 of iso2rootfs: reboot directly in firstboot stage
by Yu Chuan 10 Mar '21

10 Mar '21

[Why] Firstboot stage: the first boot after iso installation is complete. At this stage, the system will run some initial configurations. So we need copy out the rootfs in second boot stage, and do nothing in firstboot stage. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/install-iso-firstboot.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 jobs/install-iso-firstboot.yaml diff --git a/jobs/install-iso-firstboot.yaml b/jobs/install-iso-firstboot.yaml new file mode 100644 index 000000000000..ccec24c1c162 --- /dev/null +++ b/jobs/install-iso-firstboot.yaml @@ -0,0 +1,3 @@ +suite: install-iso-firstboot + +sleep: 1 -- 2.23.0

2 2

[PATCH v3 compass-ci 2/2] providers/multi-qemu: add the function of restarting VMs
by Wu Zhende 10 Mar '21

10 Mar '21

Obtain the VM to be restarted from the mq and restart it Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- providers/multi-qemu | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/providers/multi-qemu b/providers/multi-qemu index c729c8e..5979cc9 100755 --- a/providers/multi-qemu +++ b/providers/multi-qemu @@ -5,6 +5,9 @@ require 'fileutils' require 'optparse' +require 'json' + +require '../lib/mq_client' opt = {} options = OptionParser.new do |opts| @@ -42,6 +45,8 @@ HOSTNAME = opt['hostname_prefix'] || "vm-2p8g.#{ENV['HOSTNAME']}" NR_VM = opt['nr_vm'] || 1 QUEUES = opt['queues'] || "#{ENV['HOSTNAME']}.#{RUBY_PLATFORM.split('-')[0]}" LOG_DIR = '/srv/cci/serial/logs' +MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || 'localhost' +MQ_PORT = ENV['MQ_PORT'] || 5672 def main(hostname) start_time = record_runtime_log(hostname) @@ -99,7 +104,26 @@ def save_pid(arr) f.close end +def reboot_deadline_machine(hostname) + mq = MQClient.new + queue = mq.queue(hostname, {:durable => true}) + queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| + puts msg + machine_info = JSON.parse(msg) + job_id = machine_info['job_id'] + system "pkill #{job_id};echo $?" + mq.ack(info) + end +end + def multiqemu + Process.fork do + reboot_deadline_machine(HOSTNAME) + rescue StandardError => e + puts e.backtrace + sleep 5 + end + pids = [] NR_VM.to_i.times do |i| pid = Process.fork do -- 2.23.0

1 0

[PATCH v3 compass-ci 1/2] lib/mq_client.rb: add new class for rabbitmq
by Wu Zhende 10 Mar '21

10 Mar '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/mq_client.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100755 lib/mq_client.rb diff --git a/lib/mq_client.rb b/lib/mq_client.rb new file mode 100755 index 0000000..63618f9 --- /dev/null +++ b/lib/mq_client.rb @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'bunny' + +# client for rabbitmq +class MQClient + def initialize(hostname = 'localhost', port = '5672') + @conn = Bunny.new(hostname: hostname, port: port) + @conn.start + @channel = @conn.create_channel + end + + def queue(queue_name, opts = {}) + @channel.queue(queue_name, opts) + end + + def ack(delivery_info) + @channel.ack(delivery_info.delivery_tag) + end +end -- 2.23.0

1 0

[PATCH lkp-tests] lib/bootstrap.sh: change $CCI_SRC to $lab
by Wang Yong 10 Mar '21

10 Mar '21

Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- lib/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 52c9da32a..98c30a26b 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -274,7 +274,7 @@ setup_hosts() is_docker && return suffix="sh.intel.com" - [ -n "${CCI_SRC}" ] && suffix="compass-ci.net" + [ -n "${lab}" ] && suffix="compass-ci.net" # /etc/hosts may be shared when it's NFSROOT and there is no obvious # way to detect if rootfs is already RAM based. So unconditionally -- 2.23.0

1 0

[PATCH v2 lkp-tests] add new type error
by Bai Jing 10 Mar '21

10 Mar '21

[input] Error: Unknown repo: 'powertools' [output] unknown-repo.powertools: 1 Signed-off-by: Bai Jing <799286817(a)qq.com> --- stats/openeuler_docker | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/stats/openeuler_docker b/stats/openeuler_docker index f85c9ca63..b3890f568 100755 --- a/stats/openeuler_docker +++ b/stats/openeuler_docker @@ -9,9 +9,15 @@ while (line = STDIN.gets) # Error: Unable to find a match: docker-registry when /Error: Unable to find a match: (.+)/ $1.split.each do |repo| - repo_set << repo + repo_set << repo end + # Error: Unknown repo: 'powertools' + when /Error: Unknown repo: (.+)/ + repo = $1.delete!("'") + puts "unknown-repo.#{repo}: 1" + puts "unknown-repo.#{repo}.message: #{line}" + # /bin/sh: passwd: command not found when /\/bin\/sh: (.+): command not found/ puts "sh.command-not-found.#{$1}: 1" -- 2.23.0

2 2