March 2021 - Compass-ci - mailweb.openeuler.org

[PATCH compass-ci] container/srv-http: change service ports
by Lu Weitao 16 Mar '21

16 Mar '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/srv-http/start-cci | 2 +- container/srv-http/start-git | 2 +- container/srv-http/start-initrd | 2 +- container/srv-http/start-os | 2 +- container/srv-http/start-pub | 2 +- container/srv-http/start-result | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/container/srv-http/start-cci b/container/srv-http/start-cci index 8486476..00d8e4c 100755 --- a/container/srv-http/start-cci +++ b/container/srv-http/start-cci @@ -9,7 +9,7 @@ cmd=( docker run --restart=always --name srv-http-cci - -p 11307:11300 + -p 20011:11300 -v /srv/cci/libvirt-xml:/srv/cci/libvirt-xml:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-git b/container/srv-http/start-git index f2292bf..c1181a7 100755 --- a/container/srv-http/start-git +++ b/container/srv-http/start-git @@ -9,7 +9,7 @@ cmd=( docker run --restart=always --name srv-http-git - -p 11305:11300 + -p 20010:11300 -v /srv/git/archlinux:/srv/git/archlinux:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-initrd b/container/srv-http/start-initrd index 495cc25..466ec1b 100755 --- a/container/srv-http/start-initrd +++ b/container/srv-http/start-initrd @@ -9,7 +9,7 @@ cmd=( docker run --restart=always --name srv-http-initrd - -p 11303:11300 + -p 20008:11300 -v /srv/initrd/qemu-image:/srv/initrd/qemu-image:ro -v /srv/initrd/dockerimage:/srv/initrd/dockerimage:ro -v /srv/initrd/rpmbuild-pkg:/srv/initrd/rpmbuild-pkg:ro diff --git a/container/srv-http/start-os b/container/srv-http/start-os index bb344cf..d39e04e 100755 --- a/container/srv-http/start-os +++ b/container/srv-http/start-os @@ -9,7 +9,7 @@ cmd=( docker run --restart=always --name srv-http-os - -p 11304:11300 + -p 20009:11300 -v /srv/os:/srv/os:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-pub b/container/srv-http/start-pub index dc1864c..f42b117 100755 --- a/container/srv-http/start-pub +++ b/container/srv-http/start-pub @@ -9,7 +9,7 @@ cmd=( docker run --restart=always --name srv-http-pub - -p 11302:11300 + -p 20006:11300 -v /srv/pub:/srv/pub:ro -v /etc/localtime:/etc/localtime:ro -d diff --git a/container/srv-http/start-result b/container/srv-http/start-result index f9ecc97..bf0287d 100755 --- a/container/srv-http/start-result +++ b/container/srv-http/start-result @@ -9,7 +9,7 @@ cmd=( docker run --restart=always --name srv-http-result - -p 11300:11300 + -p 20007:11300 -v /srv/result:/srv/result:ro -v /etc/localtime:/etc/localtime:ro -d -- 2.23.0

1 0

[PATCH compass-ci] src/lib/job.cr: open the path for uploading rpm packages.
by Li Ping 16 Mar '21

16 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/job.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index f8cfa97..40aaa62 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -278,7 +278,7 @@ class Job mount_type = "nfs" if mount_type == "cifs" common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" - common_dir = "#{tmp_os}-#{tmp_os_version}/#{tmp_os_arch}" if @hash.has_key?("rpmbuild-pkg") + common_dir = "#{tmp_os}-#{tmp_os_version}" if @hash.has_key?("rpmbuild-pkg") return common_dir end @@ -293,7 +293,7 @@ class Job elsif @hash["cci-depends"]? package_dir = ",/initrd/deps/#{common_dir}/#{@hash["cci-depends"]["benchmark"]}" elsif @hash["rpmbuild-pkg"]? - package_dir = ",/initrd/rpmbuild-pkg/#{common_dir}/#{@hash["upstream_repo"].to_s.split("/")[-1]}" + package_dir = ",/initrd/rpmbuild-pkg/#{common_dir}" elsif @hash["build-pkg"]? if @hash["pkgbuild_repo"].to_s =~ /(packages|community)\/\// package_name = @hash["pkgbuild_repo"].to_s.split("/")[-2] -- 2.23.0

1 0

[PATCH v3 lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 16 Mar '21

16 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 83 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100755 tests/ansible_test diff --git a/tests/ansible_test b/tests/ansible_test new file mode 100755 index 000000000..9ecb4d2c5 --- /dev/null +++ b/tests/ansible_test @@ -0,0 +1,83 @@ +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + ansible_host=localhost + echo $ansible_host + + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] || { + echo "Ansible failed to install." + exit 1 + } + } +} + +download_ansible_role() +{ + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git + if [ $? -eq 0 ]; then + echo "$benchmark is successfully installed" + else + echo "Please check whether your ansible_repo is correct!" + exit 1 + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: $ansible_host + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook is running failed!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

1 1

[PATCH v4 compass-ci 2/2] multi-docker: add reboot function
by Wu Zhende 16 Mar '21

16 Mar '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- providers/docker/docker.rb | 34 +++++++++++++++++++++++++++++++++- providers/docker/run.sh | 2 ++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/providers/docker/docker.rb b/providers/docker/docker.rb index f4e954e..9d6854f 100755 --- a/providers/docker/docker.rb +++ b/providers/docker/docker.rb @@ -7,6 +7,8 @@ require 'open-uri' require 'json' require 'set' require 'fileutils' + +require '../lib/mq_client' require_relative '../../container/defconfig' BASE_DIR = '/srv/dc' @@ -19,6 +21,8 @@ defaults = relevant_defaults(names) SCHED_HOST = defaults['SCHED_HOST'] || '172.17.0.1' SCHED_PORT = defaults['SCHED_PORT'] || 3000 LOG_DIR = '/srv/cci/serial/logs' +MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || 'localhost' +MQ_PORT = ENV['MQ_PORT'] || 5672 def get_url(hostname) "http://#{SCHED_HOST}:#{SCHED_PORT}/boot.container/hostname/#{hostname}" @@ -84,7 +88,8 @@ def start_container(hostname, load_path, hash) docker_image = hash['docker_image'] system "#{ENV['CCI_SRC']}/sbin/docker-pull #{docker_image}" system( - { 'hostname' => hostname, + { 'job_id' => hash['job_id'], + 'hostname' => hostname, 'docker_image' => docker_image, 'load_path' => load_path, 'log_dir' => "#{LOG_DIR}/#{hostname}" @@ -145,6 +150,29 @@ def loop_main(hostname, queues) end end +def loop_reboot_docker(hostname) + loop do + begin + reboot_docker(hostname) + rescue StandardError => e + puts e.backtrace + sleep 5 + end + end +end + +def reboot_docker(hostname) + mq = MQClient.new(MQ_HOST, MQ_PORT) + queue = mq.queue(hostname, {:durable => true}) + queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| + puts msg + machine_info = JSON.parse(msg) + job_id = machine_info['job_id'] + system "docker rm -f #{job_id};echo $?" + mq.ack(info) + end +end + def save_pid(pids) FileUtils.cd("#{ENV['CCI_SRC']}/providers") f = File.new('dc.pid', 'a') @@ -153,6 +181,10 @@ def save_pid(pids) end def multi_docker(hostname, nr_container, queues) + Process.fork do + loop_reboot_docker(hostname) + end + pids = [] nr_container.to_i.times do |i| pid = Process.fork do diff --git a/providers/docker/run.sh b/providers/docker/run.sh index 2dfe36d..80950dd 100755 --- a/providers/docker/run.sh +++ b/providers/docker/run.sh @@ -4,6 +4,7 @@ . $LKP_SRC/lib/yaml.sh +: ${job_id:=$$} : ${docker_image:="centos:7"} : ${load_path:="${HOME}/jobs"} : ${hostname:="dc-1g-1"} @@ -23,6 +24,7 @@ busybox_path=$(command -v busybox) cmd=( docker run --rm + --name ${job_id} --hostname $host.compass-ci.net -m $memory --tmpfs /tmp:rw,exec,nosuid,nodev -- 2.23.0

1 0

[PATCH v4 compass-ci 1/2] scheduler/find_job_boot.cr: docker add job_id
by Wu Zhende 16 Mar '21

16 Mar '21

When the startup parameter is docker, the value of job_id is returned. Set job_id as docker name, so we can use "docker rm -f #{job_id}" to kill the docker. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/scheduler/find_job_boot.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index ce1a2f8..a45a6d9 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -144,6 +144,7 @@ class Sched private def get_boot_container(job : Job) response = Hash(String, String).new + response["job_id"] = job.id.to_s response["docker_image"] = "#{job.docker_image}" response["lkp"] = "http://#{INITRD_HTTP_HOST}:#{INITRD_HTTP_PORT}" + JobHelper.service_path("#{SRV_INITRD}/lkp/#{job.lkp_initrd_user}/lkp-#{job.arch}.cgz") -- 2.23.0

1 0

[PATCH v4 compass-ci] providers/multi-qemu: optimize
by Wu Zhende 16 Mar '21

16 Mar '21

1. optimize function name 2. the host and port are transferred when the MQClient is initalized 3. reboot vm cyclically Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- providers/multi-qemu | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/providers/multi-qemu b/providers/multi-qemu index 5979cc9..d3f8406 100755 --- a/providers/multi-qemu +++ b/providers/multi-qemu @@ -104,8 +104,19 @@ def save_pid(arr) f.close end -def reboot_deadline_machine(hostname) - mq = MQClient.new +def loop_reboot_vm(hostname) + loop do + begin + reboot_vm(hostname) + rescue StandardError => e + puts e.backtrace + sleep 5 + end + end +end + +def reboot_vm(hostname) + mq = MQClient.new(MQ_HOST, MQ_PORT) queue = mq.queue(hostname, {:durable => true}) queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| puts msg @@ -118,10 +129,7 @@ end def multiqemu Process.fork do - reboot_deadline_machine(HOSTNAME) - rescue StandardError => e - puts e.backtrace - sleep 5 + loop_reboot_vm(HOSTNAME) end pids = [] -- 2.23.0

1 0

[PATCH compass-ci] container/webhook: change the service port
by Li Yuanchao 15 Mar '21

15 Mar '21

For the port policy. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- container/webhook/start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/webhook/start b/container/webhook/start index 373396a..1d121a4 100755 --- a/container/webhook/start +++ b/container/webhook/start @@ -12,7 +12,7 @@ cmd = %w[ --restart=always --name webhook -d - -p 11301:11301 + -p 20005:20005 node:webhook ] -- 2.23.0

1 0

[PATCH lkp-tests] tests/build-pkg: change the local git repo path
by Lin Jiaxin 15 Mar '21

15 Mar '21

As the path of repo mirrors under /srv/git changes, introduce upstream_dir to get new path. Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- tests/build-pkg | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/build-pkg b/tests/build-pkg index 43932865..350d8fdf 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -6,6 +6,7 @@ # - pkg_ver # - config # upstream_url +# upstream_dir # We'll use the below global env vars, but not treat them as parameters to avoid # duplicates in result path like /openeuler-20.03-aarch64/openeuler-aarch64-cifs-2003/ @@ -123,9 +124,9 @@ replace_source() do echo "$url" | egrep 'git\+|\.git$' && { [ -n "$repo_dir" ] && { - url="${repo_dir}git://${LKP_SERVER}/${upstream_repo}#commit=${upstream_commit}" + url="${repo_dir}git://${LKP_SERVER}/${upstream_dir}/${upstream_repo}#commit=${upstream_commit}" [ -n "$git_tag" ] && { - url="${repo_dir}git://${LKP_SERVER}/${upstream_repo}#tag=${git_tag}" + url="${repo_dir}git://${LKP_SERVER}/${upstream_dir}/${upstream_repo}#tag=${git_tag}" } } } -- 2.23.0

1 0

[PATCH compass-ci 1/2] sbin/auto_submit: change the upstream_repo path to submit
by Lin Jiaxin 15 Mar '21

15 Mar '21

Reference: 5d6cea1bd531409f0444abf14088fdecd8b08c2d Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- sbin/auto_submit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sbin/auto_submit b/sbin/auto_submit index faebb2f..a27075b 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -33,7 +33,7 @@ class AutoSubmit git_repo = newrefs_info['git_repo'] puts "git_repo : #{git_repo}" url = Array(newrefs_info['url'])[0] - submit_argv = ["#{ENV['LKP_SRC']}/sbin/submit upstream_repo=#{git_repo} upstream_url=#{url}"] + submit_argv = ["#{ENV['LKP_SRC']}/sbin/submit upstream_repo=#{newrefs_info['belong']}/#{git_repo} upstream_url=#{url}"] if newrefs_info['pkgbuild_repo'] pkgbuild_repo = get_pkgbuild_repo(newrefs_info['pkgbuild_repo']) -- 2.23.0

2 2

[PATCH compass-ci] optimize doc
by Luan Shengde 15 Mar '21

15 Mar '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- doc/manual/build-lkp-tests-container.zh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/build-lkp-tests-container.zh.md b/doc/manual/build-lkp-tests-container.zh.md index 10a1611..407ddbf 100644 --- a/doc/manual/build-lkp-tests-container.zh.md +++ b/doc/manual/build-lkp-tests-container.zh.md @@ -60,4 +60,4 @@ submit命令： - 了解更多submit命令的使用方法，请参考：[submit user manual](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/su… + 了解更多submit命令的使用方法，请参考：[submit用户手册](https://gitee.com/wu_fengguang/compass-c… -- 2.23.0

1 0