- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci 2/5] container/dracut-initrd: add handling of pre-mount stage into setup-dracut.sh
by Xu Xijian 10 Mar '21

10 Mar '21

Add the content of set-local-sysroot.sh into the tail of dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 849480d..d447251 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -18,3 +18,7 @@ cp -a /usr/local/bin/cifs-lib.sh /usr/lib/dracut/modules.d/95cifs/ cat overlay-lkp.sh >> /usr/lib/dracut/modules.d/90overlay-root/overlay-mount.sh sed -i "/install() {/a\ inst /usr/bin/awk" /usr/lib/dracut/modules.d/40network/module-setup.sh sed -i "/install() {/a\ inst /sbin/mke2fs" /usr/lib/dracut/modules.d/98dracut-systemd/module-setup.sh +pre_mount_file="/usr/lib/dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh" +pre_mount_last_line="$(eval 'sed -n \$p $pre_mount_file')" +[ "$pre_mount_last_line" = "exit 0" ] && sed -i '$d' "$pre_mount_file" +cat set-local-sysroot.sh >> "$pre_mount_file" -- 2.23.0

2 2

[PATCH compass-ci 5/5] src/lib/job.cr: add keywords src_lv_suffix and boot_lv_suffix
by Xu Xijian 10 Mar '21

10 Mar '21

1. src_lv_suffix: to combine into use_root_partition, which specifies a logical volume to be a local source rootfs. 2. boot_lv_suffix: to combine into save_root_partition, whose rootfs is generated from use_root_partition. This rootfs will be mounted when start an os and saved locally after shutdown. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/job.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0b4bee4..424dd31 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -89,6 +89,8 @@ class Job docker_image kernel_version linux_vmlinuz_path + src_lv_suffix + boot_lv_suffix ) macro method_missing(call) -- 2.23.0

2 1

[PATCH compass-ci 2/2] container/srv-http: add /srv/initrd/rpmbuild-pkg
by Li Ping 10 Mar '21

10 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/srv-http/start-initrd | 1 + 1 file changed, 1 insertion(+) diff --git a/container/srv-http/start-initrd b/container/srv-http/start-initrd index 7c73ea1..495cc25 100755 --- a/container/srv-http/start-initrd +++ b/container/srv-http/start-initrd @@ -12,6 +12,7 @@ cmd=( -p 11303:11300 -v /srv/initrd/qemu-image:/srv/initrd/qemu-image:ro -v /srv/initrd/dockerimage:/srv/initrd/dockerimage:ro + -v /srv/initrd/rpmbuild-pkg:/srv/initrd/rpmbuild-pkg:ro -v /etc/localtime:/etc/localtime:ro -d srv-http -- 2.23.0

1 0

[PATCH compass-ci 1/2] src/lib/job.cr: optimize the get_pkg_common_dir function
by Li Ping 10 Mar '21

10 Mar '21

[why] 1. rpm has no much to do with os_mount. add a judgment 2. optimize the code style Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/job.cr | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0b4bee4..a7674d4 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -258,23 +258,28 @@ class Job end def get_pkg_common_dir - pkg_style = nil + tmp_style = nil ["cci-makepkg", "cci-depends", "build-pkg", "rpmbuild-pkg"].each do |item| - pkg_style = @hash[item]? - break if pkg_style + tmp_style = @hash[item]? + break if tmp_style end - return nil unless pkg_style + return nil unless tmp_style + pkg_style = Hash(String, JSON::Any).new {|h, k| h[k] = JSON::Any.new(nil)} + pkg_style.merge!(tmp_style.as_h? || Hash(String, JSON::Any).new) - pkg_style = JSON.parse("{}") if pkg_style == nil + tmp_os = pkg_style["os"].as_s? || "#{os}" + tmp_os_arch = pkg_style["os_arch"].as_s? || "#{os_arch}" + tmp_os_version = pkg_style["os_version"].as_s? || "#{os_version}" - tmp_os = pkg_style["os"]? == nil ? "#{os}" : pkg_style["os"] - tmp_os_arch = pkg_style["os_arch"]? == nil ? "#{os_arch}" : pkg_style["os_arch"] - tmp_os_version = pkg_style["os_version"]? == nil ? "#{os_version}" : pkg_style["os_version"] + mount_type = pkg_style["os_mount"].as_s? || "#{os_mount}" + # same usage for client + mount_type = "nfs" if mount_type == "cifs" - tmp_os_mount = pkg_style["os_mount"]? == nil ? "#{os_mount}" : pkg_style["os_mount"] - mount_type = tmp_os_mount == "cifs" ? "nfs" : tmp_os_mount.dup - - common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + if @hash["rpmbuild-pkg"]? + common_dir = "#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + else + common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + end return common_dir end -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] providers/multi-qemu: add the function of restarting VMs
by Wu Zhende 10 Mar '21

10 Mar '21

Obtain the VM to be restarted from the mq and restart it Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- providers/multi-qemu | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/providers/multi-qemu b/providers/multi-qemu index c729c8e..fb4167a 100755 --- a/providers/multi-qemu +++ b/providers/multi-qemu @@ -5,6 +5,9 @@ require 'fileutils' require 'optparse' +require 'json' + +require '../lib/mq_client' opt = {} options = OptionParser.new do |opts| @@ -42,6 +45,8 @@ HOSTNAME = opt['hostname_prefix'] || "vm-2p8g.#{ENV['HOSTNAME']}" NR_VM = opt['nr_vm'] || 1 QUEUES = opt['queues'] || "#{ENV['HOSTNAME']}.#{RUBY_PLATFORM.split('-')[0]}" LOG_DIR = '/srv/cci/serial/logs' +MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || 'localhost' +MQ_PORT = ENV['MQ_PORT'] || 5672 def main(hostname) start_time = record_runtime_log(hostname) @@ -99,7 +104,41 @@ def save_pid(arr) f.close end +def reboot_deadline_machine(hostname) + mq_queue = get_mq_queue(hostname) + mq = MQClient.new + queue = mq.queue(mq_queue, {:durable => true}) + queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| + puts msg + machine_info = JSON.parse(msg) + job_id = machine_info['job_id'] + system "pkill #{job_id};echo $?" + mq.ack(info) + end +end + +def get_pid(testbox) + testbox =~ /.*-(\d+$)/ + $1 +end + +def get_mq_queue(hostname) + if hostname.include?('.') + hostname =~ /.*\.(.*)/ + else + hostname =~ /(.*)--.*/ + end + $1 || hostname +end + def multiqemu + Process.fork do + reboot_deadline_machine(HOSTNAME) + rescue StandardError => e + puts e.backtrace + sleep 5 + end + pids = [] NR_VM.to_i.times do |i| pid = Process.fork do -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] lib/mq_client.rb: add new class for rabbitmq
by Wu Zhende 10 Mar '21

10 Mar '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/mq_client.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100755 lib/mq_client.rb diff --git a/lib/mq_client.rb b/lib/mq_client.rb new file mode 100755 index 0000000..63618f9 --- /dev/null +++ b/lib/mq_client.rb @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'bunny' + +# client for rabbitmq +class MQClient + def initialize(hostname = 'localhost', port = '5672') + @conn = Bunny.new(hostname: hostname, port: port) + @conn.start + @channel = @conn.create_channel + end + + def queue(queue_name, opts = {}) + @channel.queue(queue_name, opts) + end + + def ack(delivery_info) + @channel.ack(delivery_info.delivery_tag) + end +end -- 2.23.0

1 0

[PATCH compass-ci] lib/mq_client.rb: add new class for rabbitmq
by Wu Zhende 10 Mar '21

10 Mar '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/mq_client.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 lib/mq_client.rb diff --git a/lib/mq_client.rb b/lib/mq_client.rb new file mode 100755 index 0000000..412d7e1 --- /dev/null +++ b/lib/mq_client.rb @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "bunny" + +class MQClient + def initialize(hostname = "localhost", port = "5672") + @conn = Bunny.new(hostname: hostname, port: port) + @conn.start + @channel = @conn.create_channel + end + + def queue(queue_name, opts = {}) + @channel.queue(queue_name, opts) + end + + def ack(delivery_info) + @channel.ack(delivery_info.delivery_tag) + end +end -- 2.23.0

1 0

[PATCH compass-ci] container: fix cloning into 'deep-merge.cr' read error
by Cao Xueliang 10 Mar '21

10 Mar '21

The deep-merge repo has deleted in gitlab. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/crystal-shards/shard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/crystal-shards/shard.yml b/container/crystal-shards/shard.yml index 44d175d..ca8f72e 100644 --- a/container/crystal-shards/shard.yml +++ b/container/crystal-shards/shard.yml @@ -25,7 +25,7 @@ dependencies: github: icyleaf/any_merge branch: master deep-merge: - gitlab: peterhoeg/deep-merge.cr + github: peterhoeg/deep-merge.cr json_on_steroids: github: anykeyh/json_on_steroids rate_limiter: -- 2.23.0

1 0

[PATCH lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 10 Mar '21

10 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 84 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 tests/ansible_test diff --git a/tests/ansible_test b/tests/ansible_test new file mode 100755 index 000000000..36164bcae --- /dev/null +++ b/tests/ansible_test @@ -0,0 +1,84 @@ +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + ansible_host=localhost + echo $ansible_host + + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] && { + echo "Ansible is successfully deployed!" + } + } +} + +download_ansible_role() +{ + + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git || exit 1 + # git clone git://$GIT_SERVER/$ansible_repo.git || exit + # ansible-galaxy install $username.$rolename + if [ $? -eq 0 ]; then + echo "$benchmark is successfully installed" + else + echo "Please check whether your ansible_repo is correct!" + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: $ansible_host + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook failed to run!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

1 0

[PATCH lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 10 Mar '21

10 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 217 ++++++++++++++++++--------------------------- 1 file changed, 84 insertions(+), 133 deletions(-) diff --git a/tests/ansible_test b/tests/ansible_test index bb2ee9bbc..36164bcae 100755 --- a/tests/ansible_test +++ b/tests/ansible_test @@ -1,133 +1,84 @@ -#!/usr/bin/env ruby - -require 'json' - -def output_error(error_msg) - error_id = common_error_id error_msg - puts "error.#{error_id}: 1" - puts "error.#{error_id}.message: #{@ansible_failed_info}" -end - -def parse_msg_1(ansible_failed_json) - case ansible_failed_json.chomp - - # When running some plug-in, the file was not found - when /(No file was found when using (.+))\./ - output_error $1 - - # {"changed": false, "cmd": "csf -v", "msg": "[Errno 2] No such file or directory: b'csf': b'csf'", "rc": 2} - when /(No such file or directory: b'.+')/ - output_error $1 - - # The task includes an option with an undefined variable - # {"msg": "The task includes an option with an undefined variable. The error was: '__mysql_packages' is undefined - # The error appears to be in '/root/.ansible/roles/ansible-role-mysql/tasks/variables.yml': line 11, column 3, - # but may be elsewhere in the file depending on the exact syntax problem. The offending line appears to be: - name: - # Define mysql_packages. here"} - when /(.+ is undefined)/ - output_error $1 - - # {"changed": true, "cmd": ["packaging/installer/install-required-packages.sh", "--non-interactive", "netdata"], - # "msg": "non-zero return code", "rc": 1} - when /(non-zero return code)/ - output_error $1 - - # Some service could not find the requested host - when /(Could not find the requested service .+): host/ - output_error $1 - - # Unable to start some services - when /(Unable to start service .+):/ - output_error $1 - end -end - -def parse_msg_2(ansible_failed_json) - case ansible_failed_json.chomp - - # {"changed": false, "msg": "Failed to download packages: Cannot download 7.10.2/filebeat-7.10.2-aarch64.rpm: - # All mirrors were tried", "results": []} - when /(Failed to download packages: Cannot download .+):/ - output_error $1 - - when /(Failure downloading .+)/ - output_error $1 - - # Failed to download the yum repodata - # {"changed": false, "msg": "Failed to download metadata for repo 'docker-ce-stable': Cannot download repomd.xml", - # "rc": 1, "results": []} - when /(Failed to download metadata for repo.+)':/ - output_error $1 - - # Some role in not sipported this system - # {"changed": false, "msg": "Distribution openEuler is not supported by this role!"} - when /(.+ is not supported .+)/ - output_error $1 - - # failed to create temporary content file: The read operation timed out. - when /(.+ The read operation timed out)/ - output_error $1 - - # {"msg": "The conditional check 'java_version is defined' failed. The error was: error while evaluating conditional - # (java_version is defined): {{ java_default_version }}: {{ _java_default_version[ansible_distribution ~ '-' ~ - # ansible_distribution_major_version] | default(_java_default_version[ansible_distribution]) }}: 'dict object' has - # no attribute 'openEuler'\n\nThe error appears to be in '/root/.ansible/roles/ansible-role-java/tasks/assert.yml': - # line 19, column 3, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line - # appears to be:\n\n\n- name: test if java_version is set correctly\n ^ here\n"} - when /(The conditional check .+ failed)\./ - output_error $1 - end -end - -def parse_failures(ansible_failed_json) - ansible_failed_json.each do |items| - case items.chomp - # Failed to install some of the specified packages. - # {"changed": false, "failures": ["No package erlang available."], "msg": - # "Failed to install some of the specified packages", "rc": 1, "results": []} - when /(No package .+ available)/ - output_error $1 - end - end -end - -def parse_message(ansible_failed_json) - case ansible_failed_json.chomp - - # Could not find or access ansible yaml - # {"ansible_facts": {}, "ansible_included_var_files": [], "changed": false, "message": - # "Could not find or access 'openEuler.yml' Searched in: - # /root/.ansible/roles/ansible-unzip/vars/openEuler.yml /root/.ansible/roles/ansible-unzip/openEuler.yml i - # /root/.ansible/roles/ansible-unzip/tasks/vars/openEuler.yml /root/.ansible/roles/ansible-unzip/tasks/openEuler.yml - # /root/.ansible/vars/openEuler.yml /root/.ansible/openEuler.yml on the Ansible Controller. - # If you are using a module and expect the file to exist on the remote, see the remote_src option"} - when /(Could not find or access .+')/ - output_error $1 - end -end - -def common_error_id(line) - line = line.chomp - line.gsub!(/ /, '-') - line.gsub!(/:/, '') # error.:-'__mysql_packages'-is-undefined: 1 - line.gsub!(/,/, '') - line.gsub!(/'/, '') # Failed-to-download-metadata-for-repo-'dockerrepo: 1 - line.gsub!(/!/, '') # Distribution-openEuler-is-not-supported-by-this-role!: 1 - line.gsub!(/{%/, ' ') - line.gsub!(/%}/, ' ') - line -end - -while (line = STDIN.gets) - next unless line =~ /(FAILED!|failed:).*=>(.*)/ - - @ansible_failed_info = $2 - next if @ansible_failed_info.empty? - - ansible_failed_json = JSON.parse @ansible_failed_info - - parse_msg_1 ansible_failed_json['msg'] unless ansible_failed_json['msg'].nil? - parse_msg_2 ansible_failed_json['msg'] unless ansible_failed_json['msg'].nil? - parse_failures ansible_failed_json['failures'] unless ansible_failed_json['failures'].nil? - parse_message ansible_failed_json['message'] unless ansible_failed_json['message'].nil? -end +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + ansible_host=localhost + echo $ansible_host + + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] && { + echo "Ansible is successfully deployed!" + } + } +} + +download_ansible_role() +{ + + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git || exit 1 + # git clone git://$GIT_SERVER/$ansible_repo.git || exit + # ansible-galaxy install $username.$rolename + if [ $? -eq 0 ]; then + echo "$benchmark is successfully installed" + else + echo "Please check whether your ansible_repo is correct!" + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: $ansible_host + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook failed to run!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

1 1