- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci 2/5] container/dracut-initrd: add handling of pre-mount stage into setup-dracut.sh
by Xu Xijian 10 Mar '21

10 Mar '21

Add the content of set-local-sysroot.sh into the tail of dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 849480d..d447251 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -18,3 +18,7 @@ cp -a /usr/local/bin/cifs-lib.sh /usr/lib/dracut/modules.d/95cifs/ cat overlay-lkp.sh >> /usr/lib/dracut/modules.d/90overlay-root/overlay-mount.sh sed -i "/install() {/a\ inst /usr/bin/awk" /usr/lib/dracut/modules.d/40network/module-setup.sh sed -i "/install() {/a\ inst /sbin/mke2fs" /usr/lib/dracut/modules.d/98dracut-systemd/module-setup.sh +pre_mount_file="/usr/lib/dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh" +pre_mount_last_line="$(eval 'sed -n \$p $pre_mount_file')" +[ "$pre_mount_last_line" = "exit 0" ] && sed -i '$d' "$pre_mount_file" +cat set-local-sysroot.sh >> "$pre_mount_file" -- 2.23.0

2 2

[PATCH compass-ci 5/5] src/lib/job.cr: add keywords src_lv_suffix and boot_lv_suffix
by Xu Xijian 10 Mar '21

10 Mar '21

1. src_lv_suffix: to combine into use_root_partition, which specifies a logical volume to be a local source rootfs. 2. boot_lv_suffix: to combine into save_root_partition, whose rootfs is generated from use_root_partition. This rootfs will be mounted when start an os and saved locally after shutdown. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/job.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0b4bee4..424dd31 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -89,6 +89,8 @@ class Job docker_image kernel_version linux_vmlinuz_path + src_lv_suffix + boot_lv_suffix ) macro method_missing(call) -- 2.23.0

2 1

[PATCH compass-ci 2/2] container/srv-http: add /srv/initrd/rpmbuild-pkg
by Li Ping 10 Mar '21

10 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- container/srv-http/start-initrd | 1 + 1 file changed, 1 insertion(+) diff --git a/container/srv-http/start-initrd b/container/srv-http/start-initrd index 7c73ea1..495cc25 100755 --- a/container/srv-http/start-initrd +++ b/container/srv-http/start-initrd @@ -12,6 +12,7 @@ cmd=( -p 11303:11300 -v /srv/initrd/qemu-image:/srv/initrd/qemu-image:ro -v /srv/initrd/dockerimage:/srv/initrd/dockerimage:ro + -v /srv/initrd/rpmbuild-pkg:/srv/initrd/rpmbuild-pkg:ro -v /etc/localtime:/etc/localtime:ro -d srv-http -- 2.23.0

1 0

[PATCH compass-ci 1/2] src/lib/job.cr: optimize the get_pkg_common_dir function
by Li Ping 10 Mar '21

10 Mar '21

[why] 1. rpm has no much to do with os_mount. add a judgment 2. optimize the code style Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/job.cr | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0b4bee4..a7674d4 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -258,23 +258,28 @@ class Job end def get_pkg_common_dir - pkg_style = nil + tmp_style = nil ["cci-makepkg", "cci-depends", "build-pkg", "rpmbuild-pkg"].each do |item| - pkg_style = @hash[item]? - break if pkg_style + tmp_style = @hash[item]? + break if tmp_style end - return nil unless pkg_style + return nil unless tmp_style + pkg_style = Hash(String, JSON::Any).new {|h, k| h[k] = JSON::Any.new(nil)} + pkg_style.merge!(tmp_style.as_h? || Hash(String, JSON::Any).new) - pkg_style = JSON.parse("{}") if pkg_style == nil + tmp_os = pkg_style["os"].as_s? || "#{os}" + tmp_os_arch = pkg_style["os_arch"].as_s? || "#{os_arch}" + tmp_os_version = pkg_style["os_version"].as_s? || "#{os_version}" - tmp_os = pkg_style["os"]? == nil ? "#{os}" : pkg_style["os"] - tmp_os_arch = pkg_style["os_arch"]? == nil ? "#{os_arch}" : pkg_style["os_arch"] - tmp_os_version = pkg_style["os_version"]? == nil ? "#{os_version}" : pkg_style["os_version"] + mount_type = pkg_style["os_mount"].as_s? || "#{os_mount}" + # same usage for client + mount_type = "nfs" if mount_type == "cifs" - tmp_os_mount = pkg_style["os_mount"]? == nil ? "#{os_mount}" : pkg_style["os_mount"] - mount_type = tmp_os_mount == "cifs" ? "nfs" : tmp_os_mount.dup - - common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + if @hash["rpmbuild-pkg"]? + common_dir = "#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + else + common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + end return common_dir end -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] providers/multi-qemu: add the function of restarting VMs
by Wu Zhende 10 Mar '21

10 Mar '21

Obtain the VM to be restarted from the mq and restart it Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- providers/multi-qemu | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/providers/multi-qemu b/providers/multi-qemu index c729c8e..fb4167a 100755 --- a/providers/multi-qemu +++ b/providers/multi-qemu @@ -5,6 +5,9 @@ require 'fileutils' require 'optparse' +require 'json' + +require '../lib/mq_client' opt = {} options = OptionParser.new do |opts| @@ -42,6 +45,8 @@ HOSTNAME = opt['hostname_prefix'] || "vm-2p8g.#{ENV['HOSTNAME']}" NR_VM = opt['nr_vm'] || 1 QUEUES = opt['queues'] || "#{ENV['HOSTNAME']}.#{RUBY_PLATFORM.split('-')[0]}" LOG_DIR = '/srv/cci/serial/logs' +MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || 'localhost' +MQ_PORT = ENV['MQ_PORT'] || 5672 def main(hostname) start_time = record_runtime_log(hostname) @@ -99,7 +104,41 @@ def save_pid(arr) f.close end +def reboot_deadline_machine(hostname) + mq_queue = get_mq_queue(hostname) + mq = MQClient.new + queue = mq.queue(mq_queue, {:durable => true}) + queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| + puts msg + machine_info = JSON.parse(msg) + job_id = machine_info['job_id'] + system "pkill #{job_id};echo $?" + mq.ack(info) + end +end + +def get_pid(testbox) + testbox =~ /.*-(\d+$)/ + $1 +end + +def get_mq_queue(hostname) + if hostname.include?('.') + hostname =~ /.*\.(.*)/ + else + hostname =~ /(.*)--.*/ + end + $1 || hostname +end + def multiqemu + Process.fork do + reboot_deadline_machine(HOSTNAME) + rescue StandardError => e + puts e.backtrace + sleep 5 + end + pids = [] NR_VM.to_i.times do |i| pid = Process.fork do -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] lib/mq_client.rb: add new class for rabbitmq
by Wu Zhende 10 Mar '21

10 Mar '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/mq_client.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100755 lib/mq_client.rb diff --git a/lib/mq_client.rb b/lib/mq_client.rb new file mode 100755 index 0000000..63618f9 --- /dev/null +++ b/lib/mq_client.rb @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'bunny' + +# client for rabbitmq +class MQClient + def initialize(hostname = 'localhost', port = '5672') + @conn = Bunny.new(hostname: hostname, port: port) + @conn.start + @channel = @conn.create_channel + end + + def queue(queue_name, opts = {}) + @channel.queue(queue_name, opts) + end + + def ack(delivery_info) + @channel.ack(delivery_info.delivery_tag) + end +end -- 2.23.0

1 0

[PATCH compass-ci] lib/mq_client.rb: add new class for rabbitmq
by Wu Zhende 10 Mar '21

10 Mar '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/mq_client.rb | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 lib/mq_client.rb diff --git a/lib/mq_client.rb b/lib/mq_client.rb new file mode 100755 index 0000000..412d7e1 --- /dev/null +++ b/lib/mq_client.rb @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "bunny" + +class MQClient + def initialize(hostname = "localhost", port = "5672") + @conn = Bunny.new(hostname: hostname, port: port) + @conn.start + @channel = @conn.create_channel + end + + def queue(queue_name, opts = {}) + @channel.queue(queue_name, opts) + end + + def ack(delivery_info) + @channel.ack(delivery_info.delivery_tag) + end +end -- 2.23.0

1 0

[PATCH compass-ci] container: fix cloning into 'deep-merge.cr' read error
by Cao Xueliang 10 Mar '21

10 Mar '21

The deep-merge repo has deleted in gitlab. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/crystal-shards/shard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/crystal-shards/shard.yml b/container/crystal-shards/shard.yml index 44d175d..ca8f72e 100644 --- a/container/crystal-shards/shard.yml +++ b/container/crystal-shards/shard.yml @@ -25,7 +25,7 @@ dependencies: github: icyleaf/any_merge branch: master deep-merge: - gitlab: peterhoeg/deep-merge.cr + github: peterhoeg/deep-merge.cr json_on_steroids: github: anykeyh/json_on_steroids rate_limiter: -- 2.23.0

1 0

[PATCH lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 10 Mar '21

10 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 84 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 tests/ansible_test diff --git a/tests/ansible_test b/tests/ansible_test new file mode 100755 index 000000000..36164bcae --- /dev/null +++ b/tests/ansible_test @@ -0,0 +1,84 @@ +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + ansible_host=localhost + echo $ansible_host + + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] && { + echo "Ansible is successfully deployed!" + } + } +} + +download_ansible_role() +{ + + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git || exit 1 + # git clone git://$GIT_SERVER/$ansible_repo.git || exit + # ansible-galaxy install $username.$rolename + if [ $? -eq 0 ]; then + echo "$benchmark is successfully installed" + else + echo "Please check whether your ansible_repo is correct!" + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: $ansible_host + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook failed to run!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

1 0

[PATCH lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 10 Mar '21

10 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 217 ++++++++++++++++++--------------------------- 1 file changed, 84 insertions(+), 133 deletions(-) diff --git a/tests/ansible_test b/tests/ansible_test index bb2ee9bbc..36164bcae 100755 --- a/tests/ansible_test +++ b/tests/ansible_test @@ -1,133 +1,84 @@ -#!/usr/bin/env ruby - -require 'json' - -def output_error(error_msg) - error_id = common_error_id error_msg - puts "error.#{error_id}: 1" - puts "error.#{error_id}.message: #{@ansible_failed_info}" -end - -def parse_msg_1(ansible_failed_json) - case ansible_failed_json.chomp - - # When running some plug-in, the file was not found - when /(No file was found when using (.+))\./ - output_error $1 - - # {"changed": false, "cmd": "csf -v", "msg": "[Errno 2] No such file or directory: b'csf': b'csf'", "rc": 2} - when /(No such file or directory: b'.+')/ - output_error $1 - - # The task includes an option with an undefined variable - # {"msg": "The task includes an option with an undefined variable. The error was: '__mysql_packages' is undefined - # The error appears to be in '/root/.ansible/roles/ansible-role-mysql/tasks/variables.yml': line 11, column 3, - # but may be elsewhere in the file depending on the exact syntax problem. The offending line appears to be: - name: - # Define mysql_packages. here"} - when /(.+ is undefined)/ - output_error $1 - - # {"changed": true, "cmd": ["packaging/installer/install-required-packages.sh", "--non-interactive", "netdata"], - # "msg": "non-zero return code", "rc": 1} - when /(non-zero return code)/ - output_error $1 - - # Some service could not find the requested host - when /(Could not find the requested service .+): host/ - output_error $1 - - # Unable to start some services - when /(Unable to start service .+):/ - output_error $1 - end -end - -def parse_msg_2(ansible_failed_json) - case ansible_failed_json.chomp - - # {"changed": false, "msg": "Failed to download packages: Cannot download 7.10.2/filebeat-7.10.2-aarch64.rpm: - # All mirrors were tried", "results": []} - when /(Failed to download packages: Cannot download .+):/ - output_error $1 - - when /(Failure downloading .+)/ - output_error $1 - - # Failed to download the yum repodata - # {"changed": false, "msg": "Failed to download metadata for repo 'docker-ce-stable': Cannot download repomd.xml", - # "rc": 1, "results": []} - when /(Failed to download metadata for repo.+)':/ - output_error $1 - - # Some role in not sipported this system - # {"changed": false, "msg": "Distribution openEuler is not supported by this role!"} - when /(.+ is not supported .+)/ - output_error $1 - - # failed to create temporary content file: The read operation timed out. - when /(.+ The read operation timed out)/ - output_error $1 - - # {"msg": "The conditional check 'java_version is defined' failed. The error was: error while evaluating conditional - # (java_version is defined): {{ java_default_version }}: {{ _java_default_version[ansible_distribution ~ '-' ~ - # ansible_distribution_major_version] | default(_java_default_version[ansible_distribution]) }}: 'dict object' has - # no attribute 'openEuler'\n\nThe error appears to be in '/root/.ansible/roles/ansible-role-java/tasks/assert.yml': - # line 19, column 3, but may\nbe elsewhere in the file depending on the exact syntax problem.\n\nThe offending line - # appears to be:\n\n\n- name: test if java_version is set correctly\n ^ here\n"} - when /(The conditional check .+ failed)\./ - output_error $1 - end -end - -def parse_failures(ansible_failed_json) - ansible_failed_json.each do |items| - case items.chomp - # Failed to install some of the specified packages. - # {"changed": false, "failures": ["No package erlang available."], "msg": - # "Failed to install some of the specified packages", "rc": 1, "results": []} - when /(No package .+ available)/ - output_error $1 - end - end -end - -def parse_message(ansible_failed_json) - case ansible_failed_json.chomp - - # Could not find or access ansible yaml - # {"ansible_facts": {}, "ansible_included_var_files": [], "changed": false, "message": - # "Could not find or access 'openEuler.yml' Searched in: - # /root/.ansible/roles/ansible-unzip/vars/openEuler.yml /root/.ansible/roles/ansible-unzip/openEuler.yml i - # /root/.ansible/roles/ansible-unzip/tasks/vars/openEuler.yml /root/.ansible/roles/ansible-unzip/tasks/openEuler.yml - # /root/.ansible/vars/openEuler.yml /root/.ansible/openEuler.yml on the Ansible Controller. - # If you are using a module and expect the file to exist on the remote, see the remote_src option"} - when /(Could not find or access .+')/ - output_error $1 - end -end - -def common_error_id(line) - line = line.chomp - line.gsub!(/ /, '-') - line.gsub!(/:/, '') # error.:-'__mysql_packages'-is-undefined: 1 - line.gsub!(/,/, '') - line.gsub!(/'/, '') # Failed-to-download-metadata-for-repo-'dockerrepo: 1 - line.gsub!(/!/, '') # Distribution-openEuler-is-not-supported-by-this-role!: 1 - line.gsub!(/{%/, ' ') - line.gsub!(/%}/, ' ') - line -end - -while (line = STDIN.gets) - next unless line =~ /(FAILED!|failed:).*=>(.*)/ - - @ansible_failed_info = $2 - next if @ansible_failed_info.empty? - - ansible_failed_json = JSON.parse @ansible_failed_info - - parse_msg_1 ansible_failed_json['msg'] unless ansible_failed_json['msg'].nil? - parse_msg_2 ansible_failed_json['msg'] unless ansible_failed_json['msg'].nil? - parse_failures ansible_failed_json['failures'] unless ansible_failed_json['failures'].nil? - parse_message ansible_failed_json['message'] unless ansible_failed_json['message'].nil? -end +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + ansible_host=localhost + echo $ansible_host + + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] && { + echo "Ansible is successfully deployed!" + } + } +} + +download_ansible_role() +{ + + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git || exit 1 + # git clone git://$GIT_SERVER/$ansible_repo.git || exit + # ansible-galaxy install $username.$rolename + if [ $? -eq 0 ]; then + echo "$benchmark is successfully installed" + else + echo "Please check whether your ansible_repo is correct!" + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: $ansible_host + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook failed to run!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

1 1

[PATCH lab-z9] add dc-16/32g and vm-2p32g
by Cao Xueliang 10 Mar '21

10 Mar '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- hosts/dc-16g | 2 ++ hosts/dc-32g | 2 ++ hosts/vm-2p32g | 7 +++++++ 3 files changed, 11 insertions(+) create mode 100644 hosts/dc-16g create mode 100644 hosts/dc-32g create mode 100644 hosts/vm-2p32g diff --git a/hosts/dc-16g b/hosts/dc-16g new file mode 100644 index 0000000..f366892 --- /dev/null +++ b/hosts/dc-16g @@ -0,0 +1,2 @@ +arch: aarch64 +memory: 16g diff --git a/hosts/dc-32g b/hosts/dc-32g new file mode 100644 index 0000000..4006e66 --- /dev/null +++ b/hosts/dc-32g @@ -0,0 +1,2 @@ +arch: aarch64 +memory: 32g diff --git a/hosts/vm-2p32g b/hosts/vm-2p32g new file mode 100644 index 0000000..c7cc583 --- /dev/null +++ b/hosts/vm-2p32g @@ -0,0 +1,7 @@ +arch: aarch64 +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 32G +hdd_partitions: /dev/vda -- 2.23.0

1 0

[PATCH lkp-tests] hosts: add dc-32g file
by Lin Jiaxin 10 Mar '21

10 Mar '21

[Why] linux compile need big memory Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- hosts/dc-32g | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 hosts/dc-32g diff --git a/hosts/dc-32g b/hosts/dc-32g new file mode 100644 index 00000000..e9c80b37 --- /dev/null +++ b/hosts/dc-32g @@ -0,0 +1,2 @@ +nr_cpu: 8 +memory: 32g -- 2.23.0

1 0

[PATCH v3 compass-ci 1/2] sbin/result2stats: support call function from stats/script
by Lu Weitao 10 Mar '21

10 Mar '21

[Why] old logical: lkp-tests/stats/script < script_log > tmpfile lkp-tests/sbin/dump-stat < tmpfile > $script.json this flow will escape special characters [How] use return of function to keep original characters lkp-tests/stats/script.rb func(read(script_log)) --> hash compass-ci/lib/dump-stats.rb hash --> dump_stats() --> $script.json [Dependence] lkp-tests/stats/$script.rb, for example: file_name: lkp-tests/stats/proc-vmstats.rb function name: proc_vmstats() input: Array(String) ["time: 231231234", "nr_free_page: 89721", ...] output: Hash(String, Array(Number|String) | Number | String) { "time" => [231231234, 231231235], "nr_free_page" => [89721,88564], ... } or { "time" => 231231234, "nr_free_page" => 89721, ... } Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/stats.rb | 19 ++++++++++++++----- lib/stats_wrapper.rb | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/lib/stats.rb b/lib/stats.rb index c01d38d..94e370a 100644 --- a/lib/stats.rb +++ b/lib/stats.rb @@ -14,11 +14,16 @@ end def available_stats vailable_stats = [] + available_stats_func = [] Dir.open(LKP_SRC + '/stats').each do |file| - vailable_stats << file unless file.start_with?('.') + if file.end_with?('.rb') + available_stats_func << file.chomp('.rb') + else + vailable_stats << file unless file.start_with?('.') + end end - vailable_stats + [vailable_stats, available_stats_func] end def assign_default_stats(stats_list) @@ -32,13 +37,17 @@ class Stats def initialize(result_root) @result_root = result_root @job = load_yaml(File.join(result_root, 'job.yaml')) - @available_stats = available_stats + @available_stats, @available_stats_func = available_stats end def extract_stats stats_list = assign_stats_list stats_list.each do |stat| - StatsWrapper.wrapper(*stat) + if @available_stats_func.include?(stat) + StatsWrapper.wrapper_func(*stat) + else + StatsWrapper.wrapper(*stat) + end end end @@ -48,7 +57,7 @@ class Stats stats_list << ['time', @job['suite'] + '.time'] stats_list << 'stderr' @job.each_key do |k| - if @available_stats.include?(k) + if @available_stats.include?(k) || @available_stats_func.include?(k) stats_list << k end end diff --git a/lib/stats_wrapper.rb b/lib/stats_wrapper.rb index 8a6d6a9..faa81df 100644 --- a/lib/stats_wrapper.rb +++ b/lib/stats_wrapper.rb @@ -6,6 +6,11 @@ LKP_SRC ||= ENV['LKP_SRC'] || '/c/lkp-tests' require "#{LKP_SRC}/lib/log.rb" require 'tempfile' require 'English' +require_relative './dump_stat' + +Dir[File.expand_path("#{LKP_SRC}/stats/*.rb")].uniq.each do |file| + require file +end PROGRAM_DIR = "#{LKP_SRC}/stats" @@ -35,6 +40,27 @@ module StatsWrapper delete_log_package end + def self.wrapper_func(program, program_time = nil) + @program = program + @stats_group = program_time || program + @log = "#{RESULT_ROOT}/#{@stats_group}" + + return unless File.exist?("#{(a)log}.yaml") || pretreatment + + if File.exist?("#{(a)log}.yaml") + log_lines = read_log("#{(a)log}.yaml") + stat_result = parse_simple_log_yaml(log_lines) + else + log_lines = read_log(@log) + call_func_cmd = "#{(a)program.gsub('-', '_')}(log_lines)" # eg: proc_vmstats(log_lines) + stat_result = eval(call_func_cmd) + end + return unless DumpStat.dump_stat(@stats_group, stat_result) + + check_empty_json + delete_log_package + end + def self.pretreatment return unless available_program? return unless unzip_log @@ -261,3 +287,21 @@ module StatsWrapper File.delete(@tmpfile) if File.exist?(@tmpfile) end end + +# read line of log +# return Array(String) +def read_log(log_path) + return nil unless File.exist?(log_path) + + File.readlines(log_path) +end + +def parse_simple_log_yaml(log_lines) + result = Hash.new { |hash, key| hash[key] = [] } + log_lines.each do |line| + key, value = line.split(/:?\s+/) + result[key] << value + end + + result +end -- 2.23.0

2 2

[PATCH lkp-tests 05/11] feat: step 4 of iso2rootfs: add entry job
by Yu Chuan 10 Mar '21

10 Mar '21

Framework of install-iso-pre. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/install-iso-pre.yaml | 52 +++++++++++++++++++++++ tests/install-iso-pre | 86 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 jobs/install-iso-pre.yaml create mode 100755 tests/install-iso-pre diff --git a/jobs/install-iso-pre.yaml b/jobs/install-iso-pre.yaml new file mode 100644 index 000000000000..a9dbe46b4bf3 --- /dev/null +++ b/jobs/install-iso-pre.yaml @@ -0,0 +1,52 @@ +suite: install-iso-pre + +install-iso-pre: + ################# + # iso info + ################# + iso_os: + iso_arch: + iso_version: + + ################# + # http server info, which stored the installation source + ################# + HTTP_SERVER_HOST: + HTTP_SERVER_PORT: + HTTP_ROOT_DIR: + + ################# + # rsync-server info, used for upload ks + ################# + RSYNC_SERVER_HOST: + RSYNC_SERVER_PORT: + + ################# + # tftp server info, used for kickstart auto install + ################# + TFTP_SERVER_HOST: + + ################# + # nfs server info, used for store rootfs + ################# + NFS_SERVER_HOST: + NFS_ROOT_DIR: + + ################# + # result rootfs suffix, format: $(date "+%Y%m%d%H%M%S") + ################# + ROOTFS_SUFFIX: + + ################# + # name of volume group which stores the logical volume of rootfs + ################# + VG_NAME: + + ################# + # lab_z9 git repo clone url, used for find host file when submit + ################# + LAB_Z9_URL: + +secrets: + ROOTFS_ROOT_PASSWD: + MY_TOKEN: diff --git a/tests/install-iso-pre b/tests/install-iso-pre new file mode 100755 index 000000000000..4e8cbe22fcd4 --- /dev/null +++ b/tests/install-iso-pre @@ -0,0 +1,86 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +set -e + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/log.sh + +check_params() +{ + log_info "start check params" + + local required_vars=( + "iso_os" + "iso_arch" + "iso_version" + "ROOTFS_SUFFIX" + "VG_NAME" + + "HTTP_SERVER_HOST" + "HTTP_SERVER_PORT" + "HTTP_ROOT_DIR" + "ROOTFS_ROOT_PASSWD" + "rootfs_disk" + + "RSYNC_SERVER_HOST" + "RSYNC_SERVER_PORT" + "TFTP_SERVER_HOST" + "NFS_SERVER_HOST" + "NFS_ROOT_DIR" + + "LAB_Z9_URL" + "MY_TOKEN" + ) + + for i in "${required_vars[@]}" + do + [ -n "$(eval echo \$$i)" ] || die "cannot get value of var: $i" + done +} + +prepare_lv() +{ + : +} + +prepare_ks() +{ + : +} + +prepare_submit() +{ + : +} + +submit_install_iso_yaml() +{ + : +} + +submit_install_iso_firstboot_yaml() +{ + : +} + +submit_install_iso_release_yaml() +{ + : +} + +main() +{ + check_params + + prepare_lv + prepare_ks + prepare_submit + + submit_install_iso_yaml + submit_install_iso_firstboot_yaml + submit_install_iso_release_yaml +} + +main -- 2.23.0

2 1

[PATCH lkp-tests] add new type error
by Bai Jing 10 Mar '21

10 Mar '21

[input] Error: Unknown repo: 'powertools' [output] unknown-repo.powertools: 1 Signed-off-by: Bai Jing <799286817(a)qq.com> --- stats/openeuler_docker | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/stats/openeuler_docker b/stats/openeuler_docker index f85c9ca63..bcc5eb3b5 100755 --- a/stats/openeuler_docker +++ b/stats/openeuler_docker @@ -8,17 +8,23 @@ while (line = STDIN.gets) case line.chomp! # Error: Unable to find a match: docker-registry when /Error: Unable to find a match: (.+)/ - $1.split.each do |repo| - repo_set << repo + $1.split.each do |repo| + repo_set << repo end + # Error: Unknown repo: 'powertools' + when /Error: Unknown repo: (.+)/ + repo = $1.delete!("'") + puts "unknown-repo.#{repo}: 1" + puts "unknown-repo.#{repo}.message: #{line}" + # /bin/sh: passwd: command not found when /\/bin\/sh: (.+): command not found/ puts "sh.command-not-found.#{$1}: 1" puts "sh.command-not-found.#{$1}.message: #{line}" # RUN yum -y swap -- remove fakesystemd -- install systemd systemd-libs - # yum swap: error: unrecognized arguments: install systemd systemd-libs + # yum swap: error: unrecognized arguments: install systemd systemd-libs when /yum swap: error: .*: install (.+)/ $1.split.each do |sys| sys_set << sys -- 2.23.0

1 1

[PATCH v3 compass-ci] src/lib/job.cr: optimize the get_pkg_common_dir function
by Li Ping 10 Mar '21

10 Mar '21

[why] 1. rpm has no much to do with os_mount. add a judgment 2. optimize the code style Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/job.cr | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0b4bee4..f91e637 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -258,23 +258,28 @@ class Job end def get_pkg_common_dir - pkg_style = nil + tmp = nil ["cci-makepkg", "cci-depends", "build-pkg", "rpmbuild-pkg"].each do |item| - pkg_style = @hash[item]? - break if pkg_style + tmp = @hash[item]? + break if tmp end - return nil unless pkg_style + return nil unless tmp + pkg_style = Hash(String, JSON::Any).new {|h, k| h[k] = JSON::Any.new(nil)} + pkg_style.merge!(tmp.as_h? || Hash(String, JSON::Any).new) - pkg_style = JSON.parse("{}") if pkg_style == nil + tmp_os = pkg_style["os"].as_s? || "#{os}" + tmp_os_arch = pkg_style["os_arch"].as_s? || "#{os_arch}" + tmp_os_version = pkg_style["os_version"].as_s? || "#{os_version}" - tmp_os = pkg_style["os"]? == nil ? "#{os}" : pkg_style["os"] - tmp_os_arch = pkg_style["os_arch"]? == nil ? "#{os_arch}" : pkg_style["os_arch"] - tmp_os_version = pkg_style["os_version"]? == nil ? "#{os_version}" : pkg_style["os_version"] + mount_type = pkg_style["os_mount"].as_s? || "#{os_mount}" + # same usage for client + mount_type = "nfs" if mount_type == "cifs" - tmp_os_mount = pkg_style["os_mount"]? == nil ? "#{os_mount}" : pkg_style["os_mount"] - mount_type = tmp_os_mount == "cifs" ? "nfs" : tmp_os_mount.dup - - common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + if @hash["rpmbuild-pkg"]? + common_dir = "#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + else + common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + end return common_dir end -- 2.23.0

3 5

[PATCH v2 compass-ci] doc/manual: add chinese document
by Luan Shengde 10 Mar '21

10 Mar '21

Add chinese document for submit container. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- doc/manual/build-lkp-tests-container.zh.md | 63 ++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 doc/manual/build-lkp-tests-container.zh.md diff --git a/doc/manual/build-lkp-tests-container.zh.md b/doc/manual/build-lkp-tests-container.zh.md new file mode 100644 index 0000000..407ddbf --- /dev/null +++ b/doc/manual/build-lkp-tests-container.zh.md @@ -0,0 +1,63 @@ +# 前言 + +为了适应在各个版本的Linux操作系统上运行，我们提供了lkp-tests的容器化解决方案。 +你无需在您的本地服务上安装lkp-tests，也将避免缺乏依赖包而导致的安装失败。 + +# 前置准备 + + 请在您的本地服务器上完成下面步骤： + + - 安装docker + - 申请账号并配置默认的yaml文件 + - 生成本地的公私钥 + +# 构建容器 + +## 1. 下载资源 + + 本地下载lkp-tests和compass-ci + + 下载资源： + + git clone https://gitee.com/wu_fengguang/compass-ci.git + git clone https://gitee.com/wu_fengguang/lkp-tests.git + +## 2. 添加环境变量 + + 配置环境变量： + + echo "export LKP_SRC=$PWD/lkp-tests" >> ~/.${SHELL##*/}rc + echo "export CCI_SRC=$PWD/compass-ci" >> ~/.${SHELL##*/}rc + source ~/.${SHELL##*/}rc + +## 3. 构建镜像 + + 镜像构建： + + cd compass-ci/container/submit + ./build + +## 4. 添加可执行文件 + + 创建可执行文件： + + ln -s $CCI_SRC/container/submit/submit /usr/bin/submit + +# 开始第一个任务 + + 说明： + + 和在你的本地服务器上安装lkp-tests一样，您可以直接使用‘submit’命令提交任务。 + 每一个任务都会生成一个一次性的容器来执行。 + 可执行文件submit会将您下载的lkp-tests挂载到容器内的lkp-tests路径上，这样您在本地编辑lkp-tests文件，在容器内也会生效。 + + + + 提交任务命令示例： + + + submit -c -m testbox=vm-2p8g borrow-1h.yaml + + submit命令： + + 了解更多submit命令的使用方法，请参考：[submit用户手册](https://gitee.com/wu_fengguang/compass-c… -- 2.23.0

1 0

[PATCH v5 compass-ci] src/lib/job.cr: optimize the get_pkg_common_dir function
by Li Ping 10 Mar '21

10 Mar '21

[why] 1. rpm has no much to do with os_mount. add a judgment 2. optimize the code style Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/job.cr | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0b4bee4..a7674d4 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -258,23 +258,28 @@ class Job end def get_pkg_common_dir - pkg_style = nil + tmp_style = nil ["cci-makepkg", "cci-depends", "build-pkg", "rpmbuild-pkg"].each do |item| - pkg_style = @hash[item]? - break if pkg_style + tmp_style = @hash[item]? + break if tmp_style end - return nil unless pkg_style + return nil unless tmp_style + pkg_style = Hash(String, JSON::Any).new {|h, k| h[k] = JSON::Any.new(nil)} + pkg_style.merge!(tmp_style.as_h? || Hash(String, JSON::Any).new) - pkg_style = JSON.parse("{}") if pkg_style == nil + tmp_os = pkg_style["os"].as_s? || "#{os}" + tmp_os_arch = pkg_style["os_arch"].as_s? || "#{os_arch}" + tmp_os_version = pkg_style["os_version"].as_s? || "#{os_version}" - tmp_os = pkg_style["os"]? == nil ? "#{os}" : pkg_style["os"] - tmp_os_arch = pkg_style["os_arch"]? == nil ? "#{os_arch}" : pkg_style["os_arch"] - tmp_os_version = pkg_style["os_version"]? == nil ? "#{os_version}" : pkg_style["os_version"] + mount_type = pkg_style["os_mount"].as_s? || "#{os_mount}" + # same usage for client + mount_type = "nfs" if mount_type == "cifs" - tmp_os_mount = pkg_style["os_mount"]? == nil ? "#{os_mount}" : pkg_style["os_mount"] - mount_type = tmp_os_mount == "cifs" ? "nfs" : tmp_os_mount.dup - - common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + if @hash["rpmbuild-pkg"]? + common_dir = "#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + else + common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + end return common_dir end -- 2.23.0

1 0

[PATCH compass-ci] sbin/auto_submit: change the path to do git command
by Li Yuanchao 10 Mar '21

10 Mar '21

As the path of repo mirrors under /srv/git changes, the path to execute git command should change too. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/sbin/auto_submit b/sbin/auto_submit index 857af29..faebb2f 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -44,14 +44,16 @@ class AutoSubmit submit_argv end - def submit_one_job(submit_argv, git_repo, branch, commit_id) - inactive_time = %x(git -C /srv/git/#{git_repo}.git log --pretty=format:"%cr" -1 #{commit_id}) + def submit_one_job(submit_argv, git_repo, belong, branch, commit_id) + mirror_dir = "/srv/git/#{belong}/#{git_repo}.git" + inactive_time = %x(git -C #{mirror_dir} log --pretty=format:"%cr" -1 #{commit_id}) return if inactive_time =~ /(month|year)/ real_argvs = Array.new(submit_argv) real_argvs.push("upstream_branch=#{branch}") real_argvs.push("upstream_commit=#{commit_id}") - tag = %x(git -C /srv/git/#{git_repo}.git tag --points-at #{commit_id}) + real_argvs.push("upstream_dir=#{belong}") + tag = %x(git -C #{mirror_dir} tag --points-at #{commit_id}) real_argvs.push("upstream_tag=#{tag}") unless tag.empty? system(real_argvs.join(' ')) @@ -60,16 +62,17 @@ class AutoSubmit def submit_specific_branch(submit_argv, newrefs_info, branches) branches.each do |branch_name| branch = "refs/heads/#{branch_name}" - next unless newrefs_info['new_refs']['heads'][branch] + commit_id = newrefs_info['new_refs']['heads'][branch] + next unless commit_id - submit_one_job(submit_argv, newrefs_info['git_repo'], branch_name, newrefs_info['new_refs']['heads'][branch]) + submit_one_job(submit_argv, newrefs_info['git_repo'], newrefs_info['belong'], branch_name, commit_id) end end - def submit_non_specific(non_specific_commits, git_repo, submit_argv) - non_specific_commits.each do |branch, commit_id| + def submit_non_specific(newrefs_info, submit_argv) + newrefs_info['new_refs']['heads'].each do |branch, commit_id| branch = branch.delete_prefix('refs/heads/') - submit_one_job(submit_argv, git_repo, branch, commit_id) + submit_one_job(submit_argv, newrefs_info['git_repo'], newrefs_info['belong'], branch, commit_id) end end @@ -96,7 +99,7 @@ class AutoSubmit if element['branches'] submit_specific_branch(argvs, newrefs_info, element['branches']) else - submit_non_specific(newrefs_info['new_refs']['heads'], newrefs_info['git_repo'], argvs) + submit_non_specific(newrefs_info, argvs) end end end -- 2.23.0

1 0

[PATCH lkp-tests] stats/proc-vmstat.rb: add the execute permission
by Wei Jihui 10 Mar '21

10 Mar '21

[error output when submit job] 2021-03-10 10:20:04 +0800 WARN -- skip non-executable /c/lkp-tests/stats/proc-vmstat.rb Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- stats/proc-vmstat.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 stats/proc-vmstat.rb diff --git a/stats/proc-vmstat.rb b/stats/proc-vmstat.rb old mode 100644 new mode 100755 -- 2.23.0

1 0

[PATCH v4 compass-ci] src/lib/job.cr: optimize the get_pkg_common_dir function
by Li Ping 10 Mar '21

10 Mar '21

[why] 1. rpm has no much to do with os_mount. add a judgment 2. optimize the code style Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/job.cr | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0b4bee4..25d8462 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -258,23 +258,28 @@ class Job end def get_pkg_common_dir - pkg_style = nil + tmp_style = nil ["cci-makepkg", "cci-depends", "build-pkg", "rpmbuild-pkg"].each do |item| - pkg_style = @hash[item]? - break if pkg_style + tmp_style = @hash[item]? + break if tmp_style end - return nil unless pkg_style + return nil unless tmp_style + pkg_style = Hash(String, JSON::Any).new {|h, k| h[k] = JSON::Any.new(nil)} + pkg_style.merge!(tmp.as_h? || Hash(String, JSON::Any).new) - pkg_style = JSON.parse("{}") if pkg_style == nil + tmp_os = pkg_style["os"].as_s? || "#{os}" + tmp_os_arch = pkg_style["os_arch"].as_s? || "#{os_arch}" + tmp_os_version = pkg_style["os_version"].as_s? || "#{os_version}" - tmp_os = pkg_style["os"]? == nil ? "#{os}" : pkg_style["os"] - tmp_os_arch = pkg_style["os_arch"]? == nil ? "#{os_arch}" : pkg_style["os_arch"] - tmp_os_version = pkg_style["os_version"]? == nil ? "#{os_version}" : pkg_style["os_version"] + mount_type = pkg_style["os_mount"].as_s? || "#{os_mount}" + # same usage for client + mount_type = "nfs" if mount_type == "cifs" - tmp_os_mount = pkg_style["os_mount"]? == nil ? "#{os_mount}" : pkg_style["os_mount"] - mount_type = tmp_os_mount == "cifs" ? "nfs" : tmp_os_mount.dup - - common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + if @hash["rpmbuild-pkg"]? + common_dir = "#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + else + common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + end return common_dir end -- 2.23.0

1 1

[PATCH v3 compass-ci 2/2] lib/dump-stat.rb: refactor lkp-tests/sbin/dump-stat
by Lu Weitao 10 Mar '21

10 Mar '21

[Why] support handle hash from lkp-tests/stats/$script.rb Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/dump_stat.rb | 194 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 lib/dump_stat.rb diff --git a/lib/dump_stat.rb b/lib/dump_stat.rb new file mode 100644 index 0000000..1c0695e --- /dev/null +++ b/lib/dump_stat.rb @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: GPL-2.0-only + +LKP_SRC ||= ENV['LKP_SRC'] || '/c/lkp-tests' + +require "#{LKP_SRC}/lib/statistics" +require "#{LKP_SRC}/lib/bounds" +require "#{LKP_SRC}/lib/yaml" +require "#{LKP_SRC}/lib/job" +require "#{LKP_SRC}/lib/string_ext" +require "#{LKP_SRC}/lib/log" +require 'set' + +UNSTRUCTURED_MONITORS = %w[ftrace].to_set + +def warn_stat(msg, monitor) + log_warn msg + log_warn "check #{RESULT_ROOT}/#{monitor}" +end + +# dump stat which input by lkp-tests/stats/$script return +# input: +# eg-1: +# { +# "pgfree" => [275506, 280018], +# ... +# } +# eg-2: +# { +# "iperf.tcp.sender.bps" => 34804801216.197174, +# "iperf.tcp.receiver.bps" => "34804762215.18231" +# } +module DumpStat + def self.dump_stat(monitor, stat_result) + @result = {} + @invalid_records = [] + @record_index = 0 + @monitor = monitor + + stat_result.each do |key, value| + key = key.resolve_invalid_bytes + next if key[0] == '#' + next if value.empty? || value == 0 + next if monitor =~ /^(dmesg|kmsg)$/ && key =~ /^(message|pattern):/ + + if key =~ /[ \t]/ + @invalid_records.push @record_index + warn_stat "whitespace in stats name: #{key}", @monitor + return nil # for exit current stats/script dump-stat + end + next if assign_log_message(key, value) + + k = @monitor + '.' + key + @result[k] ||= [] + fill_zero(k) + if value.is_a?(String) + value = check_string_value(k, value, @monitor) + next unless value + return nil unless number?(value, @invalid_records) + + value = value.index('.') ? value.to_f : value.to_i + elsif value.is_a?(Array) + (0..value.size - 1).each do |i| + next unless value[i].is_a?(String) + + value[i] = check_string_value(k, value[i], @monitor) + next unless value[i] + return nil unless number?(value[i], @invalid_records) + + value[i] = value[i].index('.') ? value[i].to_f : value[i].to_i + valid_stats_verification(k, value[i]) + end + @result[k] = value + next + end + valid_stats_verification(k, value) + @result[k].push value + end + return nil if @result.empty? + + remove_zero_stats + delete_invalid_number(@result, @invalid_records, @monitor) + cols_verifation + return nil unless useful_result?(@result) + + save_json(@result, "#{RESULT_ROOT}/#{(a)monitor}.json", @result.size * @min_cols > 1000) + end + + # keep message | log line which key end with .message|.log + def self.assign_log_message(key, value) + if key.end_with?('.message', '.log') + k = @monitor + '.' + key + @result[k] = value + return true + end + + false + end + + def self.fill_zero(key) + size = @result[key].size + if @record_index < size + @record_index = size + elsif (@record_index - size).positive? + # fill 0 for missing values + @result[key].concat([0] * (@record_index - size)) + end + end + + def self.valid_stats_verification(key, value) + return nil if valid_stats_range? key, value + + @invalid_records.push @record_index + puts "outside valid range: #{value} in #{key} #{RESULT_ROOT}" + end + + def self.remove_zero_stats + @max_cols = 0 + @min_cols = Float::INFINITY + @min_cols_stat = '' + @max_cols_stat = '' + zero_stats = [] + @result.each do |key, val| + next if key.end_with?('.message', '.log') + + if @max_cols < val.size + @max_cols = val.size + @max_cols_stat = key + end + if @min_cols > val.size + @min_cols = val.size + @min_cols_stat = key + end + next if val[0] != 0 + next if val[-1] != 0 + next if val.sum != 0 + + zero_stats << key + end + zero_stats.each { |x| @result.delete x } + end + + def self.cols_verifation + return nil unless @min_cols < @max_cols && !UNSTRUCTURED_MONITORS.include?(@monitor) + + if @min_cols == @max_cols - 1 + @result.each { |_k, y| y.pop if y.size == @max_cols } + puts "Last record seems incomplete. Truncated #{RESULT_ROOT}/#{(a)monitor}.json" + else + warn_stat "Not a matrix: value size is different - #{@min_cols_stat}: #{@min_cols} != #{@max_cols_stat}: #{@max_cols}: #{RESULT_ROOT}/#{(a)monitor}.json", @monitor + end + end +end + +def check_string_value(key, value, monitor) + # value terminator is expected. If not, throw out an error warning. + warn_stat "no line terminator in stats value: #{value}", monitor if value.chomp!.nil? + + value.strip! + if value.empty? + warn_stat "empty stat value of #{key}", monitor + return nil + end + + return value +end + +# only number is valid +def number?(value, invalid_records) + unless value.numeric? + invalid_records.push record_index + warn_stat "invalid stats value: #{value}", monitor + return nil + end + + true +end + +def delete_invalid_number(result, invalid_records, monitor) + return nil if monitor == 'ftrace' + + invalid_records.reverse_each do |index| + result.each do |_k, value| + value.delete_at index + end + end +end + +def useful_result?(result) + return nil if result.empty? + return nil if result.values[0].size.zero? + return nil if result.values[-1].size.zero? + + true +end -- 2.23.0

1 0

[PATCH v4 lkp-tests 2/2] sbin/makepkg: fix whether to change domain of submodules
by Liu Shaofei 09 Mar '21

09 Mar '21

the param "--rewritegitserver" indicates that all of submodules will be changed domian name, and downloaded from Z9/Crystal server. Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- sbin/makepkg | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/sbin/makepkg b/sbin/makepkg index 7a92ea71b..cb0516413 100755 --- a/sbin/makepkg +++ b/sbin/makepkg @@ -89,6 +89,7 @@ SIGNPKG='' SPLITPKG=0 SOURCEONLY=0 VERIFYSOURCE=0 +REWRITEGITSERVER=0 # Forces the pkgver of the current PKGBUILD. Used by the fakeroot call # when dealing with svn/cvs/etc PKGBUILDs. @@ -587,6 +588,16 @@ download_git() { fi } +# modify submodules address to local server address. +# (( REWRITEGITSERVER )) is true, replace submodules domain, and download it from local end. +modify_gitmodules() { + (( REWRITEGITSERVER )) || return + + if [[ -f ".gitmodules" ]] && [[ -n ${GIT_SERVER} ]]; then + git config --system url."git://${GIT_SERVER}/".insteadOf https:// + fi +} + extract_git() { local netfile=$1 @@ -623,10 +634,7 @@ extract_git() { cd_safe "${dir##*/}" - # modify submodules address to local server address. - if [[ -f ".gitmodules" ]] && [[ -n ${LKP_SERVER} ]]; then - echo -e "[url \"git://${LKP_SERVER}/\"]\n\tinsteadOf = https://" >> /etc/gitconfig - fi + modify_gitmodules local ref=origin/HEAD if [[ -n $fragment ]]; then @@ -3421,6 +3429,7 @@ usage() { printf -- "$(gettext " --skipinteg Do not perform any verification checks on source files")\n" printf -- "$(gettext " --skippgpcheck Do not verify source files with PGP signatures")\n" printf -- "$(gettext " --verifysource Download source files (if needed) and perform integrity checks")\n" + printf -- "$(gettext " --rewritegitserver Change domain name when downloading submodules repository")\n" echo printf -- "$(gettext "These options can be passed to %s:")\n" "pacman" echo @@ -3462,7 +3471,7 @@ OPT_LONG=('allsource' 'check' 'clean' 'cleanbuild' 'config:' 'force' 'geninteg' 'help' 'holdver' 'ignorearch' 'install' 'key:' 'log' 'noarchive' 'nobuild' 'nocolor' 'nocheck' 'nodeps' 'noextract' 'noprepare' 'nosign' 'pkg:' 'repackage' 'rmdeps' 'sign' 'skipchecksums' 'skipinteg' 'skippgpcheck' 'source' 'syncdeps' - 'verifysource' 'version') + 'verifysource' 'version' 'rewritegitserver') # Pacman Options OPT_LONG+=('asdeps' 'noconfirm' 'needed' 'noprogressbar') @@ -3514,6 +3523,7 @@ while true; do -s|--syncdeps) DEP_BIN=1 ;; -S|--source) SOURCEONLY=1 ;; --verifysource) VERIFYSOURCE=1 ;; + --rewritegitserver) REWRITEGITSERVER=1 ;; -h|--help) usage; exit 0 ;; # E_OK -V|--version) version; exit 0 ;; # E_OK -- 2.23.0

1 0

[PATCH v4 lkp-tests 1/2] tests/build-pkg: download submodules from local end
by Liu Shaofei 09 Mar '21

09 Mar '21

[why] When building software package by build-pkg, the source codes repository contains submodules. By default, the submodules can be downloaded from remote end during building packages. [how] By using --rewritegitserver param, submodules can be downloaded from Z9/Crystal. Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- tests/build-pkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build-pkg b/tests/build-pkg index 6a083a9b3..439328651 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -93,7 +93,7 @@ create_cgz_name() cgz_path="$PKG_MNT/${pack_to}/${pkgname}" cgz_name="${cgz_path}/${upstream_commit}.cgz" - pkg_args="-A --check --skippgpcheck" + pkg_args="-A --check --skippgpcheck --rewritegitserver" [ "${os}" == "archlinux" ] && pkg_args="${pkg_args} -s --needed --noconfirm" [ -n "$git_tag" ] && { -- 2.23.0

1 0

[PATCH compass-ci] lib/git_mirror.rb: don't count new ref if it is not new
by Li Yuanchao 09 Mar '21

09 Mar '21

When repos are added, new_refs_count will increase. But many repos are not updated that day, that cause new_refs_count increase much more than real new refs number. Now only count those really new. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 1fa8c9e..f711747 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -203,7 +203,7 @@ class MirrorMain feedback_info[:new_refs] = new_refs send_message(feedback_info) - new_refs_log(git_repo, new_refs[:heads].length) + new_refs_log(git_repo, new_refs[:heads].length) if last_commit_new?(git_repo) end def do_push(fork_key) @@ -336,7 +336,7 @@ class MirrorMain def update_fork_stat(git_repo, possible_new_refs) update_stat_fetch(git_repo) - update_stat_new_refs(git_repo) if possible_new_refs + update_stat_new_refs(git_repo) if possible_new_refs && last_commit_new?(git_repo) es_repo_update(git_repo) end end @@ -530,4 +530,11 @@ class MirrorMain nr_new_branch: nr_new_branch }.to_json) end + + def last_commit_new?(git_repo) + inactive_time = %x(git -C /srv/git/#{@git_info[git_repo]['belong']}/#{git_repo}.git log --pretty=format:"%cr" -1) + return false if inactive_time =~ /(day|week|month|year)/ + + return true + end end -- 2.23.0

1 0