- Compass-ci - mailweb.openeuler.org

[PATCH v6 lkp-tests] add: sysbench-threads-git result parsing file
by Hu Xuejiao 21 Nov '20

21 Nov '20

[input] General statistics: total time: 0.3987s total number of events: 10000 total time taken by event execution: 0.3971 response time: min: 0.04ms [output] total time: 0.3987s Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- stats/sysbench-threads-git | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 stats/sysbench-threads-git diff --git a/stats/sysbench-threads-git b/stats/sysbench-threads-git new file mode 100755 index 00000000..afe5d664 --- /dev/null +++ b/stats/sysbench-threads-git @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby + +# General statistics: +# total time: 0.2671s +# total number of events: 10000 +# total time taken by event execution: 0.3971s +# response time: +# min: 0.04ms + +$stdin.each_line do |line| + next unless line =~ /^\s+total time:\s+([0-9.]+)/ + + puts "total time: #{$1.to_f}s" +end -- 2.23.0

2 1

[PATCH v2 lkp-tests] stats/lmbench3: add new parsing parameters for performance test
by Bai Jing 21 Nov '20

21 Nov '20

[input] Memory read bandwidth 0.000512 16353.73 0.001024 16577.89 0.002048 16752.28 [output] L1_$: 1.539 L2_$: 0.539 Signed-off-by: Bai Jing <799286817(a)qq.com> --- stats/lmbench3 | 62 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/stats/lmbench3 b/stats/lmbench3 index 211c3abc..750e8bfc 100755 --- a/stats/lmbench3 +++ b/stats/lmbench3 @@ -21,6 +21,27 @@ def largest_bandwidth bandwidth end +def designated_file_size + designated_hash = {} + result_str = '' + + $stdin.each_line do |line| + result_str += line + break if line =~ /(^\s|^\t)/ + end + + result_str.split("\n").each do |e| + designated_hash['L1_$'] = e.split[1] if e =~ /0.01758/ + designated_hash['Rand_mem'] = e.split[1] if e =~ /512.0000/ + designated_hash['Mmap_Latency'] = e.split[1] if e =~ /536.870912/ + designated_hash['L2_$'] = e.split[1] if e =~ /0.05078/ + end + + designated_hash +end + +null_io_array = [] + while (line = STDIN.gets) line = line.resolve_invalid_bytes case line @@ -32,14 +53,15 @@ while (line = STDIN.gets) # Simple fstat: 0.3517 microseconds # Simple open/close: 1.3696 microseconds when /^Simple (\S+): (\d+.\d+) microseconds$/ + null_io_array << $2.to_f if $1.to_s == 'read' || $1.to_s == 'write' puts "syscall.#{$1}.latency.us: #{$2}" # Extract select test result. # Select on 100 fd's: 1.2293 microseconds # Select on 100 tcp fd's: 5.0377 microseconds - when /^Select on 100 fd\'s: (\d+.\d+) microseconds$/ + when /^Select on 100 fd's: (\d+.\d+) microseconds$/ puts "Select.100fd.latency.us: #{$1}" - when /^Select on 100 tcp fd\'s: (\d+.\d+) microseconds$/ + when /^Select on 100 tcp fd's: (\d+.\d+) microseconds$/ puts "Select.100tcp.latency.us: #{$1}" # Extract proc test result. @@ -198,5 +220,41 @@ while (line = STDIN.gets) # ... when /^"Mmap read open2close bandwidth$/ puts "MMAP.read_open2close.bandwidth.MB/sec: #{largest_bandwidth}" + + # "Random Memory load latency" + # 416.00000 92.824 + # 448.00000 92.772 + # 512.00000 92.640 + # ... + when /^Random load latency$/ + result_rand_mem = designated_file_size + puts "Rand_mem: #{result_rand_mem['Rand_mem']}" + + # "Memory load latency" + # 0.00049 1.539 + # 0.00098 1.539 + # 512.00000 74.421 + # ... + when /^Memory load latency$/ + result_rand_mem = designated_file_size + puts "L1_$: #{result_rand_mem['L1_$']}" + puts "L2_$: #{result_rand_mem['L2_$']}" + + # "mappings" + # 67.108864 82 + # 134.217728 151 + # 268.435456 309 + # 536.870912 615 + # ... + when /^"mappings$/ + result_map = designated_file_size + puts "Mmap_Latency: #{result_map['Mmap_Latency']}" end end + +num = 0 +null_io_array.each do |v| + num += v +end +null_num = format('%.4f', num / 2).to_f +puts "null_io: #{null_num}" -- 2.23.0

3 6

git count
by Wu Fengguang 20 Nov '20

20 Nov '20

Shengde is our code star, thanks Shengde! wfg@crystal ~/todo% bin/git-count 21 Luan Shengde 13 Sun Yukui 10 Li Yuanchao 8 Hu Xuejiao 8 Xiao Shenwei 8 Zhang Yale 8 Yu Chuan 8 Lin Jiaxin 7 Bai Jing 7 Liu Yinsi 6 Liu Shaofei 6 Zhang Yu 5 Cao Xueliang 5 Cui Yili 5 Wang Yong 4 Xu Xijian 4 Wei Jihui 4 Lu Kaiyi 3 Wu Zhende 2 Lu Weitao 2 Wang Chenglong 2 Zhang Yuhang 1 Wu Fengguang 1 Ren Wen 1 Li Ping

1 0

review star
by Wu Fengguang 20 Nov '20

20 Nov '20

Xueliang is our review star, thanks Xueliang! wfg@crystal /c/todo% bin/review-star Review email records in the last 7 days total review paragraph point people 51 44 107 34 Wu Fengguang 118 37 48 24 Cao Xueliang 99 35 50 20 Wei Jihui 29 18 22 14 Xu Xijian 33 6 18 9 Xiao Shenwei 75 23 54 9 Luan Shengde 47 19 21 7 Liu Shaofei 61 14 13 7 Lu Kaiyi 31 13 17 6 Zhang Yu 16 6 11 6 Zhang Yuhang 52 9 11 5 Bai Jing 18 5 10 4 Wang Yong 34 6 11 4 Wang Chenglong 22 14 17 4 Cui Yili 56 7 10 4 Lin Jiaxin 13 3 5 4 Lu Weitao 19 6 8 3 Wu Zhende 61 7 9 3 Hu Xuejiao 47 3 8 2 Yu Chuan 31 7 9 1 Liu Yinsi 8 0 0 0 Li Ping 1 1 1 0 Zhang Dewan 12 2 3 0 Ren Wen 28 9 9 -2 Zhang Yale 23 5 9 -3 Li Yuanchao 41 15 26 -3 Sun Yukui

1 0

help star
by Wu Fengguang 20 Nov '20

20 Nov '20

Yuchuan and Jihui is our help star, thanks Yuchuan and Jihui! wfg@crystal ~/todo% bin/help-star people/*/credit/2020-11-20.md 18.8 Wu Fengguang 17.2 Yu Chuan 17.2 Wei Jihui 16.3 Cao Xueliang 13.7 Xiao Shenwei 12.0 Wu Zhende 11.6 Xu Xijian 10.1 Wang Chenglong 9.8 Liu Shaofei 9.7 Lu Weitao 9.5 Hu Xuejiao 9.0 Luan Shengde 8.5 Cui Yili 8.4 Zhang Yuhang 7.9 Liu Yinsi 7.2 Wang Yong 7.0 Zhang Dewan 6.9 Zhang Yu 6.4 Lin Jiaxin 6.0 Ren Wen 5.8 Li Yuanchao 5.0 Bai Jing 4.3 Du Kaitian 4.2 Zhang Yale 3.3 Lu Kaiyi 2.0 Sun Yukui 1.4 Tong Qunfeng 0.0 Wang Shuai 0.0 Li Ping 0.0 Cui Xucui

1 0

[PATCH compass-ci] doc: add Linux article
by Wu Fengguang 20 Nov '20

20 Nov '20

Signed-off-by: Wu Fengguang <wfg(a)mail.ustc.edu.cn> --- doc/learning-resources.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/learning-resources.md b/doc/learning-resources.md index 62716cf..e631f4a 100644 --- a/doc/learning-resources.md +++ b/doc/learning-resources.md @@ -55,6 +55,9 @@ https://github.com/DocSpring/ruby_crystal_codemod shell ===== +Linux 的概念与体系 +https://www.cnblogs.com/vamei/archive/2012/10/10/2718229.html + man bash https://devhints.io/bash https://github.com/denysdovhan/bash-handbook/blob/master/translations/zh-CN… -- 2.23.0

1 1

[PATCH v2 compass-ci] scheduler: fix: alter processes of request_state
by Ren Wen 20 Nov '20

20 Nov '20

There're 2 "finished" states before, one is *daemon* request, the other is *test* request. Now change *daemon*'s to "started", means daemon started. This state will save to redis key "sched/cluster_state". Why move "finished" to another "when condition"? When all nodes in cluster do *run_test*, no *start_daemon*, the node will restart immediately after its tests finished. Now need to wait other nodes, so move "finished" to another "when condition" to wait all nodes finished. Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/scheduler/request_cluster_state.cr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/scheduler/request_cluster_state.cr b/src/scheduler/request_cluster_state.cr index ac6cb8e..5bbbad1 100644 --- a/src/scheduler/request_cluster_state.cr +++ b/src/scheduler/request_cluster_state.cr @@ -13,13 +13,14 @@ class Sched cluster_state = "" states = {"abort" => "abort", + "started" => "start", "finished" => "finish", "failed" => "abort", "wait_ready" => "ready", "wait_finish" => "finish"} case request_state - when "abort", "finished", "failed" + when "abort", "started", "failed" # update node state only update_cluster_state(cluster_id, job_id, {"state" => states[request_state]}) when "wait_ready" @@ -30,7 +31,7 @@ class Sched } return cluster_state - when "wait_finish" + when "wait_finish", "finished" update_cluster_state(cluster_id, job_id, {"state" => states[request_state]}) while 1 sleep(10) -- 2.23.0

1 0

[PATCH compass-ci] scheduler: fix: alter processes of request_state
by Ren Wen 20 Nov '20

20 Nov '20

There're 2 "finished" states before, one is *daemon* request, the other is *test* request. Now change *daemon*'s to "started", means daemon started. This state will save to redis key "sched/cluster_state". Why move "finished" to another "when condition"? When all nodes in cluster do *run_test*, no *start_daemon*, the node will restart immediately after its tests finished. Now need to wait other nodes, so move "finished" to another "when condition" to wait all nodes finished. Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/scheduler/request_cluster_state.cr | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/scheduler/request_cluster_state.cr b/src/scheduler/request_cluster_state.cr index ac6cb8e..fffa4cf 100644 --- a/src/scheduler/request_cluster_state.cr +++ b/src/scheduler/request_cluster_state.cr @@ -13,13 +13,14 @@ class Sched cluster_state = "" states = {"abort" => "abort", + "started" => "started", "finished" => "finish", "failed" => "abort", "wait_ready" => "ready", "wait_finish" => "finish"} case request_state - when "abort", "finished", "failed" + when "abort", "started", "failed" # update node state only update_cluster_state(cluster_id, job_id, {"state" => states[request_state]}) when "wait_ready" @@ -30,7 +31,7 @@ class Sched } return cluster_state - when "wait_finish" + when "wait_finish", "finished" update_cluster_state(cluster_id, job_id, {"state" => states[request_state]}) while 1 sleep(10) -- 2.23.0

2 3

[PATCH v2 compass-ci] sbin/es-jobs-mapping.sh: Support more fields in "jobs" index
by Zhang Yuhang 20 Nov '20

20 Nov '20

[why] "index.mapping.total_fields.limit": The maximum number of fields in an index. The default value is 1000. 1000 is small for "jobs" index. Signed-off-by: Zhang Yuhang <zhangyuhang25(a)huawei.com> --- sbin/es-jobs-mapping.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbin/es-jobs-mapping.sh b/sbin/es-jobs-mapping.sh index 076bdd3..018f57e 100755 --- a/sbin/es-jobs-mapping.sh +++ b/sbin/es-jobs-mapping.sh @@ -149,5 +149,10 @@ else if [ $? -ne 0 ] then echo "create jobs index failed." + else + echo "set index.mapping.total_fields.limit: 10000" + curl -XPUT 127.0.0.1:9200/jobs/_settings -H 'Content-Type: application/json' -d ' + {"index.mapping.total_fields.limit": 10000} + ' fi fi -- 2.23.0

3 3

[PATCH compass-ci] sbin/es-jobs-mapping.sh: set "index.mapping.total_fields.limit: 10000" when create "jobs" mapping.
by Zhang Yuhang 20 Nov '20

20 Nov '20

[why] "index.mapping.total_fields.limit": The maximum number of fields in an index. The default value is 1000. 1000 is small for "jobs" index. Signed-off-by: Zhang Yuhang <zhangyuhang25(a)huawei.com> --- sbin/es-jobs-mapping.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sbin/es-jobs-mapping.sh b/sbin/es-jobs-mapping.sh index 076bdd3..018f57e 100755 --- a/sbin/es-jobs-mapping.sh +++ b/sbin/es-jobs-mapping.sh @@ -149,5 +149,10 @@ else if [ $? -ne 0 ] then echo "create jobs index failed." + else + echo "set index.mapping.total_fields.limit: 10000" + curl -XPUT 127.0.0.1:9200/jobs/_settings -H 'Content-Type: application/json' -d ' + {"index.mapping.total_fields.limit": 10000} + ' fi fi -- 2.23.0

3 4

[PATCH compass-ci] container/ssh-r: fix address already in use
by Liu Yinsi 20 Nov '20

20 Nov '20

[why] when execute ssh-r/start, error message: docker: Error response from daemon: driver failed programming external connectivity on endpoint ssh_r (55638ba16aa1264dff0fdc1badc93176eec07d2b848d4a8d418d07011d84313d): Error starting userland proxy: listen tcp 0.0.0.0:51998: bind: address already in use. [how] because ssh-r will bind ports 50000-51999, set ip_local_port_range to prevent occupation. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/0-package/common | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparrow/0-package/common b/sparrow/0-package/common index 523438c..c327707 100755 --- a/sparrow/0-package/common +++ b/sparrow/0-package/common @@ -12,6 +12,8 @@ cat >> /etc/sysctl.conf <<EOF net.ipv4.ip_forward=1 net.ipv6.bindv6only=1 vm.max_map_count=262144 +net.ipv4.ip_local_port_range = 1024 48999 + EOF sysctl -p -- 2.23.0

2 2

[PATCH v2 compass-ci] sparrow/0-package: fix failed to start ssh-r
by Liu Yinsi 20 Nov '20

20 Nov '20

[why] when execute ssh-r/start, error message: docker: Error response from daemon: driver failed programming external connectivity on endpoint ssh_r (55638ba16aa1264dff0fdc1badc93176eec07d2b848d4a8d418d07011d84313d): Error starting userland proxy: listen tcp 0.0.0.0:51998: bind: address already in use. [how] because ssh-r will bind ports 50000-51999, set ip_local_port_range to prevent occupation. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/0-package/common | 1 + 1 file changed, 1 insertion(+) diff --git a/sparrow/0-package/common b/sparrow/0-package/common index 523438c..5eb9f76 100755 --- a/sparrow/0-package/common +++ b/sparrow/0-package/common @@ -12,6 +12,7 @@ cat >> /etc/sysctl.conf <<EOF net.ipv4.ip_forward=1 net.ipv6.bindv6only=1 vm.max_map_count=262144 +net.ipv4.ip_local_port_range = 1024 48999 EOF sysctl -p -- 2.23.0

1 0

[PATCH v5 compass-ci] lib/log.rb: add new class for json log
by Wu Zhende 20 Nov '20

20 Nov '20

[Why] The built-in log system prints logs in string format by default. The logs should be formatted in json format for subsequent analysis and processing. [Example1] use: log.info("test") output: {"level":"INFO","level_num":1,"datetime":"2020-11-19 16:57:17+0800","progname":null,"message":"test"} [Example2] use: log.error({"job_id" => "1", "job_state" => "submit"}.to_json) output: {"level":"ERROR","level_num":3,"datetime":"2020-11-19 16:57:17+0800","progname":null,"message":"", "job_id":"1","job_state":"submit","caller":["/usr/share/ruby/logger.rb:545:in `error'","./test.rb:6:in`<main>'"]} Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/log.rb | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 lib/log.rb diff --git a/lib/log.rb b/lib/log.rb new file mode 100644 index 0000000..aad00ca --- /dev/null +++ b/lib/log.rb @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'logger' +require 'json' + +# print logs in JSON format +class Log < Logger + LEVEL_INFO = { + 'DEBUG' => 0, + 'INFO' => 1, + 'WARN' => 2, + 'ERROR' => 3, + 'FATAL' => 4, + 'UNKNOWN' => 5 + }.freeze + + FORMATTER = proc { |severity, datetime, progname, msg| + level_num = LEVEL_INFO[severity] + begin + message = JSON.parse(msg) + rescue JSON::ParserError + message = { 'message' => msg } + end + h = { + 'level' => severity.to_s, + 'level_num' => level_num, + 'datetime' => datetime, + 'progname' => progname, + 'message' => '' + } + h.merge!(message) + h.merge!({ 'caller' => caller }) if level_num >= 2 + h.to_json + } + + def initialize(logdev = STDOUT, formatter = FORMATTER) + super(logdev, formatter: formatter) + end +end -- 2.23.0

2 2

[PATCH lkp-tests 1/2] submit: support add job to personal queue
by Xiao Shenwei 20 Nov '20

20 Nov '20

usage: submit --my-queue testbox=vm-2p8g jobs/borrow-1h.yaml if use this parameter, your job will be add to the queue which is $tbox_group~$USER Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- sbin/submit | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sbin/submit b/sbin/submit index 9bba1018..53ea5bf0 100755 --- a/sbin/submit +++ b/sbin/submit @@ -16,6 +16,7 @@ opt_output_dir = nil opt_auto_define_files = false opt_monitor = false opt_monitor_query = {} +personal_job = false actions = ['output', 'stop'] options = OptionParser.new do |opts| @@ -54,6 +55,11 @@ options = OptionParser.new do |opts| ARGV.shift end end + + opts.on('--my-queue', 'add to personal queue') do + personal_job = true + end + end options.parse!(ARGV) @@ -93,6 +99,11 @@ def find_jobfile(jobfile) exit 1 end +def add_personal_queue(job) + queue = "#{job['tbox_group']}~#{ENV['USER']}" + job['queue'] = queue +end + ARGV.each do |jobfile| jobfile = find_jobfile(jobfile) jobs = Job2sh.new @@ -134,6 +145,7 @@ ARGV.each do |jobfile| job_hash_list << job_hash job_hash = job_hash.merge(sh_hash) + add_personal_queue(job_hash) if personal_job # save job to yaml if opt_output_dir prefix = File.join(opt_output_dir, File.basename(jobfile, '.yaml')) -- 2.23.0

1 0

[PATCH v3 lkp-tests] stats/lmbench3: add new parsing parameters for performance test
by Bai Jing 20 Nov '20

20 Nov '20

[input] Memory read bandwidth 0.000512 16353.73 0.001024 16577.89 0.002048 16752.28 [output] L1_$: 1.539 L2_$: 0.539 Signed-off-by: Bai Jing <799286817(a)qq.com> --- stats/lmbench3 | 62 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/stats/lmbench3 b/stats/lmbench3 index 211c3abc..ba36dbb6 100755 --- a/stats/lmbench3 +++ b/stats/lmbench3 @@ -21,6 +21,27 @@ def largest_bandwidth bandwidth end +def designated_file_size + designated_hash = {} + result_str = '' + + $stdin.each_line do |line| + result_str += line + break if line =~ /(^\s|^\t)/ + end + + result_str.split("\n").each do |e| + designated_hash['L1_$'] = e.split[1] if e =~ /0.01758/ + designated_hash['Rand_mem'] = e.split[1] if e =~ /512.0000/ + designated_hash['Mmap_Latency'] = e.split[1] if e =~ /536.870912/ + designated_hash['L2_$'] = e.split[1] if e =~ /0.05078/ + end + + designated_hash +end + +null_io_array = [] + while (line = STDIN.gets) line = line.resolve_invalid_bytes case line @@ -32,14 +53,15 @@ while (line = STDIN.gets) # Simple fstat: 0.3517 microseconds # Simple open/close: 1.3696 microseconds when /^Simple (\S+): (\d+.\d+) microseconds$/ + null_io_array << $2.to_f if $1.to_s == 'read' || $1.to_s == 'write' puts "syscall.#{$1}.latency.us: #{$2}" # Extract select test result. # Select on 100 fd's: 1.2293 microseconds # Select on 100 tcp fd's: 5.0377 microseconds - when /^Select on 100 fd\'s: (\d+.\d+) microseconds$/ + when /^Select on 100 fd's: (\d+.\d+) microseconds$/ puts "Select.100fd.latency.us: #{$1}" - when /^Select on 100 tcp fd\'s: (\d+.\d+) microseconds$/ + when /^Select on 100 tcp fd's: (\d+.\d+) microseconds$/ puts "Select.100tcp.latency.us: #{$1}" # Extract proc test result. @@ -198,5 +220,41 @@ while (line = STDIN.gets) # ... when /^"Mmap read open2close bandwidth$/ puts "MMAP.read_open2close.bandwidth.MB/sec: #{largest_bandwidth}" + + # Random load latency + # 416.00000 92.824 + # 448.00000 92.772 + # 512.00000 92.640 + # ... + when /^Random load latency$/ + result_rand_mem = designated_file_size + puts "Rand_mem: #{result_rand_mem['Rand_mem']}" + + # Memory load latency + # 0.00049 1.539 + # 0.00098 1.539 + # 512.00000 74.421 + # ... + when /^Memory load latency$/ + result_rand_mem = designated_file_size + puts "L1_$: #{result_rand_mem['L1_$']}" + puts "L2_$: #{result_rand_mem['L2_$']}" + + # "mappings + # 67.108864 82 + # 134.217728 151 + # 268.435456 309 + # 536.870912 615 + # ... + when /^"mappings$/ + result_map = designated_file_size + puts "Mmap_Latency: #{result_map['Mmap_Latency']}" end end + +num = 0 +null_io_array.each do |v| + num += v +end +null_num = format('%.4f', num / 2).to_f +puts "null_io: #{null_num}" -- 2.23.0

1 0

[PATCH v2 compass-ci 2/3] git_mirror: fix that too few repos in upstream-repos cause memory leak
by Li Yuanchao 20 Nov '20

20 Nov '20

[why] when there is only a few repo files in upstream-repos, that means the same repo will be called very frequently. As the main loop is too fast, it seems the new flow would be called before the last flow of the same repo end completely. That cause a problem. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 72ca328..5b6b889 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -203,6 +203,7 @@ class MirrorMain push_git_queue handle_feedback Signal.trap(:SIGCHLD, 'SIG_IGN') + sleep(0.1) end end end -- 2.23.0

2 2

[PATCH v2 compass-ci 1/3] git_mirror: fix that README.md in upstream-repos cause a load error
by Li Yuanchao 20 Nov '20

20 Nov '20

[why] As README.md or other docs in upstream-repos, and they are not yaml file. But git-mirror load all files in upstream-repos as yaml file, so there comes out a load error. Now files whose dirname is 'xxx/upstream-repos' will not be loaded. [errmsg] in `load_repo_file': undefined method `[]=' for x:Integer (NoMethodError) Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 9648063..72ca328 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -135,6 +135,8 @@ class MirrorMain traverse_repodir("#{repodir}/#{entry}") end else + return if File.dirname(repodir) == REPO_DIR + project = File.dirname(repodir).delete_prefix("#{REPO_DIR}/") fork_name = File.basename(repodir) load_repo_file(repodir, project, fork_name) -- 2.23.0

2 2

[PATCH lkp-tests] daemon/multi-qemu: add a default value for the queues field
by Zhang Yu 20 Nov '20

20 Nov '20

[why] For multi-qemu.yaml, if the queues field is not set, a default value is required. [how] If the multi-qemu.yaml not set queues field, there are two queues by default: "$tbox_group.$HOSTNAME, $tbox_group.$(arch)". Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- daemon/multi-qemu | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/multi-qemu b/daemon/multi-qemu index 8d9dd38d..7cc625f4 100755 --- a/daemon/multi-qemu +++ b/daemon/multi-qemu @@ -6,6 +6,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +: ${queues:="$tbox_group.$HOSTNAME, $tbox_group.$(arch)"} multi_qemu() { export CCI_SRC=/c/compass-ci -- 2.23.0

3 3

[PATCH v2 lkp-tests] lib: fix request state
by Ren Wen 20 Nov '20

20 Nov '20

When run *start_daemon*, lkp reports a "finished" state to scheduler if daemon started. When run *run_test*, lkp also reports a "finished" state if tests finished. Now i want to stop machines after all tests finished, so alter state "finished" to "started" when running *start_daemon*. This makes it easier to handle request state at scheduler. Signed-off-by: Ren Wen <15991987063(a)163.com> --- lib/job.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/job.sh b/lib/job.sh index 81523a98..b84e6b18 100755 --- a/lib/job.sh +++ b/lib/job.sh @@ -278,7 +278,7 @@ start_daemon() run_program daemon "$@" fi - sync_cluster_state 'finished' + sync_cluster_state 'started' # If failed to start the daemon above, the job will abort. # LKP server on notice of the failed job will abort the other waiting nodes. -- 2.23.0

1 0

[PATCH lkp-tests] lib: fix request state
by Ren Wen 20 Nov '20

20 Nov '20

When run *start_daemon*, lkp reports a "finished" state to scheduler if daemon started. When run *run_test*, lkp also reports a "finished" state if tests finished. Now i want to stop machines after all tests finishing, so alter state "finished" to "started" when run *start_daemon*. This makes it easier to handle request state at scheduler. Signed-off-by: Ren Wen <15991987063(a)163.com> --- lib/job.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/job.sh b/lib/job.sh index 81523a98..b84e6b18 100755 --- a/lib/job.sh +++ b/lib/job.sh @@ -278,7 +278,7 @@ start_daemon() run_program daemon "$@" fi - sync_cluster_state 'finished' + sync_cluster_state 'started' # If failed to start the daemon above, the job will abort. # LKP server on notice of the failed job will abort the other waiting nodes. -- 2.23.0

2 2

[PATCH compass-ci] container/logging-es: fix failed to build images in x86 machine
by Liu Yinsi 20 Nov '20

20 Nov '20

[why] when build logging-es images in x86 machine, error message: libcontainer: container start initialization failed: standard_init_linux.go:220: exec user process caused "exec format error" The command '/bin/sh -c sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/elasticsearch/config/elasticsearch.yml && sed -i '$a path.data: /srv/es/logging-es' /usr/share/elasticsearch/config/elasticsearch.yml && sed -i '$a node.name: node-1' /usr/share/elasticsearch/config/elasticsearch.yml && sed -i '$a cluster.initial_master_nodes: ["node-1"]' /usr/share/elasticsearch/config/elasticsearch.yml && sed -i '$a indices.memory.index_buffer_size: 20%' /usr/share/elasticsearch/config/elasticsearch.yml && sed -i '$a thread_pool.write.queue_size: 2000' /usr/share/elasticsearch/config/elasticsearch.yml && sed -i "s/-Xms1g/-Xms30g/g" /usr/share/elasticsearch/config/jvm.options && sed -i "s/-Xmx1g/-Xmx30g/g" /usr/share/elasticsearch/config/jvm.options' returned a non-zero code: 1 because arm base image not support to build in x86 machine. [how] 1. use images dict to store arm and x86 base images 2. use $(arch) to choose base image according to different system architecture In addition, 'declare', arrays, arrays references are undefined in POSIX sh by shellcheck, so change '#!/bin/sh' to '#!/bin/bash'. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/logging-es/Dockerfile | 3 ++- container/logging-es/build | 13 +++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/container/logging-es/Dockerfile b/container/logging-es/Dockerfile index 707299f..6c01c43 100644 --- a/container/logging-es/Dockerfile +++ b/container/logging-es/Dockerfile @@ -1,7 +1,8 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM gagara/elasticsearch-oss-arm64:7.6.2 +ARG BASE_IMAGE +FROM $BASE_IMAGE # docker image borrowed from hub.docker.com/r/gagara/elasticsearch-oss-arm64 diff --git a/container/logging-es/build b/container/logging-es/build index 0ddecbf..451041b 100755 --- a/container/logging-es/build +++ b/container/logging-es/build @@ -1,5 +1,14 @@ -#!/bin/sh +#!/bin/bash # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -docker build -t logging-es:7.6.2 . +declare -A BASE_IMAGE_DICT + +BASE_IMAGE_DICT=( + ["aarch64"]="gagara/elasticsearch-oss-arm64:7.6.2" + ["x86_64"]="elasticsearch:7.6.2" +) + +BASE_IMAGE=${BASE_IMAGE_DICT[$(arch)]} + +docker build -t logging-es:7.6.2 --build-arg BASE_IMAGE="$BASE_IMAGE" . -- 2.23.0

1 0

[PATCH compass-ci] container/minio: fix build permission denied
by Liu Yinsi 20 Nov '20

20 Nov '20

lys@crystal ~/compass-ci/container/minio% ./build zsh: permission denied: ./build Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/minio/build | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 container/minio/build diff --git a/container/minio/build b/container/minio/build old mode 100644 new mode 100755 -- 2.23.0

1 0

[PATCH lkp-tests] stats: add new parsing parameters for performance test
by Bai Jing 20 Nov '20

20 Nov '20

Signed-off-by: Bai Jing <799286817(a)qq.com> --- stats/lmbench3 | 62 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/stats/lmbench3 b/stats/lmbench3 index 211c3abc..750e8bfc 100755 --- a/stats/lmbench3 +++ b/stats/lmbench3 @@ -21,6 +21,27 @@ def largest_bandwidth bandwidth end +def designated_file_size + designated_hash = {} + result_str = '' + + $stdin.each_line do |line| + result_str += line + break if line =~ /(^\s|^\t)/ + end + + result_str.split("\n").each do |e| + designated_hash['L1_$'] = e.split[1] if e =~ /0.01758/ + designated_hash['Rand_mem'] = e.split[1] if e =~ /512.0000/ + designated_hash['Mmap_Latency'] = e.split[1] if e =~ /536.870912/ + designated_hash['L2_$'] = e.split[1] if e =~ /0.05078/ + end + + designated_hash +end + +null_io_array = [] + while (line = STDIN.gets) line = line.resolve_invalid_bytes case line @@ -32,14 +53,15 @@ while (line = STDIN.gets) # Simple fstat: 0.3517 microseconds # Simple open/close: 1.3696 microseconds when /^Simple (\S+): (\d+.\d+) microseconds$/ + null_io_array << $2.to_f if $1.to_s == 'read' || $1.to_s == 'write' puts "syscall.#{$1}.latency.us: #{$2}" # Extract select test result. # Select on 100 fd's: 1.2293 microseconds # Select on 100 tcp fd's: 5.0377 microseconds - when /^Select on 100 fd\'s: (\d+.\d+) microseconds$/ + when /^Select on 100 fd's: (\d+.\d+) microseconds$/ puts "Select.100fd.latency.us: #{$1}" - when /^Select on 100 tcp fd\'s: (\d+.\d+) microseconds$/ + when /^Select on 100 tcp fd's: (\d+.\d+) microseconds$/ puts "Select.100tcp.latency.us: #{$1}" # Extract proc test result. @@ -198,5 +220,41 @@ while (line = STDIN.gets) # ... when /^"Mmap read open2close bandwidth$/ puts "MMAP.read_open2close.bandwidth.MB/sec: #{largest_bandwidth}" + + # "Random Memory load latency" + # 416.00000 92.824 + # 448.00000 92.772 + # 512.00000 92.640 + # ... + when /^Random load latency$/ + result_rand_mem = designated_file_size + puts "Rand_mem: #{result_rand_mem['Rand_mem']}" + + # "Memory load latency" + # 0.00049 1.539 + # 0.00098 1.539 + # 512.00000 74.421 + # ... + when /^Memory load latency$/ + result_rand_mem = designated_file_size + puts "L1_$: #{result_rand_mem['L1_$']}" + puts "L2_$: #{result_rand_mem['L2_$']}" + + # "mappings" + # 67.108864 82 + # 134.217728 151 + # 268.435456 309 + # 536.870912 615 + # ... + when /^"mappings$/ + result_map = designated_file_size + puts "Mmap_Latency: #{result_map['Mmap_Latency']}" end end + +num = 0 +null_io_array.each do |v| + num += v +end +null_num = format('%.4f', num / 2).to_f +puts "null_io: #{null_num}" -- 2.23.0

3 4

[PATCH v3 lkp-tests 6/6] iso2rootfs: allow the use of job.yaml and test script from the internet
by Yu Chuan 20 Nov '20

20 Nov '20

[Why] Not all the yaml and script of test job will be committed to the lkp-tests repo. [How] Support three optional fields for every test in job.yaml: - testX_git_url: the url of git repo which contains yaml and script; - testX_git_yaml: the relative path of the yaml file in the git repo; - testX_git_script: the relative path of the test script in the git repo; Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 5 +++++ tests/iso2rootfs | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 1631aca6d23f..1a643b538ac1 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -26,3 +26,8 @@ iso2rootfs: test1_yaml: test1_os_mount: test1_testbox: + ## the following three fields is required when your job.yaml and + ## script for this test are from the internet. + test1_git_url: + test1_git_yaml: + test1_git_script: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index 4135cdea7059..0353c045a4bc 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -457,10 +457,36 @@ generate_submit_append_str() echo "os=${iso_os} os_arch=${iso_arch} os_version=${iso_version} os_mount=$1 testbox=$2" } +prepare_submit() +{ + local submit_dir="/tmp/iso2rootfs/submit_tmp" + mkdir -p "${submit_dir}" + + local submit_git_dir="${submit_dir}/${test_git_url##*/}" + submit_git_dir="${submit_git_dir%.git}" + + [ -d "${submit_git_dir}" ] || { + cd "${submit_dir}" + git clone "${test_git_url}" + } + + local test_git_yaml="${submit_git_dir}/$(eval echo "\$${test_num}_git_yaml")" + local test_git_script="${submit_git_dir}/$(eval echo "\$${test_num}_git_script")" + + [ -f "${test_git_yaml}" ] || die "cannot find file: ${test_num}_git_yaml." + cp "${test_git_yaml}" "${LKP_SRC}/jobs/" + + [ -f "${test_git_script}" ] || die "cannot find file: ${test_num}_git_script." + cp "${test_git_script}" "${LKP_SRC}/tests/" +} + submit_one_yaml() { log_info "starting submit ${test_yaml} ..." + local test_git_url="$(eval echo "\$${test_num}_git_url")" + [ -z "${test_git_url}" ] || prepare_submit + cd "${LKP_SRC}/jobs" [ -f "${test_yaml}" ] || die "cannot find yaml in LKP_SRC/jobs: ${test_yaml}." -- 2.23.0

1 0

[PATCH v3 lkp-tests 5/6] iso2rootfs: allow submit multiple test job.yaml for result rootfs
by Yu Chuan 20 Nov '20

20 Nov '20

[Why] Result rootfs of dailybuild requires multiple test jobs. [How] 1. jobs/iso2rootfs.yaml. Before: test_yaml, test_os_mount, test_testbox After: test1_yaml, test1_os_mount, test1_testbox test2_yaml, test2_os_mount, test2_testbox ... 2. tests/iso2rootfs. - Use while-loop to traverse if $testX_yaml exists. - [ -n "${test1_yaml}" ] || return 0. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 8 ++++--- tests/iso2rootfs | 50 ++++++++++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 629b53febb2f..1631aca6d23f 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -21,6 +21,8 @@ iso2rootfs: dailybuild_iso_url_file: # submit test yaml related fields - test_yaml: - test_os_mount: - test_testbox: + ## 1. you can add as many jobs as you like. + ## 2. the following three fields is requied for every test job. + test1_yaml: + test1_os_mount: + test1_testbox: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index 3015c6f62947..4135cdea7059 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -457,17 +457,33 @@ generate_submit_append_str() echo "os=${iso_os} os_arch=${iso_arch} os_version=${iso_version} os_mount=$1 testbox=$2" } -submit_test_job_yaml() +submit_one_yaml() { - [ -n ${test_yaml} ] || return 0 - log_info "starting submit ${test_yaml} ..." - [ -d "${LKP_SRC}" ] || die "cannot find value of var: LKP_SRC." + cd "${LKP_SRC}/jobs" + [ -f "${test_yaml}" ] || die "cannot find yaml in LKP_SRC/jobs: ${test_yaml}." + + local test_os_mount="$(eval echo "\$${test_num}_os_mount")" + local test_testbox="$(eval echo "\$${test_num}_testbox")" + + [ -n "${test_os_mount}" ] || die "cannot find value of var: ${test_num}_os_mount." + [ -n "${test_testbox}" ] || die "cannot find value of var: ${test_num}_testbox." - [ -n "${test_os_mount}" ] || die "cannot find value of var: test_os_mount." - [ -n "${test_testbox}" ] || die "cannot find value of var: test_testbox." + "${LKP_SRC}/sbin/submit" \ + $(generate_submit_append_str "${test_os_mount}" "${test_testbox}") \ + "${test_yaml}" + log_info "submit ${test_yaml} finished" +} + +submit_yamls() +{ + [ -n "$test1_yaml" ] || return 0 + + log_info "starting submit test yamls ..." + + [ -d "${LKP_SRC}" ] || die "cannot find value of var: LKP_SRC." cd "${LKP_SRC}" && { chmod +x ./sbin/install-dependencies.sh ./sbin/install-dependencies.sh @@ -475,17 +491,19 @@ submit_test_job_yaml() generate_global_yaml } - cd "${LKP_SRC}/jobs" && { - [ -f "${test_yaml}" ] || - die "cannot find test yaml in LKP_SRC/jobs. - test yaml: ${test_yaml}." + local test_yaml_index=1 + local test_num="test${test_yaml_index}" + local test_yaml="$(eval echo "\$${test_num}_yaml")" - "${LKP_SRC}/sbin/submit" \ - $(generate_submit_append_str "${test_os_mount}" "${test_testbox}") \ - "${test_yaml}" - } + while [ -n "${test_yaml}" ] + do + submit_one_yaml + test_yaml_index=$((${test_yaml_index} +1)) + test_num="test${test_yaml_index}" + test_yaml="$(eval echo "\$${test_num}_yaml")" + done - log_info "submit ${test_yaml} finished" + log_info "submit test yamls finished" } ############ post works ############ @@ -525,7 +543,7 @@ main() config_rootfs test_rootfs - submit_test_job_yaml + submit_yamls post_works } -- 2.23.0

1 0