- Compass-ci - mailweb.openeuler.org

[PATCH v2 compass-ci] monitoring/filter.cr: query value support regularity
by Wu Zhende 04 Nov '20

04 Nov '20

[Why] Enhanced monitoring function. Regular expressions can be used in query's value. query can be {"job_id": "z9.*"} Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/monitoring/filter.cr | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/monitoring/filter.cr b/src/monitoring/filter.cr index bc6f901..d520e6e 100644 --- a/src/monitoring/filter.cr +++ b/src/monitoring/filter.cr @@ -56,13 +56,21 @@ class Filter def match_query(query : Hash(String, JSON::Any), msg : Hash(String, JSON::Any)) query.each do |key, value| + return false unless msg.has_key?(key) + value = value.as_a - if value.includes?(nil) - return false unless msg.has_key?(key) - else - return false unless value.includes?(msg[key]?) - end + next if value.includes?(nil) + next if value.includes?(msg[key]?) + + return false unless regular_match(value, msg[key]?.to_s) end return true end + + private def regular_match(rules, string) + rules.each do |rule| + return true if string =~ /#{rule}/ + end + return false + end end -- 2.23.0

2 1

[PATCH v1 compass-ci] improve multi-qemu code to support queue parameter
by Xiao Shenwei 04 Nov '20

04 Nov '20

Usage: multi-qemu [-n] [-c] [-q] -n, --name HOSTNAME_PREFIX specify used hostname_prefix -c, --count count how much VM do you need -q, --queues queues queues from which tasks are obtained -h, --help show this message example: ./multi-qemu -n vm-2p8g.xsw -c 2 -q vm-2p8g~xsw,vm-2p8g.aarch64 Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- providers/multi-qemu | 98 +++++++++++++++++++++++++++++++++----------- 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/providers/multi-qemu b/providers/multi-qemu index 23e0451..03564f7 100755 --- a/providers/multi-qemu +++ b/providers/multi-qemu @@ -4,41 +4,91 @@ # frozen_string_literal: true require 'fileutils' +require 'optparse' +require 'time' -PWD = Dir.pwd +opt = {} +options = OptionParser.new do |opts| + opts.banner = 'Usage: multi-qemu [-n] [-c] [-q]' + + opts.separator '' + opts.on('-n HOSTNAME_PREFIX', '--name HOSTNAME_PREFIX', 'specify used hostname_prefix') do |name| + opt['hostname_prefix'] = name + end + + opts.on('-c count', '--count count', 'how much VM do you need') do |num| + opt['nr_vm'] = num + end + + opts.on('-q queues', '--queues queues', 'requested queues, use "," to separate more than 2 values') do |queues| + opt['queues'] = queues + end + + opts.on_tail('-h', '--help', 'show this message') do + puts opts + exit + end +end + +if ARGV.size.zero? + puts options + exit +end + +options.parse!(ARGV) # Run multiple QEMU in parallel -HOSTNAME = ARGV[0] || "vm-2p8g--#{ENV['USER']}" -NR_VM = ARGV[1] || 1 +PWD = Dir.pwd +HOSTNAME = opt['hostname_prefix'] || "vm-2p8g.#{ENV['USER']}" +NR_VM = opt['nr_vm'] || 1 +QUEUES = opt['queues'] || "vm-2p8g.#{RUBY_PLATFORM.split('-')[0]}" +LOG_DIR = '/srv/cci/serial/logs' -def run(seqno) - loop do - start_time = Time.new - hostname = "#{HOSTNAME}-#{seqno}" - log_file = "/srv/cci/serial/logs/#{hostname}" +def main(hostname) + start_time = Time.new.strftime('%Y-%m-%d %H:%M:%S') + log_file = "#{LOG_DIR}/#{hostname}" + record_runtime(log_file, start_time) + run_vm(hostname) + duration = ((Time.new - Time.parse(start_time)) / 60).round(2) + record_runtime(log_file, duration, is_start: false) +end +def record_runtime(log_file, message, is_start: true) + if is_start File.open(log_file, 'w') do |f| # fluentd refresh time is 1s # let fluentd to monitor this file first sleep(2) - f.puts "\n#{start_time.strftime('%Y-%m-%d %H:%M:%S')} starting QEMU" + f.puts "\n#{message} starting QEMU" end + return + end + File.open(log_file, 'a') do |f| + f.puts "\nTotal QEMU duration: #{message} minutes" + end +end - pwd_hostname = File.join(PWD, hostname) - FileUtils.mkdir_p(pwd_hostname) unless File.exist?(pwd_hostname) - FileUtils.cd(pwd_hostname) - system( - { 'hostname' => hostname }, - ENV['CCI_SRC'] + '/providers/qemu.sh' - ) - - duration = ((Time.new - start_time) / 60).round(2) - File.open(log_file, 'a') do |f| - f.puts "\nTotal QEMU duration: #{duration} minutes" - end +def run_vm(hostname) + pwd_hostname = File.join(PWD, hostname) + FileUtils.mkdir_p(pwd_hostname) unless File.exist?(pwd_hostname) + FileUtils.cd(pwd_hostname) + system( + { 'hostname' => hostname, 'queues' => QUEUES }, + ENV['CCI_SRC'] + '/providers/qemu.sh' + ) +end - # sleep 5s is for fluentd to collect it's log - sleep(5) +def loop_main(hostname) + loop do + begin + main(hostname) + rescue StandardError => e + puts e.backtrace + # if an exception happend, request the next time after 30 seconds + sleep 25 + ensure + sleep 5 + end end end @@ -55,7 +105,7 @@ def multiqemu pids = [] NR_VM.to_i.times do |i| pid = Process.fork do - run i + loop_main("#{HOSTNAME}-#{i}") end pids << pid end -- 2.23.0

2 3

[PATCH v2 lkp-tests 1/3] (job2sh.rb) add on_fail to job.sh
by Hu Xuejiao 04 Nov '20

04 Nov '20

[why] If job.yaml is defined: on_fail: sleep(tests): 300(runtime) [how] By sh_on_fail, it will appear in job.sh: on_fail() { . $LKP_SRC/lib/http.sh . $LKP_SRC/lib/job.sh . $LKP_SRC/lib/env.sh export_top_env run_test $LKP_SRC/tests/wrapper sleep 500 } Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- lib/job2sh.rb | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/lib/job2sh.rb b/lib/job2sh.rb index 9c95ff4b..6e00b088 100755 --- a/lib/job2sh.rb +++ b/lib/job2sh.rb @@ -142,7 +142,7 @@ class Job2sh < Job end def parse_one(ancestors, key, val, pass) - if ['pp', 'define_files', 'monitors'].include?(key) + if ['pp', 'define_files', 'monitors','on_fail'].include?(key) return false end tabs = indent(ancestors) @@ -266,11 +266,13 @@ class Job2sh < Job sh_run_job(job) local_script_lines += @script_lines - sh_extract_stats(job) local_script_lines += @script_lines sh_define_files() + local_script_lines += @script_lines + + sh_on_fail(job) out_line '"$@"' @@ -357,6 +359,30 @@ class Job2sh < Job out_line "\tchmod +x $LKP_SRC/#{file}" end end - out_line '}' + out_line "}\n\n" + end + + def sh_on_fail(job = nil) + @script_lines = [] + + @programs = available_programs(:workload_elements) + + if !job + job = (@jobx || @job).clone + end + + @cur_func = :on_fail + + out_line 'on_fail()' + out_line '{' + out_line + out_line "\t. $LKP_SRC/lib/http.sh" + out_line "\t. $LKP_SRC/lib/job.sh" + out_line "\t. $LKP_SRC/lib/env.sh" + out_line + out_line "\texport_top_env" + out_line + parse_hash [], job['on_fail'] + out_line "}" end end -- 2.23.0

3 2

[PATCH lkp-tests] add testcase for sysbench-threads-git
by Hu Xuejiao 04 Nov '20

04 Nov '20

[why] old version package needs to debug by new testcase Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- tests/sysbench-threads-git | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 tests/sysbench-threads-git diff --git a/tests/sysbench-threads-git b/tests/sysbench-threads-git new file mode 100755 index 00000000..712b3359 --- /dev/null +++ b/tests/sysbench-threads-git @@ -0,0 +1,17 @@ +#!/bin/sh +# - nr_threads +# - thread_yields +# - thread_locks + +. "$LKP_SRC/lib/reproduce-log.sh" + +: "${nr_threads:=2}" +: "${thread_yields:=100}" +: "${thread_locks:=2}" + +args="\ + --threads=$nr_threads\ + --thread-yields=$thread_yields\ + --thread-locks=$thread_locks\ +" +log_cmd sysbench --test=threads $args run -- 2.23.0

2 1

[PATCH lkp-tests] add PKGBUILD for sysbench-threads-git
by Hu Xuejiao 04 Nov '20

04 Nov '20

[why] Adapting sysbench-threads needs to old version to user. Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- pkg/sysbench-threads-git/PKGBUILD | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 pkg/sysbench-threads-git/PKGBUILD diff --git a/pkg/sysbench-threads-git/PKGBUILD b/pkg/sysbench-threads-git/PKGBUILD new file mode 100644 index 00000000..2779c294 --- /dev/null +++ b/pkg/sysbench-threads-git/PKGBUILD @@ -0,0 +1,21 @@ +pkgname=sysbench-threads-git +pkgver=git +pkgrel=0.4.8 +pkgdesc="Benchmark tool for evaluating OS parameters that are important for a system running a database under intensive load." +url="http://github.com/dallasmarlow/sysbench.git" +arch=('x86_64' 'i386' 'aarch64') +license=('GPL') +source=("http://github.com/dallasmarlow/sysbench.git") +md5sums=('SKIP') + +build() { + cd "$srcdir/$pkgname" + ./autogen.sh + ./configure --prefix=/usr --without-gcc-arch --without-mysql + make +} + +package() { + cd "$srcdir/$pkgname" + make DESTDIR=$pkgdir install +} -- 2.23.0

1 0

[PATCH lkp-tests] add PKGBUILD for sysbech-threads-git
by Hu Xuejiao 04 Nov '20

04 Nov '20

[why] Adapting sysbench-threads needs to old version to user. Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- pkg/sysbench-threads-git/PKGBUILD | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 pkg/sysbench-threads-git/PKGBUILD diff --git a/pkg/sysbench-threads-git/PKGBUILD b/pkg/sysbench-threads-git/PKGBUILD new file mode 100644 index 00000000..bf47d5a1 --- /dev/null +++ b/pkg/sysbench-threads-git/PKGBUILD @@ -0,0 +1,21 @@ +pkgname=sysbench-threads-git +pkgver=git +pkgrel=0.4.8 +pkgdesc="Benchmark tool for evaluating OS parameters that are important for a system running a database under intensive load." +url="https://github.com/akopytov/sysbench.git" +arch=('x86_64' 'i386' 'aarch64') +license=('GPL') +source=("http://github.com/dallasmarlow/sysbench.git") +md5sums=('SKIP') + +build() { + cd "$srcdir/$pkgname" + ./autogen.sh + ./configure --prefix=/usr --without-gcc-arch --without-mysql + make +} + +package() { + cd "$srcdir/$pkgname" + make DESTDIR=$pkgdir install +} -- 2.23.0

1 0

[PATCH v2 lkp-tests 3/3] submit on_fail to the scheduler
by Hu Xuejiao 04 Nov '20

04 Nov '20

[why] When job2sh.rb defined sh_on_fail, it needs to be invoked by the scheduler. Finally, the task is in the waiting state when the task is failed. Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sbin/submit | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sbin/submit b/sbin/submit index bc90e2e0..0bb41f2d 100755 --- a/sbin/submit +++ b/sbin/submit @@ -115,12 +115,14 @@ ARGV.each do |jobfile| sh_run_job = job.sh_run_job sh_extract_stats = job.sh_extract_stats sh_define_files = job.sh_define_files + sh_on_fail = job.sh_on_fail sh_hash = { 'job2sh' => { 'run_job' => sh_run_job, 'extract_stats' => sh_extract_stats, - 'define_files' => sh_define_files + 'define_files' => sh_define_files, + 'on_fail' => sh_on_fail } } -- 2.23.0

1 0

[PATCH v2 lkp-tests 2/3] job-task is waiting when exit_code != 0
by Hu Xuejiao 04 Nov '20

04 Nov '20

[why] when a job-task fails(exit_code != 0), the on_fail function needs to be called to keep the program in the waiting state to user adjustment Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- bin/run-lkp | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/run-lkp b/bin/run-lkp index 26669331..c4d43db4 100755 --- a/bin/run-lkp +++ b/bin/run-lkp @@ -97,6 +97,7 @@ elif [ "$run_job_exit_code" != 0 ]; then else set_job_state 'failed' fi + $LKP_DEBUG_PREFIX $job_script on_fail elif [ -f '/var/log/wtmp' ] && has_cmd last && last -n1 | grep -q -E '(pts/|tty)[0-9].*(still logged in)'; then set_job_state 'disturbed' disturbed=1 -- 2.23.0

1 0

[PATCH v1 lkp-tests 2/2] accept queues parameter
by Xiao Shenwei 04 Nov '20

04 Nov '20

add queues field on multi-qemu.yaml: multi-qemu-0: nr_vm: 2 tbox_group: vm-2p8g queues: vm-2p8g.$HOSTNAME,xxx Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- daemon/multi-qemu | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/daemon/multi-qemu b/daemon/multi-qemu index 5f4d976b..3918a6f6 100755 --- a/daemon/multi-qemu +++ b/daemon/multi-qemu @@ -1,6 +1,7 @@ #!/bin/sh # - tbox_group # - nr_vm +# - queues # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. @@ -9,10 +10,10 @@ multi_qemu() { [ -f "/usr/share/qemu-efi-aarch64/QEMU_EFI.fd" ] || return export CCI_SRC=/c/compass-ci - local hostname=$tbox_group--$HOSTNAME + local hostname=$tbox_group.$HOSTNAME cd "$CACHE_DIR" - $CCI_SRC/providers/multi-qemu "$hostname" "$nr_vm" + $CCI_SRC/providers/multi-qemu -n "$hostname" -c "$nr_vm" -q "$queues" } multi_qemu -- 2.23.0

1 1

[PATCH v1 lkp-tests 1/2] check QEMU_EFI.fd file for multi-qemu
by Xiao Shenwei 04 Nov '20

04 Nov '20

[why] for multi-qemu job, the qemu-efi container will build error, then the QEMU_EFI.fd will not found, it caused one job will be remove from redis, but kvm can't execute it. so, this job looks like it's been executed, but not result file. Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- daemon/multi-qemu | 1 + 1 file changed, 1 insertion(+) diff --git a/daemon/multi-qemu b/daemon/multi-qemu index b407251f..5f4d976b 100755 --- a/daemon/multi-qemu +++ b/daemon/multi-qemu @@ -7,6 +7,7 @@ multi_qemu() { + [ -f "/usr/share/qemu-efi-aarch64/QEMU_EFI.fd" ] || return export CCI_SRC=/c/compass-ci local hostname=$tbox_group--$HOSTNAME -- 2.23.0

1 1

[PATCH compass-ci] sparrow/3-code: add ENV LOGGING_ES_HOST and LOGGING_ES_PORT
by Liu Yinsi 04 Nov '20

04 Nov '20

need LOGGING_ES_HOST and LOGGING_ES_PORT to start logging-es service. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 64f34f8..01393f8 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -9,6 +9,8 @@ server_ip=$(ip route get 1.2.3.4 | awk '{print $7; exit}') : ${initrd_http_port:=8800} : ${gitcache_host:=$server_ip} : ${gitcache_port:=5000} +: $logging_es_host:=$server_ip +: $logging_es_port:=9202 : ${MASTER_FLUENTD_HOST:=$server_ip} : ${MASTER_FLUENTD_PORT:=24224} : ${DOCKER_REGISTRY_HOST:=$server_ip} @@ -28,6 +30,8 @@ MASTER_FLUENTD_HOST: $MASTER_FLUENTD_HOST MASTER_FLUENTD_PORT: $MASTER_FLUENTD_PORT DOCKER_REGISTRY_HOST: $DOCKER_REGISTRY_HOST DOCKER_REGISTRY_PORT: $DOCKER_REGISTRY_PORT +LOGGING_ES_HOST: $logging_es_host +LOGGING_ES_PORT: $logging_es_port EOF cat > /etc/profile.d/compass.sh <<'EOF' -- 2.23.0

2 2

[PATCH v4 compass-ci 3/3] container/assign-account: answerback-email.rb
by Luan Shengde 04 Nov '20

04 Nov '20

1. disable use golbal variable apply_info [why]: rubocop: Style/GlobalVars: Do not introduce global variables. [how]: use local variables instead of global variables 2. add transfer user info when execute apply account command [why]: when applying account, the assign-account service need to write the my info to user's default config file ~/.config/compass-ci/defaults/account.yaml my info: - my_email - my_name - my_uuid [how]: transfer the user info along with the pub_key when sending apply account request Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/answerback-email.rb | 80 ++++++++++++-------- 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index bb8e809..5e6cc58 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -12,35 +12,38 @@ require 'mail' require 'set' require 'optparse' require_relative '../defconfig' +require_relative '../../lib/es_client' names = Set.new %w[ JUMPER_HOST JUMPER_PORT - SEND_MAIL_HOST_INTERNET - SEND_MAIL_PORT_INTERNET + SEND_MAIL_HOST + SEND_MAIL_PORT ] defaults = relevant_defaults(names) JUMPER_HOST = defaults['JUMPER_HOST'] || 'api.compass-ci.openeuler.org' JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 -SEND_MAIL_HOST = defaults['SEND_MAIL_HOST_INTERNET'] || 'localhost' -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT_INTERNET'] || 11312 +SEND_MAIL_HOST = defaults['SEND_MAIL_HOST'] || 'localhost' +SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 49000 -$apply_info = { +apply_info = { 'my_email' => nil, + 'my_name' => nil, + 'my_uuid' => %x(uuidgen).chomp, 'my_ssh_pubkey' => nil } -def init_info(email_file) +def init_info(email_file, apply_info) mail_content = Mail.read(email_file) + apply_info['my_email'] = mail_content.from[0] + apply_info['my_name'] = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') + apply_info['my_ssh_pubkey'] = if mail_content.part[1].filename == 'id_rsa.pub' + mail_content.part[1].body.decoded + end - $apply_info['my_email'] = mail_content.from[0] - $apply_info['my_ssh_pubkey'] = if mail_content.part[1].filename == 'id_rsa.pub' - mail_content.part[1].body.decoded.gsub(/\r|\n/, '') - end - - $apply_info + apply_info end options = OptionParser.new do |opts| @@ -52,15 +55,17 @@ options = OptionParser.new do |opts| opts.separator 'options:' opts.on('-e|--email email_address', 'appoint email address') do |email_address| - $apply_info['my_email'] = email_address + apply_info['my_email'] = email_address + # when apply account with email address, will get no user name + apply_info['my_name'] = '' end opts.on('-s|--ssh-pubkey pub_key_file', 'ssh pub_key file, enable password-less login') do |pub_key_file| - $apply_info['my_ssh_pubkey'] = File.read(pub_key_file) + apply_info['my_ssh_pubkey'] = File.read(pub_key_file) end opts.on('-f|--raw-email email_file', 'email file') do |email_file| - init_info(email_file) + init_info(email_file, apply_info) end opts.on_tail('-h|--help', 'show this message') do @@ -71,10 +76,10 @@ end options.parse!(ARGV) -def build_message(email, acct_infos) +def build_message(email, account_info) message = <<~EMAIL_MESSAGE To: #{email} - Subject: jumper account is ready + Subject: [compass-ci] jumper account is ready Dear user: @@ -82,10 +87,10 @@ def build_message(email, acct_infos) You can use the following command to login the jumper server: login command: - ssh -p #{acct_infos['jumper_port']} #{acct_infos['account']}@#{acct_infos['jumper_ip']} + ssh -p #{account_info['jumper_port']} #{account_info['my_login_name']}@#{account_info['jumper_host']} account password: - #{acct_infos['passwd']} + #{account_info['my_password']} regards compass-ci @@ -94,26 +99,37 @@ def build_message(email, acct_infos) return message end -def account_info(pub_key) - account_info_str = if pub_key.nil? - %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account') - else - %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") - end +def apply_account(apply_info) + account_info_str = %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d '#{apply_info.to_json}') JSON.parse account_info_str end -def send_account +def send_account(apply_info) message = "No email address specified\n" message += "use -e email_address add a email address\n" message += 'or use -f to add a email file' - raise message if $apply_info['my_email'].nil? - - acct_info = account_info($apply_info['my_ssh_pubkey']) - - message = build_message($apply_info['my_email'], acct_info) + raise message if apply_info['my_email'].nil? + + account_info = apply_account(apply_info) + my_info = { + 'my_email' => apply_info['my_email'], + 'my_name' => apply_info['my_name'], + # there is no need to add a commit url for this tool + # the es has the key: my_commit_url + 'my_commit_url' => '', + 'my_login_name' => account_info['my_login_name'], + 'my_uuid' => apply_info['my_uuid'] + } + + store_account_info(my_info) + message = build_message(apply_info['my_email'], account_info) %x(curl -XPOST '#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end -send_account +def store_account_info(my_info) + es = ESClient.new(index: 'accounts') + es.put_source_by_id(my_info['my_email'], my_info) +end + +send_account(apply_info) -- 2.23.0

1 0

[PATCH v4 compass-ci 2/3] container/assign-account: get_account_info
by Luan Shengde 04 Nov '20

04 Nov '20

add function: config default yaml file [why]: easier for user to config the default yaml file [how]: write received user infos to the defaults yaml file: ~/.config/compass-ci/default/account.yaml include: - my_email - my_name - my_uuid Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/get_account_info.rb | 51 +++++++++++++------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/container/assign-account/get_account_info.rb b/container/assign-account/get_account_info.rb index 2f93d5b..b2d2474 100755 --- a/container/assign-account/get_account_info.rb +++ b/container/assign-account/get_account_info.rb @@ -44,6 +44,8 @@ the returned data for setup_jumper_account_info like: =end +require 'fileutils' + # get jumper and account info class AccountStorage ACCOUNT_DIR = '/opt/account_data/' @@ -93,35 +95,48 @@ class AccountStorage def setup_jumper_account_info account_info = read_account_info jumper_info = read_jumper_info - pub_key = @data['pub_key'] unless @data.nil? - - jumper_ip = jumper_info[0].chomp - jumper_port = jumper_info[1].chomp - account = account_info[0] - passwd = if pub_key.nil? - account_info[1] - else - 'Use pub_key to login' - end + pub_key = @data['my_ssh_pubkey'] unless @data['my_ssh_pubkey'].nil? + + login_name = account_info[0] + password = if pub_key.nil? + account_info[1] + else + 'Use pub_key to login' + end + jumper_account_info = { - 'account' => account, - 'passwd' => passwd, - 'jumper_ip' => jumper_ip, - 'jumper_port' => jumper_port + 'my_login_name' => login_name, + 'my_password' => password, + 'jumper_host' => jumper_info[0].chomp, + 'jumper_port' => jumper_info[1].chomp } - setup_authorized_key(account, pub_key) + setup_authorized_key(login_name, pub_key) unless pub_key.nil? + setup_default_yaml(login_name) + return jumper_account_info end - def setup_authorized_key(account, pub_key) - ssh_dir = File.join('/home/', account, '.ssh') + def setup_default_yaml(login_name) + default_yaml_dir = File.join('/home', login_name, '.config/compass-ci/defaults') + FileUtils.mkdir_p default_yaml_dir + + File.open("#{default_yaml_dir}/account.yaml", 'a') do |file| + file.puts "my_email: #{@data['my_email']}" + file.puts "my_name: #{@data['my_name']}" + file.puts "my_uuid: #{@data['my_uuid']}" + end + %x(chown -R #{login_name}:#{login_name} "/home/#{login_name}/.config") + end + + def setup_authorized_key(login_name, pub_key) + ssh_dir = File.join('/home/', login_name, '.ssh') Dir.mkdir ssh_dir, 0o700 Dir.chdir ssh_dir f = File.new('authorized_keys', 'w') f.puts pub_key f.close File.chmod 0o600, 'authorized_keys' - %x(chown -R #{account}:#{account} #{ssh_dir}) + %x(chown -R #{login_name}:#{login_name} #{ssh_dir}) end end -- 2.23.0

1 0

[PATCH v4 compass-ci 1/3] container/assign-account: assign-account.rb
by Luan Shengde 04 Nov '20

04 Nov '20

disable assigning account for user if there is no: - my_email - my_name - my_uuid [why]: my_email, my_name, my_uuid is required when initialize the default config file [how]: check if the parsed data has keys: - my_email - my_name - my_uuid Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/assign-account.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/container/assign-account/assign-account.rb b/container/assign-account/assign-account.rb index e356c18..af10524 100755 --- a/container/assign-account/assign-account.rb +++ b/container/assign-account/assign-account.rb @@ -7,7 +7,7 @@ require 'sinatra' require 'open3' require 'json' require 'yaml' -require_relative 'get_account_info.rb' +require_relative 'get_account_info' set :bind, '0.0.0.0' set :port, 29999 @@ -17,8 +17,17 @@ get '/assign_account' do data = YAML.safe_load request.body.read rescue StandardError => e puts e.message + puts e.backtrace end + check_to_assign_account(data) +end + +def check_to_assign_account(data) + lack_info = %w[my_email my_name my_uuid] - data.keys + error_message = "lack of my infos: #{lack_info.join(', ')}." + raise error_message unless lack_info.empty? + ref_account_info = AccountStorage.new(data) account_info = ref_account_info.setup_jumper_account_info -- 2.23.0

1 0

[PATCH v3 lkp-tests 3/3] tests/iozone: modify the way of parsing parameter
by Lu Kaiyi 04 Nov '20

04 Nov '20

[why] iozone-bs.yaml has combined multiple parameter to single, so, iozone tool also need change the way of parsing parameter. [how] modify the way of parsing parameter. Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- tests/iozone | 65 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/tests/iozone b/tests/iozone index 88a92a18..b85bc4a9 100755 --- a/tests/iozone +++ b/tests/iozone @@ -1,42 +1,51 @@ #!/bin/sh # - block_size # - file_size -# - write_rewrite -# - read_reread -# - random_read_write -# - read_backwards -# - rewrite_record -# - stride_read -# - fwrite_refwrite -# - fread_refread -# - random_mix -# - pwrite_repwrite -# - pread_repread -# - pwritev_repwritev -# - preadv_repreadv +# - test ## IOzone is a filesystem benchmark tool. The benchmark generates ## and measures a variety of file operations. . $LKP_SRC/lib/reproduce-log.sh - args="iozone" if [ -n "$block_size" ]; then args+=" -r $block_size" - [ -n "$file_size" ] && args+=" -s $file_size" - [ -n "$write_rewrite" ] && args+=" -i 0" - [ -n "$read_reread" ] && args+=" -i 1" - [ -n "$random_read_write" ] && args+=" -i 2" - [ -n "$read_backwards" ] && args+=" -i 3" - [ -n "$rewrite_record" ] && args+=" -i 4" - [ -n "$stride_read" ] && args+=" -i 5" - [ -n "$fwrite_refwrite" ] && args+=" -i 6" - [ -n "$fread_refread" ] && args+=" -i 7" - [ -n "$random_mix" ] && args+=" -i 8" - [ -n "$pwrite_repwrite" ] && args+=" -i 9" - [ -n "$pread_repread" ] && args+=" -i 10" - [ -n "$pwritev_repwritev" ] && args+=" -i 11" - [ -n "$preadv_repreadv" ] && args+=" -i 12" + [ -n "$file_size" ] && args+=" -s $file_size" + OLD_IFS="$IFS" + IFS="," + array=($test) + IFS="$OLD_IFS" + for ele in ${array[@]} + do + case $ele in + "write_rewrite") args+=" -i 0" + ;; + "read_reread") args+=" -i 1" + ;; + "random_read_write") args+=" -i 2" + ;; + "read_backwards") args+=" -i 3" + ;; + "rewrite_record") args+=" -i 4" + ;; + "stride_read") args+=" -i 5" + ;; + "fwrite_refwrite") args+=" -i 6" + ;; + "fread_refread") args+=" -i 7" + ;; + "random_mix") args+=" -i 8" + ;; + "pwrite_repwrite") args+=" -i 9" + ;; + "pread_repread") args+=" -i 10" + ;; + "pwritev_repwritev") args+=" -i 11" + ;; + "preadv_repreadv") args+=" -i 12" + ;; + esac + done else args+=" -a" fi -- 2.23.0

2 1

[PATCH v2 lab-z9] lab-z9/host: fix taishan200-2280-2s48p-256g--a95 to taishan200-2280-2s48p-512g--a95
by Zhang Yale 04 Nov '20

04 Nov '20

[Why] The memory of taishan200-2280-2s48p-256g--a95 is 512g, so need to fix 256g to 512g. Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- ...an200-2280-2s48p-256g--a95 => taishan200-2280-2s48p-512g--a95} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename hosts/{taishan200-2280-2s48p-256g--a95 => taishan200-2280-2s48p-512g--a95} (100%) diff --git a/hosts/taishan200-2280-2s48p-256g--a95 b/hosts/taishan200-2280-2s48p-512g--a95 similarity index 100% rename from hosts/taishan200-2280-2s48p-256g--a95 rename to hosts/taishan200-2280-2s48p-512g--a95 -- 2.23.0

1 0

Re: [PATCH v3 compass-ci 3/3] container/assign-account: answerback-email.rb
by Zhang Yuhang 04 Nov '20

04 Nov '20

Comment message not clear to specify patch info. Reply-To: In-Reply-To: <20201103075716.1022070-3-shdluan(a)163.com> On Tue, Nov 03, 2020 at 03:57:16PM +0800, Luan Shengde wrote: >1. disable use golbal variable apply_info spell mistake: global > [why] > Style/GlobalVars: Do not introduce global variables. > [how] > use local variables instead of global variables > >2. enable transfer user info when execute apply account command > [why] > when applying account, the assign-account service will write the > my info to user's default config file > ~/.config/compass-ci/defaults/account.yaml > my info: > - my_email > - my_name > - my_uuid > [how] > transfer the user info along with the pub_key when sending apply > account request. Suggest to split it into two patch. >Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> >--- > container/assign-account/answerback-email.rb | 80 ++++++++++++-------- > 1 file changed, 48 insertions(+), 32 deletions(-) > >diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb >index bb8e809..5e6cc58 100755 >--- a/container/assign-account/answerback-email.rb >+++ b/container/assign-account/answerback-email.rb >@@ -12,35 +12,38 @@ require 'mail' > require 'set' > require 'optparse' > require_relative '../defconfig' >+require_relative '../../lib/es_client' > > names = Set.new %w[ > JUMPER_HOST > JUMPER_PORT >- SEND_MAIL_HOST_INTERNET >- SEND_MAIL_PORT_INTERNET >+ SEND_MAIL_HOST >+ SEND_MAIL_PORT > ] > > defaults = relevant_defaults(names) > > JUMPER_HOST = defaults['JUMPER_HOST'] || 'api.compass-ci.openeuler.org' > JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 >-SEND_MAIL_HOST = defaults['SEND_MAIL_HOST_INTERNET'] || 'localhost' >-SEND_MAIL_PORT = defaults['SEND_MAIL_PORT_INTERNET'] || 11312 >+SEND_MAIL_HOST = defaults['SEND_MAIL_HOST'] || 'localhost' >+SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 49000 > >-$apply_info = { >+apply_info = { > 'my_email' => nil, >+ 'my_name' => nil, >+ 'my_uuid' => %x(uuidgen).chomp, > 'my_ssh_pubkey' => nil > } > >-def init_info(email_file) >+def init_info(email_file, apply_info) > mail_content = Mail.read(email_file) >+ apply_info['my_email'] = mail_content.from[0] >+ apply_info['my_name'] = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') >+ apply_info['my_ssh_pubkey'] = if mail_content.part[1].filename == 'id_rsa.pub' >+ mail_content.part[1].body.decoded >+ end > >- $apply_info['my_email'] = mail_content.from[0] >- $apply_info['my_ssh_pubkey'] = if mail_content.part[1].filename == 'id_rsa.pub' >- mail_content.part[1].body.decoded.gsub(/\r|\n/, '') >- end >- >- $apply_info >+ apply_info > end > > options = OptionParser.new do |opts| >@@ -52,15 +55,17 @@ options = OptionParser.new do |opts| > opts.separator 'options:' > > opts.on('-e|--email email_address', 'appoint email address') do |email_address| >- $apply_info['my_email'] = email_address >+ apply_info['my_email'] = email_address >+ # when apply account with email address, will get no user name >+ apply_info['my_name'] = '' > end > > opts.on('-s|--ssh-pubkey pub_key_file', 'ssh pub_key file, enable password-less login') do |pub_key_file| >- $apply_info['my_ssh_pubkey'] = File.read(pub_key_file) >+ apply_info['my_ssh_pubkey'] = File.read(pub_key_file) > end > > opts.on('-f|--raw-email email_file', 'email file') do |email_file| >- init_info(email_file) >+ init_info(email_file, apply_info) > end > > opts.on_tail('-h|--help', 'show this message') do >@@ -71,10 +76,10 @@ end > > options.parse!(ARGV) > >-def build_message(email, acct_infos) >+def build_message(email, account_info) > message = <<~EMAIL_MESSAGE > To: #{email} >- Subject: jumper account is ready >+ Subject: [compass-ci] jumper account is ready > > Dear user: > >@@ -82,10 +87,10 @@ def build_message(email, acct_infos) > You can use the following command to login the jumper server: > > login command: >- ssh -p #{acct_infos['jumper_port']} #{acct_infos['account']}@#{acct_infos['jumper_ip']} >+ ssh -p #{account_info['jumper_port']} #{account_info['my_login_name']}@#{account_info['jumper_host']} > > account password: >- #{acct_infos['passwd']} >+ #{account_info['my_password']} > > regards > compass-ci >@@ -94,26 +99,37 @@ def build_message(email, acct_infos) > return message > end > >-def account_info(pub_key) >- account_info_str = if pub_key.nil? >- %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account') >- else >- %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") >- end >+def apply_account(apply_info) >+ account_info_str = %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d '#{apply_info.to_json}') > JSON.parse account_info_str > end > >-def send_account >+def send_account(apply_info) The function name so wired. > message = "No email address specified\n" > message += "use -e email_address add a email address\n" > message += 'or use -f to add a email file' >- raise message if $apply_info['my_email'].nil? >- >- acct_info = account_info($apply_info['my_ssh_pubkey']) >- >- message = build_message($apply_info['my_email'], acct_info) >+ raise message if apply_info['my_email'].nil? Could we do like this? raise message unless apply_info['my_email'] Thanks, Zhang Yuhang >+ >+ account_info = apply_account(apply_info) >+ my_info = { >+ 'my_email' => apply_info['my_email'], >+ 'my_name' => apply_info['my_name'], >+ # there is no need to add a commit url for this tool >+ # the es has the key: my_commit_url >+ 'my_commit_url' => '', >+ 'my_login_name' => account_info['my_login_name'], >+ 'my_uuid' => apply_info['my_uuid'] >+ } >+ >+ store_account_info(my_info) >+ message = build_message(apply_info['my_email'], account_info) > > %x(curl -XPOST '#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") > end > >-send_account >+def store_account_info(my_info) >+ es = ESClient.new(index: 'accounts') >+ es.put_source_by_id(my_info['my_email'], my_info) >+end >+ >+send_account(apply_info) >-- >2.23.0 >

1 0

[PATCH lab-z9] lab-z9/hosts: rename taishan200-2280-2s48p-256g--a95 to taishan200-2280-2s48p-512g--a95
by Zhang Yale 04 Nov '20

04 Nov '20

[Why] The memory of taishan200-2280-2s48p-256g--a95 is 512g, so need to fix the host name to 512g. Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- hosts/taishan200-2280-2s48p-512g--a95 | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 hosts/taishan200-2280-2s48p-512g--a95 diff --git a/hosts/taishan200-2280-2s48p-512g--a95 b/hosts/taishan200-2280-2s48p-512g--a95 new file mode 100644 index 0000000..213d10f --- /dev/null +++ b/hosts/taishan200-2280-2s48p-512g--a95 @@ -0,0 +1,20 @@ +arch: aarch64 +nr_node: 4 +nr_cpu: 96 +memory: 512G +nr_hdd_partitions: 4 +hdd_partitions: + - /dev/disk/by-id/scsi-35000039988792e31 + - /dev/disk/by-id/scsi-35000039988793705 + - /dev/disk/by-id/scsi-35000039988792aa5 + - /dev/disk/by-id/scsi-350000399887932fd +mac_addr: + - cc:05:77:fd:fb:61 + - cc:05:77:fd:fb:62 + - cc:05:77:fd:fb:63 + - cc:05:77:fd:fb:64 + - cc:05:77:fd:fb:65 + - cc:05:77:fd:fb:66 + - cc:05:77:fd:fb:67 + - cc:05:77:fd:fb:68 +model_name: Kunpeng-920 -- 2.23.0

1 1

[PATCH compass-ci 1/2] compare values by each metrics
by Lu Weitao 04 Nov '20

04 Nov '20

compare values by each metrics based on groups matrices, and format compare result as echart data_set background: For support compare with user-defined template feature, the work-flow of user-defined template feature: load_compare_template.yaml --> query_results(ES) ---> auto group jobs_list ---> create groups_matrices ---> compare_values by each metrics ---> format/show results current patch is the step: compare_values by each metrics ---> format/show results Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/compare_matrixes.rb | 107 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/lib/compare_matrixes.rb b/lib/compare_matrixes.rb index 078028a..7e6dfb1 100644 --- a/lib/compare_matrixes.rb +++ b/lib/compare_matrixes.rb @@ -399,8 +399,115 @@ def compare_group_matrices(group_matrices, options) result_str end +# compare group_matrices with each dimension(matrix) of each group, +# and reorganize the result as group-metric-compare-values +# +# Input: groups_matrices +# { +# group_key_1 => { +# dimension_1 => matrix_1, (openeuler 20.03) +# dimension_2 => matrix_2, (openeuler 20.09) +# dimension_3 => matrix_3, (centos 7.6) +# }, +# group_key_2 => {...} +# } +# +# output: compare_metrics_values +# { +# group_key_1 => { +# metric_1 => { +# 'average' => { +# 'dimension_1' => xxx +# 'dimension_2' => xxx +# 'dimension_3' => xxx +# }, +# 'standard_deviation' => { +# 'dimension_1' => xxx +# 'dimension_2' => xxx +# 'dimension_3' => xxx +# }, +# 'change' => { +# 'dimension_2 vs dimension_1' => xxx +# 'dimension_3 vs dimension_1' => xxx +# 'dimension_3 vs dimension_2' => xxx +# } +# }, +# metric_2 => {...} +# } +# } +def compare_metrics_values(groups_matrices) + metrics_compare_values = {} + groups_matrices.each do |group_key, dimensions| + metrics_compare_values[group_key] = get_metric_values(dimensions) + end + metrics_compare_values +end + +def get_metric_values(dimensions) + metrics_values = {} + dimensions.each do |dim, matrix| + matrix.each do |metric, values| + metrics_values[metric] ||= {} + metrics_values[metric]['average'] ||= {} + metrics_values[metric]['standard_deviation'] ||= {} + metric_value = get_values(values, true) + metrics_values[metric]['average'][dim] = metric_value[:average] + metrics_values[metric]['standard_deviation'][dim] = metric_value[:stddev] + end + end + get_metric_change(metrics_values) + metrics_values +end + +def get_metric_change(metrics_values) + metrics_values.each do |metric, values| + metrics_values[metric]['change'] = {} + next if values['average'].size < 2 + + dimension_list = values['average'].keys + change_titles = get_change_titles(dimension_list) + change_titles.each do |pk| + change = get_compare_value(values['average'][pk[0]], values['average'][pk[1]], true) + values['change'] = { "#{pk[1]} vs #{pk[0]}" => change } + end + end +end + +# create all dimension combinations as change_title +# input: dimension_list +# eg: ['openeuler 20.03', 'debian sid', 'centos 7.6'] +# return: Array(base_dimension: String, pk_dimension: String) +# [ +# ['openeuler 20.03', 'debian sid'], +# ['openeuler 20.03', 'centos 7.6'], +# ['debian sid', 'centos 7.6'] +# ] +def get_change_titles(dimension_list) + dims = [] + dimension_list_size = dimension_list.size + (1..dimension_list_size - 1).each do |i| + (i..dimension_list_size - 1).each do |j| + dims << [dimension_list[i - 1], dimension_list[j]] + end + end + dims +end + +def show_chart_compare_result(chart_result) + print JSON.pretty_generate(chart_result) +end + # Format Fields +def format_for_chart(metrics_compare_values, template_params) + chart_result = {} + chart_result['title'] = template_params['title'] || nil + chart_result['unit'] = template_params['unit'] || nil + chart_result['x_name'] = template_params['x_params'].join('|') || nil + chart_result['tables'] = metrics_compare_values + chart_result +end + def format_fails_runs(fails, runs) fails_width = (SUB_LONG_COLUMN_WIDTH * FAILS_PROPORTION).to_i runs_width = SUB_LONG_COLUMN_WIDTH - fails_width - 1 -- 2.23.0

2 7

[PATCH v3 lkp-tests 2/3] jobs/iozone-bs.yaml: combine iozone's multiple -i parameter to single
by Lu Kaiyi 04 Nov '20

04 Nov '20

[why] avoid explosion of parameter for iozone-bs.yaml Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- jobs/iozone-bs.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jobs/iozone-bs.yaml b/jobs/iozone-bs.yaml index e2cd9f48..f9ab305f 100644 --- a/jobs/iozone-bs.yaml +++ b/jobs/iozone-bs.yaml @@ -2,9 +2,7 @@ suite: iozone category: benchmark file_size: 4g -write_rewrite: true -read_reread: true -random_read_write: true +test: write_rewrite,read_reread,random_read_write block_size: - 64k -- 2.23.0

1 0

[PATCH v3 lkp-tests 1/3] jobs/iozone-bs.yaml: fix mapper_parsing_exception
by Lu Kaiyi 04 Nov '20

04 Nov '20

[why] when key and value of fs: xfs in same line for iozone-bs.yaml, submit job will cause some error occurred like below: submit /home/lukaiyi/lkp-tests/jobs/iozone-bs.yaml failed, got job_id=z9.144590, error: [{"type" => "mapper_parsing_exception", "reason" => "failed to parse [pp.fs.fs]"}] [how] change key and value of fs: xfs to different line, same to iosched: kyber Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- jobs/iozone-bs.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jobs/iozone-bs.yaml b/jobs/iozone-bs.yaml index 868979fc..e2cd9f48 100644 --- a/jobs/iozone-bs.yaml +++ b/jobs/iozone-bs.yaml @@ -14,6 +14,11 @@ block_size: - 16m disk: 1HDD -fs: xfs -iosched: kyber + +fs: +- xfs + +iosched: +- kyber + iozone: -- 2.23.0

1 0

[PATCH compass-ci] dnsmasq: fix script will continue to execute when cp failed
by Yu Chuan 04 Nov '20

04 Nov '20

[Why] 'load custom dnsmasq.conf' step will cp custom config to WORKSPACE, but when a user without permission execute this step, cp will failed. So add 'set -e' at the beginning of the script to solve this problem. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- container/dnsmasq/start | 2 ++ 1 file changed, 2 insertions(+) diff --git a/container/dnsmasq/start b/container/dnsmasq/start index ff4247046eb1..b2eefbd89c16 100755 --- a/container/dnsmasq/start +++ b/container/dnsmasq/start @@ -2,6 +2,8 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +set -e + . $CCI_SRC/container/defconfig.sh docker_rm dnsmasq -- 2.23.0

3 5

[PATCH compass-ci 2/2] add template compare mode
by Lu Weitao 03 Nov '20

03 Nov '20

add template compare mode in sbin/compare(command-line tool) by: - load and parse compare.yaml - calls lib/es_query.rb, query jobs data set from ES - calls lib/matrix.rb, convert jobs data set to group matrices - calls lib/compare_matrixes.rb, use groups matrices to compare background: For support compare with user-defined template feature, the work-flow of user-defined template feature: load_compare_template.yaml --> query_results(ES) ---> auto group jobs_list ---> create groups_matrices ---> compare_values by each metrics ---> format/show results example: compare -t/--template compare_template.yaml, the yaml like: compare_metrics: - fio.write_iops - fio.read_iops filter: suite: - fio-basic os_arch: - aarch64 - x86 compare_dimensions: - os: debian os_version: sid - os: openeuler os_version: 20.03 x_params: - bs - test_size title: Hackbench Performance Testing unit: KB/s output:{ "title": "Hackbench Performance Testing", "unit": "KB/s", "x_name": "bs|test_size", "tables": { "4k|1G": { "fio.write_iops": { "average": { "openeuler 20.03": 308298.5524761111, "debian sid": 220289.07563 }, "standard_deviation": { "openeuler 20.03": null, "debian sid": null }, "change": { "debian sid vs openeuler 20.03": -28.5 } } }, "16k|1G": {...}, ... } } Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/compare.rb | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ sbin/compare | 16 +++++++++++++--- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/lib/compare.rb b/lib/compare.rb index 2837edc..26e6b49 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -6,6 +6,7 @@ require_relative './es_query.rb' require_relative './matrix2.rb' require_relative './compare_matrixes.rb' require_relative './constants.rb' +require 'yaml' # ------------------------------------------------------------------------------------------- # compare_matrices_list @@ -66,3 +67,54 @@ def create_groups_matrices_list(conditions, dims) query_results = es.multi_field_query(conditions) combine_group_query_data(query_results, dims) end + +# ------------------------------------------------------------------------------------------- +# compare with user defined compare_template.yaml +# the data change flow: +# load_compare_template.yaml --> query_results(ES) ---> auto_group_jobs_list +# ---> groups_matrices ---> compare_values ---> format/show_results +# + +def compare_by_template(template) + template_params = load_template(template) + groups_matrices = create_groups_matrices(template_params) + compare_values = compare_metrics_values(groups_matrices) + chart_results = format_for_chart(compare_values, template_params) + show_chart_compare_result(chart_results) +end + +def load_template(template) + unless File.file?(template) + warn 'template does not exist' + exit + end + YAML.safe_load(File.open(template)) +end + +# according to user's template to create groups_matrices +# input: template_params: Hash +# eg: +# { +# "compare_metrics"=>["fio.write_iops", "fio.read_iops"], +# "filter"=>[ +# {"suite"=>["fio-bisic"]}, +# {"os_arch"=>["aarch_64"]} +# ], +# "compare_dimensions"=>[ +# {"os"=>"openeuler", "os_version"=>20.03}, +# {"os"=>"centos", "os_version"=>7.6} +# ], +# "x_params"=>["block_size", "package_size"], +# "title"=>"Hackbench Performance Testing", +# "unit"=>"KB/s" +# } +def create_groups_matrices(template_params) + es = ESQuery.new + query_results = es.multi_field_query(template_params['filter']) + combine_group_query_data( + query_results, + template_params['x_params'], + template_params['compare_dimensions'], + template_params['compare_metrics'] + ) +end diff --git a/sbin/compare b/sbin/compare index cf3980c..b1bc49a 100755 --- a/sbin/compare +++ b/sbin/compare @@ -6,6 +6,7 @@ # Usage: # compare "conditions_1" "conditions_2" ... -c "common_conditions" # compare "conditions" -d "dimensions" +# compare -t template.yaml # Eg: # compare "id=6000,6001" "id=7000,7001" # compare "commit=a12d232e" "commit=b3bacc31" @@ -21,6 +22,7 @@ is_group = false dimensions = nil colorful = nil options = {} +template = nil opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: compare "conditions" ... [option]' @@ -45,6 +47,10 @@ opt_parser = OptionParser.new do |opts| colorful = color end + opts.on('-t', '--template template', 'compare with user-defined template') do |t| + template = t + end + opts.on_tail('-h', '--help', 'show this message') do puts opts exit @@ -60,8 +66,12 @@ opt_parser.parse!(argv) options = { theme: colorful } if colorful -if is_group - compare_group(argv, dimensions, options) +if template + compare_by_template(template) else - compare_matrices_list(argv, common_conditions, options) + if is_group + compare_group(argv, dimensions, options) + else + compare_matrices_list(argv, common_conditions, options) + end end -- 2.23.0

2 2

[PATCH lkp-tests] job.rb: fix add second program to pp
by Wei Jihui 03 Nov '20

03 Nov '20

[Why] if we add new field in job yaml like this: on_fail: sleep: 300 if will do not add sleep to pp [How] when add sleep to pp, we check sleep's parent hash, not all job. it will find sleep in on_fail'keys. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/job.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/job.rb b/lib/job.rb index 5c2c9517..50323d31 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -576,10 +576,10 @@ class Job @job[comment_to_symbol('auto generated by add_pp()')] = nil @job["pp"] = Hash.new() init_program_options() - for_each_in(@job, @referenced_programs.keys) do |_pk, h, p_n, p_args| - k = hash_key_re_string(@job, p_n, @job['pp'].keys) - if k && !(a)job[k].is_a?(Hash) - @job['pp'][k] = @job[k] + for_each_in(@job, @referenced_programs.keys) do |_pk, h, p_n, _p_args| + k = hash_key_re_string(h, p_n, @job['pp'].keys) + if k && !h[k].is_a?(Hash) && !h[k].nil? + @job['pp'][k] = h[k] next end options_hash = @referenced_programs[p_n] -- 2.23.0

3 2

[PATCH v2 compass-ci] specifies the queue from which the VM obtains tasks
by Xiao Shenwei 03 Nov '20

03 Nov '20

[why] specify the VM that need to consume the queues one job will added one queue, but one VM can consume job from multi queues. [how] to solve this problem, the scheduler should obtains which queues should be request. solution-1: register mac hostname and queues relation before: mac2host change to: mac2host and host2queues then scheduler can get queues based on mac solution-2: /boot.ipxe/mac/${mac}?queues=xxx,yyy our HW and PXE-VM use /tftpboot/boot.ipxe to send request, unable to specify parameter, so solution-1 may be better. Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- providers/my-qemu.sh | 4 +++- providers/qemu.sh | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/providers/my-qemu.sh b/providers/my-qemu.sh index 954d3a0..8845a34 100755 --- a/providers/my-qemu.sh +++ b/providers/my-qemu.sh @@ -4,6 +4,8 @@ [[ $tbox_group ]] || tbox_group=vm-2p8g -export hostname=$tbox_group--$USER-$$ +export hostname=$tbox_group.$USER-$$ +# specify which queues will be request, use " " to separate more than 2 values +export queues="$tbox_group~$USER" $CCI_SRC/providers/qemu.sh diff --git a/providers/qemu.sh b/providers/qemu.sh index d97fd85..6c61f86 100755 --- a/providers/qemu.sh +++ b/providers/qemu.sh @@ -9,6 +9,7 @@ load_cci_defaults : ${hostname:="vm-1p1g-1"} +: ${queues:="vm-1p1g.$(arch)"} # unicast prefix: x2, x6, xA, xE export mac=$(echo $hostname | md5sum | sed 's/^$..$$..$$..$$..$$..$.*$/0a-\1-\2-\3-\4-\5/') echo hostname: $hostname @@ -17,14 +18,22 @@ echo $mac > mac echo "arp -n | grep ${mac//-/:}" > ip.sh chmod +x ip.sh -curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/set_host_mac?hostname=${hostname}&mac=${mac}" +set_host_info() +{ + # use "," replace " " + local api_queues=$(echo $queues | sed -r 's/ +/,/g') + curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/set_host_mac?hostname=${hostname}&mac=${mac}" + curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/set_host2queues?host=${hostname}&queues=${api_queues}" +} +set_host_info -del_host_mac() +del_host_info() { curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/del_host_mac?mac=${mac}" > /dev/null 2>&1 + curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/del_host2queues?host=${hostname}" > /dev/null 2>&1 } -trap del_host_mac EXIT +trap del_host_info EXIT ( if [[ $hostname =~ ^(.*)-[0-9]+$ ]]; then @@ -33,7 +42,7 @@ trap del_host_mac EXIT tbox_group=$hostname fi - host=${tbox_group%%--*} + host=${tbox_group%.*} create_yaml_variables "$LKP_SRC/hosts/${host}" -- 2.23.0

3 4

help star
by Wu Fengguang 03 Nov '20

03 Nov '20

Xueliang is our code star, thanks Xueliang! wfg@crystal /c/todo% bin/help-star people/*/credit/2020-10-30.md 24.2 Cao Xueliang 18.2 Wu Fengguang 18.0 Yu Chuan 16.9 Wei Jihui 15.0 Wu Zhende 13.6 Liu Shaofei 9.8 Liu Yinsi 9.7 Xu Xijian 8.9 Wang Yong 8.8 Lu Weitao 8.7 Zhou Shengrui 8.1 Tong Qunfeng 7.7 Zhang Yuhang 7.4 Bai Jing 6.2 Cui Yili 6.0 Xiao Shenwei 5.6 Luan Shengde 5.6 Shi Zhichao 5.2 Wang Chenglong 5.1 Zhang Dewan 4.8 Zhang Yale 4.5 Zhang Yu 4.0 Lin Jiaxin 3.9 Sun Yukui 3.5 Li Ping 3.4 Ren Wen 2.3 Du Kaitian 2.3 Hu Xuejiao 1.5 Li Yuanchao 1.1 Lu Kaiyi 0.0 Wang Shuai 0.0 Cui Xucui

2 2

[PATCH compass-ci] os-cifs: keep the container name consistent with the name in code
by Yu Chuan 03 Nov '20

03 Nov '20

[Why] Current container name is 'samba', it is neither easy to remember nor easy to search. So change the container name from 'samba' to 'os-cifs'. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- container/os-cifs/start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/os-cifs/start b/container/os-cifs/start index cd33e207c709..6cc08c629fbd 100755 --- a/container/os-cifs/start +++ b/container/os-cifs/start @@ -17,7 +17,7 @@ cmd=( -v /etc/localtime:/etc/localtime:ro -v /srv/initrd:/srv/initrd -v /srv/result:/srv/result - --name samba + --name os-cifs --restart=always alpine/samba ) -- 2.23.0

2 1

[PATCH compass-ci] taskqueue: close connection after API call
by Cao Xueliang 03 Nov '20

03 Nov '20

[why] most client call to taskqueue do not need keep-alive. when API call is finished, close the connection. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/taskqueue/taskqueue.cr | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/taskqueue/taskqueue.cr b/src/taskqueue/taskqueue.cr index 13f48db..f41cfa8 100644 --- a/src/taskqueue/taskqueue.cr +++ b/src/taskqueue/taskqueue.cr @@ -42,6 +42,7 @@ class TaskQueue # # response: TaskQueue(a)v0.0.1 is alive. get "/" do |env| + env.response.headers["Connection"] = "close" response = "TaskQueue@v#{VERSION} is alive." debug_message(env, response, Time.utc) @@ -60,6 +61,7 @@ class TaskQueue # 400 "Missing parameter <queue>" # 400 "Missing http body" post "/add" do |env| + env.response.headers["Connection"] = "close" response = queue_respond_add(env) debug_message(env, response, Time.utc) response if env.response.status_code == 200 @@ -73,6 +75,7 @@ class TaskQueue # 201 ## when there has no task in queue (scheduler/$tbox_group) # 400 "Missing parameter <queue>" put "/consume" do |env| + env.response.headers["Connection"] = "close" response = queue_respond_consume(env) debug_message(env, response, Time.utc) response if env.response.status_code == 200 @@ -86,6 +89,7 @@ class TaskQueue # 400 "Missing parameter <from|to|id>" # 409 "Can not find id <$id> in queue <scheduler/$tbox_group>" put "/hand_over" do |env| + env.response.headers["Connection"] = "close" response = queue_respond_hand_over(env) debug_message(env, response, Time.utc) nil @@ -99,6 +103,7 @@ class TaskQueue # 400 "Missing parameter <queue|id>" # 409 "Can not find id <$id> in queue <scheduler/$tbox_group>" put "/delete" do |env| + env.response.headers["Connection"] = "close" response = queue_respond_delete(env) debug_message(env, response, Time.utc) nil @@ -114,6 +119,7 @@ class TaskQueue # 400 "Missing parameter <queue>" # 413 "Query results too large keys" get "/keys" do |env| + env.response.headers["Connection"] = "close" response = queue_respond_keys(env) # debug_message(env, response, Time.utc) # maybe too large response.to_json unless env.response.status_code == 201 -- 2.23.0

1 0

[PATCH compass-ci] lkp-initrd: add step: rsync the result lkp.cgz if needed
by Yu Chuan 03 Nov '20

03 Nov '20

[Why] About this container: - Everyone will run this container. - Docker permission is required to run this script. Generally, the production env will control docker permission, and will not give docker permission to everyone. So add the rsync step to allow the execution of local rsync script to perform result lkp.cgz synchronization. > local rsync script path: /usr/local/bin/rsync-lkp-cgz Signed-off-by: Yu Chuan <13186087857(a)163.com> --- container/lkp-initrd/run | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/container/lkp-initrd/run b/container/lkp-initrd/run index 522f5098ddc2..e2bff72ab83b 100755 --- a/container/lkp-initrd/run +++ b/container/lkp-initrd/run @@ -24,3 +24,8 @@ cmd=( "${cmd[@]}" echo "result: /srv/initrd/lkp/${lkp_initrd_user:-latest}/lkp-${ARCH}.cgz" + +# rsync lkp-${ARCH}.cgz if needed +if [ -f "/usr/local/bin/rsync-lkp-cgz" ]; then + . "/usr/local/bin/rsync-lkp-cgz" +fi -- 2.23.0

2 2

[PATCH v3 compass-ci 2/2] create matrices-groups according to group jobs
by Lu Weitao 03 Nov '20

03 Nov '20

create matrices-groups according to auto group job_list background: For support compare with user-defined template feature, the work-flow are: load_compare_template.yaml --> query_results(ES) ---> auto group jobs_list ---> create groups_matrices ---> compare_values by each metrics ---> format/show results current patch is the step: create groups_matrices Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/matrix2.rb | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index 05f96dc..a284c98 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -120,3 +120,28 @@ def combine_group_query_data(query_data, dims) end groups end + +# combine_group_query_data: combine query data from ES to group matrix +# input: +# 1. query results(job_list) from es_query that will be auto group by auto_group_by_template() +# 2. params from user's template include: +# groups_params(x_params): +# eg: ['block_size', 'package_size'] +# dimensions: +# eg: [ +# {'os' => 'openeuler', 'os_version' => '20.03'}, +# {'os' => 'centos', 'os_version' => '7.6'} +# ] +# metrics: +# eg: ['fio.read_iops', 'fio_write_iops'] +# return: group_matrix of Hash(String, Hash(String, matrix)) +def combine_group_query_data(query_data, groups_params, dimensions, metrics) + job_list = query_data['hits']['hits'] + groups = auto_group_by_template(job_list, groups_params, dimensions, metrics) + groups.each do |group_key, dimensions| + dimensions.each do |dimension_key, jobs| + groups[group_key][dimension_key] = create_matrix(jobs) + end + end + groups +end -- 2.23.0

3 4

[PATCH lkp-tests] lib/job.rb: no need puts the a warning
by Li Ping 03 Nov '20

03 Nov '20

[why] we can get hostname from the repo lab-z9 there is no need to check hostname file whether exist under the path lkp-tests/hosts --- lib/job.rb | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/job.rb b/lib/job.rb index 5c2c9517..1948a479 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -341,13 +341,6 @@ class Job if lab_hosts_file hosts_file = lab_hosts_file end - - if File.file?(hosts_file) - hosts_file - else - puts("hosts_file not exist: #{hosts_file}, maybe need check testbox field") - nil - end end def include_files -- 2.23.0

2 2

[PATCH v3 compass-ci 1/2] auto group jobs with user-defined template
by Lu Weitao 03 Nov '20

03 Nov '20

auto group jobs(query form ES) with user-defined template background: For support compare with user-defined template feature, the work-flow are: load_compare_template.yaml --> query_results(ES) ---> auto group jobs_list ---> create groups_matrices ---> compare_values by each metrics ---> format/show results current patch is the step: auto group jobs_list Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/params_group.rb | 121 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 120 insertions(+), 1 deletion(-) diff --git a/lib/params_group.rb b/lib/params_group.rb index 0f07d26..15bc2dd 100644 --- a/lib/params_group.rb +++ b/lib/params_group.rb @@ -2,7 +2,7 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true -# Exammple: +# Example: # Input: jobs_list. The results of ES query. # # eg: [ jobs1, jobs2, ...... ] @@ -113,3 +113,122 @@ end def remove_singleton(groups) groups.delete_if { |_k, v| v.length < 2 } end + +# -------------------------------------------------------------------------------------------------- +# auto group job_list by user's template +# Example: +# Input: +# 1. jobs_list. +# 2. params from user's template that include: +# groups_params(x_params): +# eg: ['block_size', 'package_size'] +# dimensions: +# eg: [ +# {'os' => 'openeuler', 'os_version' => '20.03'}, +# {'os' => 'centos', 'os_version' => '7.6'} +# ] +# metrics: +# eg: ['fio.read_iops', 'fio_write_iops'] +# Output: +# eg: +# { +# '4K|1G' => { +# 'openeuler 20.03' => [ +# {'stats' => {'fio.write_iops' => 312821.002387, 'fio.read_iops' => 212821.2387}}, +# {'stats' => {'fio.write_iops' => 289661.878453}}, +# ... +# ], +# 'centos 7.6' => [...] +# }, +# '16K|1G' => {...}, +# ... +# } + +# auto_group_by_template: auto group job_list by user's template +def auto_group_by_template(jobs_list, group_params, dimensions, metrics) + job_list = extract_jobs_list(jobs_list) + get_group_by_template(job_list, group_params, dimensions, metrics) +end + +def get_group_by_template(job_list, group_params, dimensions, metrics) + groups = {} + job_list.each do |job| + next unless job['stats'] + + group_key = get_user_group_key(job, group_params) + dimension = get_user_dimension(job, dimensions) + next unless group_key && dimension + + new_job = get_new_job_by_metrics(job, metrics) + next if new_job.empty? + + groups[group_key] ||= {} + groups[group_key][dimension] ||= [] + groups[group_key][dimension] << new_job + end + groups +end + +# @group_params Array(String) +# eg: +# ['block_size', 'package_size'] +# return eg: +# '4K|1G' +def get_user_group_key(job, group_params) + group_key_list = [] + group_params.each do |param| + if job.key?(param) + group_key_list << job[param] + next + end + job.each_value do |v| + if v.is_a?(Hash) && v.key?(param) + group_key_list << v[param] + break + end + end + end + return nil if group_key_list.size < group_params.size || group_key_list.empty? + + group_key_list.join('|') +end + +# @dimension Array(Hash) +# eg: +# [ +# {os => openeuler, os_version => 20.03}, +# {os => centos, os_version => 7.6} +# ] +# return eg: +# 'openeuler 20.03' +def get_user_dimension(job, dimensions) + dimension_list = [] + dimensions.each do |dim| + dim.each do |key, value| + if job[key] == value + dimension_list << value + end + end + return nil if !dimension_list.empty? && dimension_list.size < dim.size + end + return nil if dimension_list.empty? + + dimension_list.join(' ') +end + +# @metrics Array(String) +# eg: +# ["fio.read_iops", "fio.write_iops"] +# return new_job +# eg: +# {'stats' => {'fio.write_iops' => 312821.002387, 'fio.read_iops' => 212821.2387}}, +def get_new_job_by_metrics(job, metrics) + new_job = {} + metrics.each do |metric| + if job['stats'].key?(metric) + new_job['stats'] ||= {} + new_job['stats'][metric] = job['stats'][metric] + end + end + new_job +end -- 2.23.0

3 4

[PATCH compass-ci 2/8] container/mail-robot: apply-account.rb
by Luan Shengde 03 Nov '20

03 Nov '20

check to assign account invoke ParseApplyAccountEmail to parse commit url and pubkey generate uuid and invoke ApplyAccount to apply account store account info to es send uuid Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/mail-robot/apply-account.rb | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100755 container/mail-robot/apply-account.rb diff --git a/container/mail-robot/apply-account.rb b/container/mail-robot/apply-account.rb new file mode 100755 index 0000000..55245ac --- /dev/null +++ b/container/mail-robot/apply-account.rb @@ -0,0 +1,71 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'mail' +require_relative "../../lib/es_client.rb" + +# assign uuid/account +class AssignAccount + def initialize(mail_content) + @mail_content = mail_content + @my_email = mail_content.from[0] + @my_name = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') + @send_mail_host = %x(/sbin/ip route |awk '/default/ {print $3}').chomp + end + + def check_to_send_account + my_commit_url, pub_key = check_out_commit_url_pub_key + my_uuid, acct_infos = check_to_apply_account(pub_key) + + my_info = { + 'my_email' => @my_email, + 'my_name' => @my_name, + 'my_commit_url' => my_commit_url, + 'my_login_name' => acct_infos['account'], + 'my_uuid' => my_uuid + } + store_account_info(my_info) + my_info['acct_infos'] = acct_infos + send_my_uuid(my_info) + end + + def check_out_commit_url_pub_key + parse_apply_account_email = ParseApplyAccountEmail.new(@mail_content) + my_commit_url, pub_key = parse_apply_account_email.parse_commit_url_pub_key + + return my_commit_url, pub_key + end + + def check_to_apply_account(pub_key) + my_uuid = %x(uuidgen).chomp + apply_account_info = { + 'my_email' => @my_email, + 'my_name' => @my_name, + 'my_uuid' => my_uuid, + 'my_ssh_pubkey' => pub_key + } + apply_account = ApplyAccount.new(apply_account_info) + acct_infos = apply_account.apply_jumper_account + + return my_uuid, acct_infos + end + + def store_account_info(my_info) + es = ESClient.new(index: 'accounts') + es.put_source_by_id(my_info['my_email'], my_info) + end + + def send_my_uuid(my_info) + send_uuid_email_info = { + 'my_email' => @my_email, + 'my_name' => @my_name, + 'my_info' => my_info, + 'error_message' => '' + } + send_uuid = SendMail.new(send_uuid_email_info) + send_uuid.send_uuid_email + end +end -- 2.23.0

2 3

[PATCH compass-ci] monitoring/filter.cr: query value support regularity
by Wu Zhende 03 Nov '20

03 Nov '20

[Why] Enhanced monitoring function. Regular expressions can be used in query's value. query can be {"job_id": "z9.*"} Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/monitoring/filter.cr | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/monitoring/filter.cr b/src/monitoring/filter.cr index bc6f901..5ddd547 100644 --- a/src/monitoring/filter.cr +++ b/src/monitoring/filter.cr @@ -56,12 +56,20 @@ class Filter def match_query(query : Hash(String, JSON::Any), msg : Hash(String, JSON::Any)) query.each do |key, value| + return false unless msg.has_key?(key) + value = value.as_a - if value.includes?(nil) - return false unless msg.has_key?(key) - else - return false unless value.includes?(msg[key]?) + next if value.includes?(nil) + next if value.includes?(msg[key]?) + + flag = false + value.each do |val| + if msg[key]?.to_s =~ /#{val}/ + flag = true + break + end end + return false unless flag end return true end -- 2.23.0

1 0

[PATCH v9 compass-ci 1/2] job.cr: add a key "kernel_version" for initramfs
by Xu Xijian 03 Nov '20

03 Nov '20

When run job with initramfs, there is a default combination of kernel, modules and headers, and I add a key "kernel_version" to make it optional. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/job.cr | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 1e49da6..6309809 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -87,6 +87,10 @@ class Job kernel_append_root kernel_params docker_image + kernel_version + linux_vmlinuz_path + linux_modules_initrd + linux_headers_initrd ) macro method_missing(call) @@ -135,6 +139,7 @@ class Job set_result_service() set_os_mount() set_depends_initrd() + set_kernel_version() set_initrds_uri() set_kernel_uri() set_kernel_append_root() @@ -293,9 +298,17 @@ class Job return true end + private def set_kernel_version + boot_dir = "#{SRV_OS}/#{os_dir}/boot" + suffix = "-#{kernel_version}" if self["kernel_version"]? + self["linux_vmlinuz_path"] = File.real_path("#{boot_dir}/vmlinuz#{suffix}") + self["linux_modules_initrd"] = File.real_path("#{boot_dir}/modules#{suffix}.cgz") + self["linux_headers_initrd"] = File.real_path("#{boot_dir}/headers#{suffix}.cgz") + end + private def set_kernel_uri self["kernel_uri"] = "kernel #{OS_HTTP_PREFIX}" + - "#{JobHelper.service_path("#{SRV_OS}/#{os_dir}/vmlinuz")}" + "#{JobHelper.service_path("#{linux_vmlinuz_path}")}" end private def common_initrds @@ -316,6 +329,10 @@ class Job "#{JobHelper.service_path("#{osimage_dir}/current")}" temp_initrds << "#{INITRD_HTTP_PREFIX}" + "#{JobHelper.service_path("#{osimage_dir}/run-ipconfig.cgz")}" + temp_initrds << "#{OS_HTTP_PREFIX}" + + "#{JobHelper.service_path("#{linux_modules_initrd}")}" + temp_initrds << "#{OS_HTTP_PREFIX}" + + "#{JobHelper.service_path("#{linux_headers_initrd}")}" temp_initrds.concat(initrd_deps.split(/ /)) unless initrd_deps.empty? temp_initrds.concat(initrd_pkg.split(/ /)) unless initrd_pkg.empty? -- 2.23.0

1 0

[PATCH v3 compass-ci 3/3] container/assign-account: answerback-email.rb
by Luan Shengde 03 Nov '20

03 Nov '20

1. disable use golbal variable apply_info [why] Style/GlobalVars: Do not introduce global variables. [how] use local variables instead of global variables 2. enable transfer user info when execute apply account command [why] when applying account, the assign-account service will write the my info to user's default config file ~/.config/compass-ci/defaults/account.yaml my info: - my_email - my_name - my_uuid [how] transfer the user info along with the pub_key when sending apply account request. Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/answerback-email.rb | 80 ++++++++++++-------- 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index bb8e809..5e6cc58 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -12,35 +12,38 @@ require 'mail' require 'set' require 'optparse' require_relative '../defconfig' +require_relative '../../lib/es_client' names = Set.new %w[ JUMPER_HOST JUMPER_PORT - SEND_MAIL_HOST_INTERNET - SEND_MAIL_PORT_INTERNET + SEND_MAIL_HOST + SEND_MAIL_PORT ] defaults = relevant_defaults(names) JUMPER_HOST = defaults['JUMPER_HOST'] || 'api.compass-ci.openeuler.org' JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 -SEND_MAIL_HOST = defaults['SEND_MAIL_HOST_INTERNET'] || 'localhost' -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT_INTERNET'] || 11312 +SEND_MAIL_HOST = defaults['SEND_MAIL_HOST'] || 'localhost' +SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 49000 -$apply_info = { +apply_info = { 'my_email' => nil, + 'my_name' => nil, + 'my_uuid' => %x(uuidgen).chomp, 'my_ssh_pubkey' => nil } -def init_info(email_file) +def init_info(email_file, apply_info) mail_content = Mail.read(email_file) + apply_info['my_email'] = mail_content.from[0] + apply_info['my_name'] = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') + apply_info['my_ssh_pubkey'] = if mail_content.part[1].filename == 'id_rsa.pub' + mail_content.part[1].body.decoded + end - $apply_info['my_email'] = mail_content.from[0] - $apply_info['my_ssh_pubkey'] = if mail_content.part[1].filename == 'id_rsa.pub' - mail_content.part[1].body.decoded.gsub(/\r|\n/, '') - end - - $apply_info + apply_info end options = OptionParser.new do |opts| @@ -52,15 +55,17 @@ options = OptionParser.new do |opts| opts.separator 'options:' opts.on('-e|--email email_address', 'appoint email address') do |email_address| - $apply_info['my_email'] = email_address + apply_info['my_email'] = email_address + # when apply account with email address, will get no user name + apply_info['my_name'] = '' end opts.on('-s|--ssh-pubkey pub_key_file', 'ssh pub_key file, enable password-less login') do |pub_key_file| - $apply_info['my_ssh_pubkey'] = File.read(pub_key_file) + apply_info['my_ssh_pubkey'] = File.read(pub_key_file) end opts.on('-f|--raw-email email_file', 'email file') do |email_file| - init_info(email_file) + init_info(email_file, apply_info) end opts.on_tail('-h|--help', 'show this message') do @@ -71,10 +76,10 @@ end options.parse!(ARGV) -def build_message(email, acct_infos) +def build_message(email, account_info) message = <<~EMAIL_MESSAGE To: #{email} - Subject: jumper account is ready + Subject: [compass-ci] jumper account is ready Dear user: @@ -82,10 +87,10 @@ def build_message(email, acct_infos) You can use the following command to login the jumper server: login command: - ssh -p #{acct_infos['jumper_port']} #{acct_infos['account']}@#{acct_infos['jumper_ip']} + ssh -p #{account_info['jumper_port']} #{account_info['my_login_name']}@#{account_info['jumper_host']} account password: - #{acct_infos['passwd']} + #{account_info['my_password']} regards compass-ci @@ -94,26 +99,37 @@ def build_message(email, acct_infos) return message end -def account_info(pub_key) - account_info_str = if pub_key.nil? - %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account') - else - %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") - end +def apply_account(apply_info) + account_info_str = %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d '#{apply_info.to_json}') JSON.parse account_info_str end -def send_account +def send_account(apply_info) message = "No email address specified\n" message += "use -e email_address add a email address\n" message += 'or use -f to add a email file' - raise message if $apply_info['my_email'].nil? - - acct_info = account_info($apply_info['my_ssh_pubkey']) - - message = build_message($apply_info['my_email'], acct_info) + raise message if apply_info['my_email'].nil? + + account_info = apply_account(apply_info) + my_info = { + 'my_email' => apply_info['my_email'], + 'my_name' => apply_info['my_name'], + # there is no need to add a commit url for this tool + # the es has the key: my_commit_url + 'my_commit_url' => '', + 'my_login_name' => account_info['my_login_name'], + 'my_uuid' => apply_info['my_uuid'] + } + + store_account_info(my_info) + message = build_message(apply_info['my_email'], account_info) %x(curl -XPOST '#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end -send_account +def store_account_info(my_info) + es = ESClient.new(index: 'accounts') + es.put_source_by_id(my_info['my_email'], my_info) +end + +send_account(apply_info) -- 2.23.0

1 0

[PATCH v3 compass-ci 2/3] container/assign-account: get_account_info
by Luan Shengde 03 Nov '20

03 Nov '20

add new function: config default yaml file [why]: easier for user to config the default yaml file [how]: parse received data and extract the user infos store user infos to defaults yaml file: ~/.config/compass-ci/default/account.yaml include: - my_email - my_name - my_uuid Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/get_account_info.rb | 50 +++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/container/assign-account/get_account_info.rb b/container/assign-account/get_account_info.rb index 2f93d5b..51e3e03 100755 --- a/container/assign-account/get_account_info.rb +++ b/container/assign-account/get_account_info.rb @@ -44,6 +44,8 @@ the returned data for setup_jumper_account_info like: =end +require 'fileutils' + # get jumper and account info class AccountStorage ACCOUNT_DIR = '/opt/account_data/' @@ -93,35 +95,49 @@ class AccountStorage def setup_jumper_account_info account_info = read_account_info jumper_info = read_jumper_info - pub_key = @data['pub_key'] unless @data.nil? - - jumper_ip = jumper_info[0].chomp - jumper_port = jumper_info[1].chomp - account = account_info[0] - passwd = if pub_key.nil? - account_info[1] - else - 'Use pub_key to login' - end + pub_key = @data['my_ssh_pubkey'] unless @data['my_ssh_pubkey'].nil? + + jumper_host = jumper_info[0].chomp + jumper_port = jumper_info[1].chomp + login_name = account_info[0] + password = if pub_key.nil? + account_info[1] + else + 'Use pub_key to login' + end jumper_account_info = { - 'account' => account, - 'passwd' => passwd, - 'jumper_ip' => jumper_ip, + 'my_login_name' => login_name, + 'my_password' => password, + 'jumper_host' => jumper_host, 'jumper_port' => jumper_port } - setup_authorized_key(account, pub_key) + setup_authorized_key(login_name, pub_key) unless pub_key.nil? + setup_default_yaml(login_name) + return jumper_account_info end - def setup_authorized_key(account, pub_key) - ssh_dir = File.join('/home/', account, '.ssh') + def setup_default_yaml(login_name) + default_yaml_dir = File.join('/home', login_name, '.config/compass-ci/defaults') + FileUtils.mkdir_p default_yaml_dir + + File.open("#{default_yaml_dir}/account.yaml", 'a') do |file| + file.puts "my_email: #{@data['my_email']}" + file.puts "my_name: #{@data['my_name']}" + file.puts "my_uuid: #{@data['my_uuid']}" + end + %x(chown -R #{login_name}:#{login_name} "/home/#{login_name}/.config") + end + + def setup_authorized_key(login_name, pub_key) + ssh_dir = File.join('/home/', login_name, '.ssh') Dir.mkdir ssh_dir, 0o700 Dir.chdir ssh_dir f = File.new('authorized_keys', 'w') f.puts pub_key f.close File.chmod 0o600, 'authorized_keys' - %x(chown -R #{account}:#{account} #{ssh_dir}) + %x(chown -R #{login_name}:#{login_name} #{ssh_dir}) end end -- 2.23.0

1 0

[PATCH v3 compass-ci 1/3] container/assign-account: assign-account.rb
by Luan Shengde 03 Nov '20

03 Nov '20

disable assigning account for user if there is no: - my_email - my_name - my_uuid [why]: my_email, my_name, my_uuid is required when initialize the default config file [how] check if the parsed data has keys: - my_email - my_name - my_uuid Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/assign-account.rb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/container/assign-account/assign-account.rb b/container/assign-account/assign-account.rb index e356c18..9481068 100755 --- a/container/assign-account/assign-account.rb +++ b/container/assign-account/assign-account.rb @@ -7,7 +7,7 @@ require 'sinatra' require 'open3' require 'json' require 'yaml' -require_relative 'get_account_info.rb' +require_relative 'get_account_info' set :bind, '0.0.0.0' set :port, 29999 @@ -17,8 +17,22 @@ get '/assign_account' do data = YAML.safe_load request.body.read rescue StandardError => e puts e.message + puts e.backtrace end + check_to_assign_account(data) +end + +def check_to_assign_account(data) + error_message = 'lack of my infos: my_email' + raise error_message unless data.key? 'my_email' + + error_message = 'lack of my infos: my_name' + raise error_message unless data.key? 'my_name' + + error_message = 'lack of my infos: my_uuid' + raise error_message unless data.key? 'my_uuid' + ref_account_info = AccountStorage.new(data) account_info = ref_account_info.setup_jumper_account_info -- 2.23.0

1 0

[PATCH compass-ci] taskqueue_api: each interface uses its own client to request taskqueue
by Cao Xueliang 03 Nov '20

03 Nov '20

[why] Different interfaces use one client may get incorrect data. So, each interface uses its own client to request taskqueue and close it when finish the request. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/lib/taskqueue_api.cr | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/lib/taskqueue_api.cr b/src/lib/taskqueue_api.cr index a8bbf16..f447266 100644 --- a/src/lib/taskqueue_api.cr +++ b/src/lib/taskqueue_api.cr @@ -17,20 +17,23 @@ end class TaskQueueAPI def initialize - port = ENV.has_key?("TASKQUEUE_PORT") ? ENV["TASKQUEUE_PORT"].to_i32 : 3060 - host = ENV.has_key?("TASKQUEUE_HOST") ? ENV["TASKQUEUE_HOST"] : "172.17.0.1" - @client = HTTP::Client.new(host, port: port) + @port = ENV.has_key?("TASKQUEUE_PORT") ? ENV["TASKQUEUE_PORT"].to_i32 : 3060 + @host = ENV.has_key?("TASKQUEUE_HOST") ? ENV["TASKQUEUE_HOST"] : "172.17.0.1" end def add_task(service_queue_path : String, task : JSON::Any) params = HTTP::Params.encode({"queue" => service_queue_path}) - response = loop_till_connectable { @client.post("/add?" + params, body: task.to_json) } + client = HTTP::Client.new(@host, port: @port) + response = loop_till_connectable { client.post("/add?" + params, body: task.to_json) } + client.close() arrange_response(response) end def query_keys(service_key_with_wild_char : String) params = HTTP::Params.encode({"queue" => service_key_with_wild_char}) - response = loop_till_connectable { @client.get("/keys?" + params) } + client = HTTP::Client.new(@host, port: @port) + response = loop_till_connectable { client.get("/keys?" + params) } + client.close() arrange_response(response) end @@ -50,7 +53,9 @@ class TaskQueueAPI end private def response_put_api(cmd : String, params : String) - response = loop_till_connectable { @client.put("/#{cmd}?" + params) } + client = HTTP::Client.new(@host, port: @port) + response = loop_till_connectable { client.put("/#{cmd}?" + params) } + client.close() arrange_response(response) end -- 2.23.0

1 0

[PATCH lkp-tests] sbin/monitor: fix JSON.parse error
by Wu Zhende 03 Nov '20

03 Nov '20

[Why] query's value has three types. Can be "xxx", nil, "[xxx]". Only "[xxx]" can do JSON.parse. [Error1] Traceback (most recent call last): 6: from /c/lkp-tests/sbin/monitor:73:in `<main>' 5: from /home/code/lkp-tests/lib/monitor.rb:103:in `run' 4: from /home/code/lkp-tests/lib/monitor.rb:103:in `each' 3: from /home/code/lkp-tests/lib/monitor.rb:104:in `block in run' 2: from /usr/share/ruby/json/common.rb:156:in `parse' 1: from /usr/share/ruby/json/common.rb:156:in `new' /usr/share/ruby/json/common.rb:156:in `initialize': no implicit conversion of nil into String (TypeError) [Error2] Traceback (most recent call last): 6: from /c/lkp-tests/sbin/monitor:73:in `<main>' 5: from /home/code/lkp-tests/lib/monitor.rb:103:in `run' 4: from /home/code/lkp-tests/lib/monitor.rb:103:in `each' 3: from /home/code/lkp-tests/lib/monitor.rb:104:in `block in run' 2: from /usr/share/ruby/json/common.rb:156:in `parse' 1: from /usr/share/ruby/json/common.rb:156:in `new' /usr/share/ruby/json/common.rb:156:in `initialize': no implicit conversion of nil into String (TypeError) Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/monitor.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/monitor.rb b/lib/monitor.rb index e0098390..56283e10 100755 --- a/lib/monitor.rb +++ b/lib/monitor.rb @@ -102,6 +102,7 @@ class Monitor @query.each do |k, v| @query[k] = JSON.parse(v) + rescue end query = @query.to_json puts "query=>#{query}" -- 2.23.0

1 0

[PATCH v2 lkp-tests 2/3] jobs/iozone-bs.yaml: combine iozone's multiple -i parameter to single
by Lu Kaiyi 03 Nov '20

03 Nov '20

[why] avoid explosion of parameter for iozone-bs.yaml Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- jobs/iozone-bs.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jobs/iozone-bs.yaml b/jobs/iozone-bs.yaml index e2cd9f48..f9ab305f 100644 --- a/jobs/iozone-bs.yaml +++ b/jobs/iozone-bs.yaml @@ -2,9 +2,7 @@ suite: iozone category: benchmark file_size: 4g -write_rewrite: true -read_reread: true -random_read_write: true +test: write_rewrite,read_reread,random_read_write block_size: - 64k -- 2.23.0

2 3

[PATCH v7 compass-ci 2/2] kernel_version.md: explain key "kernel_version"
by Xu Xijian 03 Nov '20

03 Nov '20

[why] Explain the meaning of new key "kernel_version" for scheduler, including what's the mapping between its typical values and actual files in the disk and how to set it. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- doc/job/kernel_version.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/job/kernel_version.md diff --git a/doc/job/kernel_version.md b/doc/job/kernel_version.md new file mode 100644 index 0000000..d76a3bd --- /dev/null +++ b/doc/job/kernel_version.md @@ -0,0 +1,18 @@ +# kernel_version + +Meaning: +- Every os has its kernel, however an os can start with different kernels according to different need. +- kernel_version is a key for users to specify a kernel version. +- If kernel_version is not given by users, it will use the default one. + +Related files: +- In initramfs boot process, every kernel version is related with a vmlinuz, module and headers. +- Files like below under $boot_dir, an example $boot_dir can be "/srv/os/openeuler/aarch64/20.03/boot". +├── headers-4.19.90-2003.cgz +├── headers.cgz -> headers-4.19.90-2003.cgz +├── modules-4.19.90-2003.cgz +├── modules.cgz -> modules-4.19.90-2003.cgz +├── vmlinuz-4.19.90-2003 + +Usage example: +- submit iperf.yaml testbox=vm-hi1620-2p8g--$USER os=openeuler os_arch=aarch64 os_version=20.03 runtime=20 kernel_version=4.19.90-2003 -- 2.23.0

2 3

[PATCH compass-ci] specifies the queue from which the VM obtains tasks
by Xiao Shenwei 03 Nov '20

03 Nov '20

[why] specify the VM that need to consume the queues one job will added one queue, but one VM can consume job from multi queues. [how] to solve this problem, the scheduler should obtains which queues should be request. solution-1: register mac hostname and queues relation before: mac2host change to: mac2host and host2queues then scheduler can get queues based on mac solution-2: /boot.ipxe/mac/${mac}?queues=xxx,yyy our HW and PXE-VM use /tftpboot/boot.ipxe to send request, unable to specify parameter, so solution-1 may be better. Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- providers/my-qemu.sh | 4 +++- providers/qemu.sh | 18 ++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/providers/my-qemu.sh b/providers/my-qemu.sh index 954d3a0..f42a1fd 100755 --- a/providers/my-qemu.sh +++ b/providers/my-qemu.sh @@ -4,6 +4,8 @@ [[ $tbox_group ]] || tbox_group=vm-2p8g -export hostname=$tbox_group--$USER-$$ +export hostname=$tbox_group.$USER-$$ +# specify which queues will be request, use " " to separate more than 2 values +export queues="vm-2p8g~$USER vm-2p8g.aarch64" $CCI_SRC/providers/qemu.sh diff --git a/providers/qemu.sh b/providers/qemu.sh index d97fd85..7e4f074 100755 --- a/providers/qemu.sh +++ b/providers/qemu.sh @@ -9,6 +9,7 @@ load_cci_defaults : ${hostname:="vm-1p1g-1"} +: ${queues:="vm-1p1g.$(arch)"} # unicast prefix: x2, x6, xA, xE export mac=$(echo $hostname | md5sum | sed 's/^$..$$..$$..$$..$$..$.*$/0a-\1-\2-\3-\4-\5/') echo hostname: $hostname @@ -17,14 +18,23 @@ echo $mac > mac echo "arp -n | grep ${mac//-/:}" > ip.sh chmod +x ip.sh -curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/set_host_mac?hostname=${hostname}&mac=${mac}" +set_host_info() +{ + # use "," replace " " + queues=$(echo $queues | sed -r 's/ +/,/g') + curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/set_host_mac?hostname=${hostname}&mac=${mac}" + curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/set_host2queues?host=${hostname}&queues=${queues}" + +} +set_host_info -del_host_mac() +del_host_info() { curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/del_host_mac?mac=${mac}" > /dev/null 2>&1 + curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/del_host2queues?host=${hostname}" > /dev/null 2>&1 } -trap del_host_mac EXIT +trap del_host_info EXIT ( if [[ $hostname =~ ^(.*)-[0-9]+$ ]]; then @@ -33,7 +43,7 @@ trap del_host_mac EXIT tbox_group=$hostname fi - host=${tbox_group%%--*} + host=${tbox_group%.*} create_yaml_variables "$LKP_SRC/hosts/${host}" -- 2.23.0

3 4

[PATCH v5 compass-ci] LICENSES: add THIRD PARTY OPEN SOURCE SOFTWARE NOTICE
by Lu Kaiyi 03 Nov '20

03 Nov '20

add THIRD PARTY OPEN SOURCE SOFTWARE NOTICE for references. the directory structure of LICENSES as below: tree . ├── CCBY-4.0 ├── coreutils │ ├── coreutils │ ├── files │ └── GPL-3.0 -> ../GPL-3.0 ├── docker-sshd │ ├── docker-sshd │ ├── files │ └── MIT -> ../MIT ├── dracut │ ├── dracut │ ├── files │ └── GPL-2.0 -> ../GPL-2.0 ├── GPL-2.0 ├── GPL-3.0 ├── kemal │ ├── files │ ├── kemal │ └── MIT -> ../MIT ├── lkp-tests │ ├── files │ ├── GPL-2.0 -> ../GPL-2.0 │ └── lkp-tests ├── MIT ├── mritd.github.io │ ├── files │ ├── MIT -> ../MIT │ └── mritd.github.io ├── MulanPSL-2.0 └── Scout ├── files ├── Scout └── MIT -> ../MIT Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- LICENSES/Scout/MIT | 1 + LICENSES/Scout/Scout | 17 +++++++++++++++++ LICENSES/Scout/files | 1 + LICENSES/coreutils/GPL-3.0 | 1 + LICENSES/coreutils/coreutils | 17 +++++++++++++++++ LICENSES/coreutils/files | 1 + LICENSES/docker-sshd/MIT | 1 + LICENSES/docker-sshd/docker-sshd | 17 +++++++++++++++++ LICENSES/docker-sshd/files | 1 + LICENSES/dracut/GPL-2.0 | 1 + LICENSES/dracut/dracut | 18 ++++++++++++++++++ LICENSES/dracut/files | 1 + LICENSES/kemal/MIT | 1 + LICENSES/kemal/files | 1 + LICENSES/kemal/kemal | 17 +++++++++++++++++ LICENSES/lkp-tests/GPL-2.0 | 1 + LICENSES/lkp-tests/files | 8 ++++++++ LICENSES/lkp-tests/lkp-tests | 18 ++++++++++++++++++ LICENSES/mritd.github.io/MIT | 1 + LICENSES/mritd.github.io/files | 1 + LICENSES/mritd.github.io/mritd.github.io | 17 +++++++++++++++++ 21 files changed, 142 insertions(+) create mode 120000 LICENSES/Scout/MIT create mode 100644 LICENSES/Scout/Scout create mode 100644 LICENSES/Scout/files create mode 120000 LICENSES/coreutils/GPL-3.0 create mode 100644 LICENSES/coreutils/coreutils create mode 100644 LICENSES/coreutils/files create mode 120000 LICENSES/docker-sshd/MIT create mode 100644 LICENSES/docker-sshd/docker-sshd create mode 100644 LICENSES/docker-sshd/files create mode 120000 LICENSES/dracut/GPL-2.0 create mode 100644 LICENSES/dracut/dracut create mode 100644 LICENSES/dracut/files create mode 120000 LICENSES/kemal/MIT create mode 100644 LICENSES/kemal/files create mode 100644 LICENSES/kemal/kemal create mode 120000 LICENSES/lkp-tests/GPL-2.0 create mode 100644 LICENSES/lkp-tests/files create mode 100644 LICENSES/lkp-tests/lkp-tests create mode 120000 LICENSES/mritd.github.io/MIT create mode 100644 LICENSES/mritd.github.io/files create mode 100644 LICENSES/mritd.github.io/mritd.github.io diff --git a/LICENSES/Scout/MIT b/LICENSES/Scout/MIT new file mode 120000 index 0000000..91e6d98 --- /dev/null +++ b/LICENSES/Scout/MIT @@ -0,0 +1 @@ +../MIT \ No newline at end of file diff --git a/LICENSES/Scout/Scout b/LICENSES/Scout/Scout new file mode 100644 index 0000000..6844406 --- /dev/null +++ b/LICENSES/Scout/Scout @@ -0,0 +1,17 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: Scout db170ce4ff160c4a2c30483e9410df1773cf572c + +Copyright notice: Copyright(c) 2010 Menno van Slooten, http://mennovanslooten.nl/ +License: The MIT License +License-Text: please refer to file MIT diff --git a/LICENSES/Scout/files b/LICENSES/Scout/files new file mode 100644 index 0000000..0d1e2e9 --- /dev/null +++ b/LICENSES/Scout/files @@ -0,0 +1 @@ +lib/themes.rb diff --git a/LICENSES/coreutils/GPL-3.0 b/LICENSES/coreutils/GPL-3.0 new file mode 120000 index 0000000..481ee24 --- /dev/null +++ b/LICENSES/coreutils/GPL-3.0 @@ -0,0 +1 @@ +../GPL-3.0 \ No newline at end of file diff --git a/LICENSES/coreutils/coreutils b/LICENSES/coreutils/coreutils new file mode 100644 index 0000000..1250a6f --- /dev/null +++ b/LICENSES/coreutils/coreutils @@ -0,0 +1,17 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: coreutils 6.9.89 + +Copyright notice: Copyright(C) 2007 Free Software Foundation, Inc. <https://fsf.org/> +License: The GPL-3.0 License +License-Text: please refer to file GPL-3.0 diff --git a/LICENSES/coreutils/files b/LICENSES/coreutils/files new file mode 100644 index 0000000..2256719 --- /dev/null +++ b/LICENSES/coreutils/files @@ -0,0 +1 @@ +container/os-nfs/entrypoint.sh diff --git a/LICENSES/docker-sshd/MIT b/LICENSES/docker-sshd/MIT new file mode 120000 index 0000000..91e6d98 --- /dev/null +++ b/LICENSES/docker-sshd/MIT @@ -0,0 +1 @@ +../MIT \ No newline at end of file diff --git a/LICENSES/docker-sshd/docker-sshd b/LICENSES/docker-sshd/docker-sshd new file mode 100644 index 0000000..603f668 --- /dev/null +++ b/LICENSES/docker-sshd/docker-sshd @@ -0,0 +1,17 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: docker-sshd 729faee00bcf0eff5f80059cd92aa425505820f0 + +Copyright notice: Copyright(c) 2015-2020 Volt Grid Pty Ltd +License: The MIT License +License-Text: please refer to file MIT diff --git a/LICENSES/docker-sshd/files b/LICENSES/docker-sshd/files new file mode 100644 index 0000000..3289565 --- /dev/null +++ b/LICENSES/docker-sshd/files @@ -0,0 +1 @@ +container/ssh-r/entry.sh diff --git a/LICENSES/dracut/GPL-2.0 b/LICENSES/dracut/GPL-2.0 new file mode 120000 index 0000000..9824569 --- /dev/null +++ b/LICENSES/dracut/GPL-2.0 @@ -0,0 +1 @@ +../GPL-2.0 \ No newline at end of file diff --git a/LICENSES/dracut/dracut b/LICENSES/dracut/dracut new file mode 100644 index 0000000..f09df60 --- /dev/null +++ b/LICENSES/dracut/dracut @@ -0,0 +1,18 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: dracut RHEL-7.1 + +Copyright notice: Copyright(C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, +Fifth Floor, Boston, MA 02110-1301 USA. +License: The GPL-2.0 License +License-Text: please refer to file GPL-2.0 diff --git a/LICENSES/dracut/files b/LICENSES/dracut/files new file mode 100644 index 0000000..d629bf6 --- /dev/null +++ b/LICENSES/dracut/files @@ -0,0 +1 @@ +container/dracut-initrd/bin/cifs-lib.sh diff --git a/LICENSES/kemal/MIT b/LICENSES/kemal/MIT new file mode 120000 index 0000000..91e6d98 --- /dev/null +++ b/LICENSES/kemal/MIT @@ -0,0 +1 @@ +../MIT \ No newline at end of file diff --git a/LICENSES/kemal/files b/LICENSES/kemal/files new file mode 100644 index 0000000..de6eda6 --- /dev/null +++ b/LICENSES/kemal/files @@ -0,0 +1 @@ +src/spec/scheduler/boot_spec.cr diff --git a/LICENSES/kemal/kemal b/LICENSES/kemal/kemal new file mode 100644 index 0000000..e08255f --- /dev/null +++ b/LICENSES/kemal/kemal @@ -0,0 +1,17 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: kemal 0.17.0 + +Copyright notice: Copyright(c) 2016 Serdar Doğruyol +License: The MIT License +License-Text: please refer to file MIT diff --git a/LICENSES/lkp-tests/GPL-2.0 b/LICENSES/lkp-tests/GPL-2.0 new file mode 120000 index 0000000..9824569 --- /dev/null +++ b/LICENSES/lkp-tests/GPL-2.0 @@ -0,0 +1 @@ +../GPL-2.0 \ No newline at end of file diff --git a/LICENSES/lkp-tests/files b/LICENSES/lkp-tests/files new file mode 100644 index 0000000..df39f15 --- /dev/null +++ b/LICENSES/lkp-tests/files @@ -0,0 +1,8 @@ +sbin/compare +providers/multi-docker +container/open-scheduler/build-depends +user-client/jobs/iperf-pxe.yaml +user-client/jobs/iperf-vm.yaml +src/features/jobs/right_iperf.yaml +lib/matrix2.rb +sbin/es-find diff --git a/LICENSES/lkp-tests/lkp-tests b/LICENSES/lkp-tests/lkp-tests new file mode 100644 index 0000000..3f40d5f --- /dev/null +++ b/LICENSES/lkp-tests/lkp-tests @@ -0,0 +1,18 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: lkp-tests fcb47ccfc50c1f4da388aed9596c0acaac04a917 + +Copyright notice: Copyright(C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, +Fifth Floor, Boston, MA 02110-1301 USA +License: The GPL-2.0 License +License-Text: please refer to file GPL-2.0 diff --git a/LICENSES/mritd.github.io/MIT b/LICENSES/mritd.github.io/MIT new file mode 120000 index 0000000..91e6d98 --- /dev/null +++ b/LICENSES/mritd.github.io/MIT @@ -0,0 +1 @@ +../MIT \ No newline at end of file diff --git a/LICENSES/mritd.github.io/files b/LICENSES/mritd.github.io/files new file mode 100644 index 0000000..eeaa82a --- /dev/null +++ b/LICENSES/mritd.github.io/files @@ -0,0 +1 @@ +container/registry/config.yml diff --git a/LICENSES/mritd.github.io/mritd.github.io b/LICENSES/mritd.github.io/mritd.github.io new file mode 100644 index 0000000..ee22568 --- /dev/null +++ b/LICENSES/mritd.github.io/mritd.github.io @@ -0,0 +1,17 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: mritd.github.io ed4350e91e9b76078a850ede8302854d3393f493 + +Copyright notice: Copyright(c) 2013-2016 Blackrock Digital LLC. +License: The MIT License +License-Text: please refer to file MIT -- 2.23.0

1 0

[PATCH v8 compass-ci 1/2] kernel_version.md: explain key "kernel_version"
by Xu Xijian 03 Nov '20

03 Nov '20

[why] Explain the meaning of new key "kernel_version" for scheduler, including what's the mapping between its typical values and actual files in the disk and how to set it. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- doc/job/kernel_version.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 doc/job/kernel_version.md diff --git a/doc/job/kernel_version.md b/doc/job/kernel_version.md new file mode 100644 index 0000000..d76a3bd --- /dev/null +++ b/doc/job/kernel_version.md @@ -0,0 +1,18 @@ +# kernel_version + +Meaning: +- Every os has its kernel, however an os can start with different kernels according to different need. +- kernel_version is a key for users to specify a kernel version. +- If kernel_version is not given by users, it will use the default one. + +Related files: +- In initramfs boot process, every kernel version is related with a vmlinuz, module and headers. +- Files like below under $boot_dir, an example $boot_dir can be "/srv/os/openeuler/aarch64/20.03/boot". +├── headers-4.19.90-2003.cgz +├── headers.cgz -> headers-4.19.90-2003.cgz +├── modules-4.19.90-2003.cgz +├── modules.cgz -> modules-4.19.90-2003.cgz +├── vmlinuz-4.19.90-2003 + +Usage example: +- submit iperf.yaml testbox=vm-hi1620-2p8g--$USER os=openeuler os_arch=aarch64 os_version=20.03 runtime=20 kernel_version=4.19.90-2003 -- 2.23.0

2 2

[PATCH v4 compass-ci] LICENSES: add THIRD PARTY OPEN SOURCE SOFTWARE NOTICE
by Lu Kaiyi 03 Nov '20

03 Nov '20

add THIRD PARTY OPEN SOURCE SOFTWARE NOTICE for references. the directory structure of LICENSES as below: tree . ├── CCBY-4.0 ├── coreutils │ ├── coreutils │ ├── files │ └── GPL-3.0 -> ../GPL-3.0 ├── docker-sshd │ ├── docker-sshd │ ├── files │ └── MIT -> ../MIT ├── dracut │ ├── dracut │ ├── files │ └── GPL-2.0 -> ../GPL-2.0 ├── GPL-2.0 ├── GPL-3.0 ├── kemal │ ├── files │ ├── kemal │ └── MIT -> ../MIT ├── lkp-tests │ ├── files │ ├── GPL-2.0 -> ../GPL-2.0 │ └── lkp-tests ├── MIT ├── mritd.github.io │ ├── files │ ├── MIT -> ../MIT │ └── mritd.github.io ├── MulanPSL-2.0 └── Scout ├── files ├── Scout └── MIT -> ../MIT Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- LICENSES/Scout/MIT | 1 + LICENSES/Scout/Scout | 17 +++++++++++++++++ LICENSES/Scout/files | 1 + LICENSES/coreutils/GPL-3.0 | 1 + LICENSES/coreutils/coreutils | 17 +++++++++++++++++ LICENSES/coreutils/files | 1 + LICENSES/docker-sshd/MIT | 1 + LICENSES/docker-sshd/docker-sshd | 17 +++++++++++++++++ LICENSES/docker-sshd/files | 1 + LICENSES/dracut/GPL-2.0 | 1 + LICENSES/dracut/dracut | 18 ++++++++++++++++++ LICENSES/dracut/files | 1 + LICENSES/kemal/MIT | 1 + LICENSES/kemal/files | 1 + LICENSES/kemal/kemal | 17 +++++++++++++++++ LICENSES/lkp-tests/GPL-2.0 | 1 + LICENSES/lkp-tests/files | 8 ++++++++ LICENSES/lkp-tests/lkp-tests | 18 ++++++++++++++++++ LICENSES/mritd.github.io/MIT | 1 + LICENSES/mritd.github.io/files | 1 + LICENSES/mritd.github.io/mritd.github.io | 17 +++++++++++++++++ 21 files changed, 142 insertions(+) create mode 120000 LICENSES/Scout/MIT create mode 100644 LICENSES/Scout/Scout create mode 100644 LICENSES/Scout/files create mode 120000 LICENSES/coreutils/GPL-3.0 create mode 100644 LICENSES/coreutils/coreutils create mode 100644 LICENSES/coreutils/files create mode 120000 LICENSES/docker-sshd/MIT create mode 100644 LICENSES/docker-sshd/docker-sshd create mode 100644 LICENSES/docker-sshd/files create mode 120000 LICENSES/dracut/GPL-2.0 create mode 100644 LICENSES/dracut/dracut create mode 100644 LICENSES/dracut/files create mode 120000 LICENSES/kemal/MIT create mode 100644 LICENSES/kemal/files create mode 100644 LICENSES/kemal/kemal create mode 120000 LICENSES/lkp-tests/GPL-2.0 create mode 100644 LICENSES/lkp-tests/files create mode 100644 LICENSES/lkp-tests/lkp-tests create mode 120000 LICENSES/mritd.github.io/MIT create mode 100644 LICENSES/mritd.github.io/files create mode 100644 LICENSES/mritd.github.io/mritd.github.io diff --git a/LICENSES/Scout/MIT b/LICENSES/Scout/MIT new file mode 120000 index 0000000..91e6d98 --- /dev/null +++ b/LICENSES/Scout/MIT @@ -0,0 +1 @@ +../MIT \ No newline at end of file diff --git a/LICENSES/Scout/Scout b/LICENSES/Scout/Scout new file mode 100644 index 0000000..6844406 --- /dev/null +++ b/LICENSES/Scout/Scout @@ -0,0 +1,17 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: Scout db170ce4ff160c4a2c30483e9410df1773cf572c + +Copyright notice: Copyright(c) 2010 Menno van Slooten, http://mennovanslooten.nl/ +License: The MIT License +License-Text: please refer to file MIT diff --git a/LICENSES/Scout/files b/LICENSES/Scout/files new file mode 100644 index 0000000..e120df8 --- /dev/null +++ b/LICENSES/Scout/files @@ -0,0 +1 @@ +/compass-ci/lib/themes.rb diff --git a/LICENSES/coreutils/GPL-3.0 b/LICENSES/coreutils/GPL-3.0 new file mode 120000 index 0000000..481ee24 --- /dev/null +++ b/LICENSES/coreutils/GPL-3.0 @@ -0,0 +1 @@ +../GPL-3.0 \ No newline at end of file diff --git a/LICENSES/coreutils/coreutils b/LICENSES/coreutils/coreutils new file mode 100644 index 0000000..1250a6f --- /dev/null +++ b/LICENSES/coreutils/coreutils @@ -0,0 +1,17 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: coreutils 6.9.89 + +Copyright notice: Copyright(C) 2007 Free Software Foundation, Inc. <https://fsf.org/> +License: The GPL-3.0 License +License-Text: please refer to file GPL-3.0 diff --git a/LICENSES/coreutils/files b/LICENSES/coreutils/files new file mode 100644 index 0000000..8a2b690 --- /dev/null +++ b/LICENSES/coreutils/files @@ -0,0 +1 @@ +/compass-ci/container/os-nfs/entrypoint.sh diff --git a/LICENSES/docker-sshd/MIT b/LICENSES/docker-sshd/MIT new file mode 120000 index 0000000..91e6d98 --- /dev/null +++ b/LICENSES/docker-sshd/MIT @@ -0,0 +1 @@ +../MIT \ No newline at end of file diff --git a/LICENSES/docker-sshd/docker-sshd b/LICENSES/docker-sshd/docker-sshd new file mode 100644 index 0000000..603f668 --- /dev/null +++ b/LICENSES/docker-sshd/docker-sshd @@ -0,0 +1,17 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: docker-sshd 729faee00bcf0eff5f80059cd92aa425505820f0 + +Copyright notice: Copyright(c) 2015-2020 Volt Grid Pty Ltd +License: The MIT License +License-Text: please refer to file MIT diff --git a/LICENSES/docker-sshd/files b/LICENSES/docker-sshd/files new file mode 100644 index 0000000..018684c --- /dev/null +++ b/LICENSES/docker-sshd/files @@ -0,0 +1 @@ +/compass-ci/container/ssh-r/entry.sh diff --git a/LICENSES/dracut/GPL-2.0 b/LICENSES/dracut/GPL-2.0 new file mode 120000 index 0000000..9824569 --- /dev/null +++ b/LICENSES/dracut/GPL-2.0 @@ -0,0 +1 @@ +../GPL-2.0 \ No newline at end of file diff --git a/LICENSES/dracut/dracut b/LICENSES/dracut/dracut new file mode 100644 index 0000000..f09df60 --- /dev/null +++ b/LICENSES/dracut/dracut @@ -0,0 +1,18 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: dracut RHEL-7.1 + +Copyright notice: Copyright(C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, +Fifth Floor, Boston, MA 02110-1301 USA. +License: The GPL-2.0 License +License-Text: please refer to file GPL-2.0 diff --git a/LICENSES/dracut/files b/LICENSES/dracut/files new file mode 100644 index 0000000..78235de --- /dev/null +++ b/LICENSES/dracut/files @@ -0,0 +1 @@ +/compass-ci/container/dracut-initrd/bin/cifs-lib.sh diff --git a/LICENSES/kemal/MIT b/LICENSES/kemal/MIT new file mode 120000 index 0000000..91e6d98 --- /dev/null +++ b/LICENSES/kemal/MIT @@ -0,0 +1 @@ +../MIT \ No newline at end of file diff --git a/LICENSES/kemal/files b/LICENSES/kemal/files new file mode 100644 index 0000000..71a8cab --- /dev/null +++ b/LICENSES/kemal/files @@ -0,0 +1 @@ +/compass-ci/src/spec/scheduler/boot_spec.cr diff --git a/LICENSES/kemal/kemal b/LICENSES/kemal/kemal new file mode 100644 index 0000000..e08255f --- /dev/null +++ b/LICENSES/kemal/kemal @@ -0,0 +1,17 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: kemal 0.17.0 + +Copyright notice: Copyright(c) 2016 Serdar Doğruyol +License: The MIT License +License-Text: please refer to file MIT diff --git a/LICENSES/lkp-tests/GPL-2.0 b/LICENSES/lkp-tests/GPL-2.0 new file mode 120000 index 0000000..9824569 --- /dev/null +++ b/LICENSES/lkp-tests/GPL-2.0 @@ -0,0 +1 @@ +../GPL-2.0 \ No newline at end of file diff --git a/LICENSES/lkp-tests/files b/LICENSES/lkp-tests/files new file mode 100644 index 0000000..fde5c05 --- /dev/null +++ b/LICENSES/lkp-tests/files @@ -0,0 +1,8 @@ +/compass-ci/sbin/compare +/compass-ci/providers/multi-docker +/compass-ci/container/open-scheduler/build-depends +/compass-ci/user-client/jobs/iperf-pxe.yaml +/compass-ci/user-client/jobs/iperf-vm.yaml +/compass-ci/src/features/jobs/right_iperf.yaml +/compass-ci/lib/matrix2.rb +/compass-ci/sbin/es-find diff --git a/LICENSES/lkp-tests/lkp-tests b/LICENSES/lkp-tests/lkp-tests new file mode 100644 index 0000000..3f40d5f --- /dev/null +++ b/LICENSES/lkp-tests/lkp-tests @@ -0,0 +1,18 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: lkp-tests fcb47ccfc50c1f4da388aed9596c0acaac04a917 + +Copyright notice: Copyright(C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, +Fifth Floor, Boston, MA 02110-1301 USA +License: The GPL-2.0 License +License-Text: please refer to file GPL-2.0 diff --git a/LICENSES/mritd.github.io/MIT b/LICENSES/mritd.github.io/MIT new file mode 120000 index 0000000..91e6d98 --- /dev/null +++ b/LICENSES/mritd.github.io/MIT @@ -0,0 +1 @@ +../MIT \ No newline at end of file diff --git a/LICENSES/mritd.github.io/files b/LICENSES/mritd.github.io/files new file mode 100644 index 0000000..38b67c1 --- /dev/null +++ b/LICENSES/mritd.github.io/files @@ -0,0 +1 @@ +/compass-ci/container/registry/config.yml diff --git a/LICENSES/mritd.github.io/mritd.github.io b/LICENSES/mritd.github.io/mritd.github.io new file mode 100644 index 0000000..ee22568 --- /dev/null +++ b/LICENSES/mritd.github.io/mritd.github.io @@ -0,0 +1,17 @@ +THIRD PARTY OPEN SOURCE SOFTWARE NOTICE + +Please note we provide an open source software notice for the third party open source software along with +this software and/or this software component contributed by Huawei (in the following just "this SOFTWARE"). +The open source software licenses are granted by respective right holders. + +Warranty Disclaimer +THE OPEN SOURCE SOFTWARE IN THIS SOFTWARE IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, +BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS +FOR A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + +Copyright Notice and License Texts +Software: mritd.github.io ed4350e91e9b76078a850ede8302854d3393f493 + +Copyright notice: Copyright(c) 2013-2016 Blackrock Digital LLC. +License: The MIT License +License-Text: please refer to file MIT -- 2.23.0

2 1

[PATCH v2 lkp-tests 3/3] tests/iozone: modify the way of parsing parameter
by Lu Kaiyi 03 Nov '20

03 Nov '20

[why] iozone-bs.yaml has combined multiple parameter to single, so, iozone tool need change the way of parsing parameter. [how] modify the way of parsing parameter. Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- tests/iozone | 50 ++++++++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/tests/iozone b/tests/iozone index 88a92a18..19c246f1 100755 --- a/tests/iozone +++ b/tests/iozone @@ -1,42 +1,36 @@ #!/bin/sh # - block_size # - file_size -# - write_rewrite -# - read_reread -# - random_read_write -# - read_backwards -# - rewrite_record -# - stride_read -# - fwrite_refwrite -# - fread_refread -# - random_mix -# - pwrite_repwrite -# - pread_repread -# - pwritev_repwritev -# - preadv_repreadv +# - test ## IOzone is a filesystem benchmark tool. The benchmark generates ## and measures a variety of file operations. . $LKP_SRC/lib/reproduce-log.sh - args="iozone" if [ -n "$block_size" ]; then args+=" -r $block_size" - [ -n "$file_size" ] && args+=" -s $file_size" - [ -n "$write_rewrite" ] && args+=" -i 0" - [ -n "$read_reread" ] && args+=" -i 1" - [ -n "$random_read_write" ] && args+=" -i 2" - [ -n "$read_backwards" ] && args+=" -i 3" - [ -n "$rewrite_record" ] && args+=" -i 4" - [ -n "$stride_read" ] && args+=" -i 5" - [ -n "$fwrite_refwrite" ] && args+=" -i 6" - [ -n "$fread_refread" ] && args+=" -i 7" - [ -n "$random_mix" ] && args+=" -i 8" - [ -n "$pwrite_repwrite" ] && args+=" -i 9" - [ -n "$pread_repread" ] && args+=" -i 10" - [ -n "$pwritev_repwritev" ] && args+=" -i 11" - [ -n "$preadv_repreadv" ] && args+=" -i 12" + [ -n "$file_size" ] && args+=" -s $file_size" + OLD_IFS="$IFS" + IFS="," + array=($test) + IFS="$OLD_IFS" + for ele in ${array[@]} + do + [ "$ele" == "write_rewrite" ] && args+=" -i 0" + [ "$ele" == "read_reread" ] && args+=" -i 1" + [ "$ele" == "random_read_write" ] && args+=" -i 2" + [ "$ele" == "read_backwards" ] && args+=" -i 3" + [ "$ele" == "rewrite_record" ] && args+=" -i 4" + [ "$ele" == "stride_read" ] && args+=" -i 5" + [ "$ele" == "fwrite_refwrite" ] && args+=" -i 6" + [ "$ele" == "fread_refread" ] && args+=" -i 7" + [ "$ele" == "random_mix" ] && args+=" -i 8" + [ "$ele" == "pwrite_repwrite" ] && args+=" -i 9" + [ "$ele" == "pread_repread" ] && args+=" -i 10" + [ "$ele" == "pwritev_repwritev" ] && args+=" -i 11" + [ "$ele" == "preadv_repreadv" ] && args+=" -i 12" + done else args+=" -a" fi -- 2.23.0

2 1

[PATCH v2 lkp-tests 1/3] jobs/iozone-bs.yaml: fix mapper_parsing_exception
by Lu Kaiyi 03 Nov '20

03 Nov '20

[why] when key and value of fs: xfs in same line for iozone-bs.yaml, submit job will cause some error occurred like below: submit /home/lukaiyi/lkp-tests/jobs/iozone-bs.yaml failed, got job_id=z9.144590, error: [{"type" => "mapper_parsing_exception", "reason" => "failed to parse [pp.fs.fs]"}] [how] change key and value of fs: xfs to different line, same to iosched: kyber Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- jobs/iozone-bs.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/jobs/iozone-bs.yaml b/jobs/iozone-bs.yaml index 868979fc..e2cd9f48 100644 --- a/jobs/iozone-bs.yaml +++ b/jobs/iozone-bs.yaml @@ -14,6 +14,11 @@ block_size: - 16m disk: 1HDD -fs: xfs -iosched: kyber + +fs: +- xfs + +iosched: +- kyber + iozone: -- 2.23.0

2 1

[PATCH v8 compass-ci 2/2] job.cr: add a key "kernel_version" for initramfs
by Xu Xijian 03 Nov '20

03 Nov '20

When run job with initramfs, there is a default combination of kernel, modules and headers, and I add a key "kernel_version" to make it optional. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/job.cr | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 1e49da6..6309809 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -87,6 +87,10 @@ class Job kernel_append_root kernel_params docker_image + kernel_version + linux_vmlinuz_path + linux_modules_initrd + linux_headers_initrd ) macro method_missing(call) @@ -135,6 +139,7 @@ class Job set_result_service() set_os_mount() set_depends_initrd() + set_kernel_version() set_initrds_uri() set_kernel_uri() set_kernel_append_root() @@ -293,9 +298,17 @@ class Job return true end + private def set_kernel_version + boot_dir = "#{SRV_OS}/#{os_dir}/boot" + suffix = "-#{kernel_version}" if self["kernel_version"]? + self["linux_vmlinuz_path"] = File.real_path("#{boot_dir}/vmlinuz#{suffix}") + self["linux_modules_initrd"] = File.real_path("#{boot_dir}/modules#{suffix}.cgz") + self["linux_headers_initrd"] = File.real_path("#{boot_dir}/headers#{suffix}.cgz") + end + private def set_kernel_uri self["kernel_uri"] = "kernel #{OS_HTTP_PREFIX}" + - "#{JobHelper.service_path("#{SRV_OS}/#{os_dir}/vmlinuz")}" + "#{JobHelper.service_path("#{linux_vmlinuz_path}")}" end private def common_initrds @@ -316,6 +329,10 @@ class Job "#{JobHelper.service_path("#{osimage_dir}/current")}" temp_initrds << "#{INITRD_HTTP_PREFIX}" + "#{JobHelper.service_path("#{osimage_dir}/run-ipconfig.cgz")}" + temp_initrds << "#{OS_HTTP_PREFIX}" + + "#{JobHelper.service_path("#{linux_modules_initrd}")}" + temp_initrds << "#{OS_HTTP_PREFIX}" + + "#{JobHelper.service_path("#{linux_headers_initrd}")}" temp_initrds.concat(initrd_deps.split(/ /)) unless initrd_deps.empty? temp_initrds.concat(initrd_pkg.split(/ /)) unless initrd_pkg.empty? -- 2.23.0

1 0

[PATCH v7 compass-ci 1/2] job.cr: add a key "kernel_version" for initramfs
by Xu Xijian 03 Nov '20

03 Nov '20

When run job with initramfs, there is a default combination of kernel, modules and headers, and I add a key "kernel_version" to make it optional. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/job.cr | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0fd57e0..2fa1125 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -87,6 +87,10 @@ class Job kernel_append_root kernel_params docker_image + kernel_version + vmlinuz_path + modules_path + headers_path ) macro method_missing(call) @@ -135,6 +139,7 @@ class Job set_result_service() set_os_mount() set_depends_initrd() + set_kernel_version() set_initrds_uri() set_kernel_uri() set_kernel_append_root() @@ -293,9 +298,17 @@ class Job return true end + private def set_kernel_version + boot_dir = "#{SRV_OS}/#{os_dir}/boot" + suffix = "-#{kernel_version}" if self["kernel_version"]? + self["vmlinuz_path"] = File.real_path("#{boot_dir}/vmlinuz#{suffix}") + self["modules_path"] = File.real_path("#{boot_dir}/modules#{suffix}.cgz") + self["headers_path"] = File.real_path("#{boot_dir}/headers#{suffix}.cgz") + end + private def set_kernel_uri self["kernel_uri"] = "kernel #{OS_HTTP_PREFIX}" + - "#{JobHelper.service_path("#{SRV_OS}/#{os_dir}/vmlinuz")}" + "#{JobHelper.service_path("#{vmlinuz_path}")}" end private def common_initrds @@ -316,6 +329,10 @@ class Job "#{JobHelper.service_path("#{osimage_dir}/current")}" temp_initrds << "#{INITRD_HTTP_PREFIX}" + "#{JobHelper.service_path("#{osimage_dir}/run-ipconfig.cgz")}" + temp_initrds << "#{OS_HTTP_PREFIX}" + + "#{JobHelper.service_path("#{modules_path}")}" + temp_initrds << "#{OS_HTTP_PREFIX}" + + "#{JobHelper.service_path("#{headers_path}")}" temp_initrds.concat(initrd_deps.split(/ /)) unless initrd_deps.empty? temp_initrds.concat(initrd_pkg.split(/ /)) unless initrd_pkg.empty? -- 2.23.0

3 4