- Compass-ci - mailweb.openeuler.org

[PATCH lkp-tests] lib/job.rb: delete matrix args
by Wei Jihui 04 Dec '20

04 Dec '20

if user add matrix args by overrides args, like: submit xxx.yaml 'matrix_key=matrix_value' ... it should delete matrix args from overrides, or it will keep matrix args in job.yaml all the time. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/job.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/job.rb b/lib/job.rb index 2869dcb3..32329e10 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -302,6 +302,7 @@ class Job hash_job[key_array[i]] = v_array[i] end hash_job.delete(key) + @overrides.delete(key) unless @overrides[key].nil? jobs_array << hash_job end } -- 2.23.0

2 1

[PATCH lkp-tests] lib: delete the space on the left and right sides of the vertical bar
by Bai Jing 04 Dec '20

04 Dec '20

[why] The multi-args is like this "test|nr_threads", if user add it like this "test |nr_threads" or the value like "- shell1 | 1 ". So we need to delete the space Signed-off-by: Bai Jing <799286817(a)qq.com> --- lib/job.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/job.rb b/lib/job.rb index 2869dcb3..47fd56d3 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -290,11 +290,11 @@ class Job hash.each { |key, value| next unless key =~ /^\w.*\|.*\w$/ - key_array = key.split('|') + key_array = key.split('|').map!{ |ele| ele.strip } [value].flatten.each do |v| next unless v =~ /^\w.*\|.*\w$/ - v_array = v.split('|') + v_array = v.split('|').map!{ |v_ele| v_ele.strip } next unless key_array.size == v_array.size hash_job = hash.clone -- 2.23.0

2 1

[PATCH compass-ci 3/3] scheduler/Sched: use JSONLogger in Sched class
by Wu Zhende 04 Dec '20

04 Dec '20

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 3 +-- src/scheduler/find_job_boot.cr | 6 +++--- src/scheduler/submit_job.cr | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 22e0e4e..abb8a33 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -34,8 +34,7 @@ class Sched @block_helper = BlockHelper.new @rgc = RemoteGitClient.new @env = env - @log = JSONLogger.new - @log.set_env(env) + @log = env.get("log").as(JSONLogger) end def normalize_mac(mac : String) diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 6f51640..3e33604 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -192,15 +192,15 @@ class Sched begin job = @es.get_job(job_id.to_s) rescue ex - puts "Invalid job (id=#{job_id}) in es. Info: #{ex}" - puts ex.inspect_with_backtrace + @log.warn("Invalid job (id=#{job_id}) in es. Info: #{ex}") + @log.warn(ex.inspect_with_backtrace) end end if job job.update({"testbox" => testbox}) job.set_result_root - puts %({"job_id": "#{job_id}", "result_root": "/srv#{job.result_root}", "job_state": "set result root"}) + @log.info(%({"job_id": "#{job_id}", "result_root": "/srv#{job.result_root}", "job_state": "set result root"})) @redis.set_job(job) end return job diff --git a/src/scheduler/submit_job.cr b/src/scheduler/submit_job.cr index a9eb3a7..f93ad2b 100644 --- a/src/scheduler/submit_job.cr +++ b/src/scheduler/submit_job.cr @@ -19,7 +19,7 @@ class Sched return submit_cluster_job(job, cluster_config) end rescue ex - puts ex.inspect_with_backtrace + @log.warn(ex.inspect_with_backtrace) return [{ "job_id" => "0", "message" => ex.to_s, -- 2.23.0

1 0

[PATCH compass-ci 1/3] lib/json_logger.cr: support initialize env
by Wu Zhende 04 Dec '20

04 Dec '20

Can use JSONLogger.new(env: env) Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/json_logger.cr | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/json_logger.cr b/src/lib/json_logger.cr index 2b5ef6a..d136bdd 100644 --- a/src/lib/json_logger.cr +++ b/src/lib/json_logger.cr @@ -6,8 +6,7 @@ require "json" require "any_merge" class JSONLogger < Logger - def initialize(logdev = STDOUT, formatter = my_formatter) - @env = nil + def initialize(logdev = STDOUT, formatter = my_formatter, @env = nil) @env_info = Hash(String, String | Int32).new super(logdev, formatter: formatter) end -- 2.23.0

1 0

[PATCH compass-ci] answerback-email: add check account exists
by Luan Shengde 04 Dec '20

04 Dec '20

add check account exists for assigning/updating account avoid re-assign accounts for single email address [why] the tool may assign a new account for the email even if the email already has one. It will cause a big waste of account resource for wrong operation or test. [how] add check account exists for the email before assigning a new account. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/assign-account/answerback-email.rb | 38 ++++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index 53c321d..836ad86 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -61,21 +61,19 @@ conf_info = { 'is_update_account' => false } -def init_info(mail_content, email_info, my_info) +def init_info(mail_content, email_info, my_info, my_info_es) my_info['my_email'] = mail_content.from[0] + + read_my_login_name(my_info['my_email'], my_info_es) email_info['my_name'] = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') - return if mail_content.attachments.empty? - email_info['new_email_pubkey'] = mail_content.attachments[0].body.decoded.strip + email_info['new_email_pubkey'] = mail_content.attachments[0].body.decoded.strip || nil end def read_my_login_name(my_email, my_info_es) - my_account_info_str = %x(curl -XGET localhost:9200/accounts/_doc/#{my_email}) - my_account_info = YAML.safe_load my_account_info_str - message = "No such email found from the ES: #{my_email}" - raise message unless my_account_info['found'] + my_account_info = ESQuery.new(index: 'accounts').query_by_id(my_email) - my_info_es.update my_account_info['_source'] + my_info_es.update my_account_info || {} end options = OptionParser.new do |opts| @@ -102,6 +100,7 @@ options = OptionParser.new do |opts| end my_info['my_email'] = email_address + read_my_login_name(email_address, my_info_es) end opts.on('-n name', '--name name', 'appoint name') do |name| @@ -134,7 +133,7 @@ options = OptionParser.new do |opts| return false end mail_content = Mail.read(email_file) - init_info(mail_content, email_info, my_info) + init_info(mail_content, email_info, my_info, my_info_es) end opts.on('-g', '--gen-sshkey', 'generate jumper rsa public/private key and return pubkey') do @@ -142,7 +141,6 @@ options = OptionParser.new do |opts| end opts.on('-u', '--update', 'updata configurations') do - read_my_login_name(my_info['my_email'], my_info_es) conf_info['is_update_account'] = true end @@ -223,7 +221,7 @@ def build_my_info_from_account_info(my_info, account_info, conf_info) end def check_server - return true if ENV['HOSTNAME'] == 'z9' + return true if ENV['HOSTNAME'] == 'crystal.ci' message = 'please run the tool on z9 server' puts message @@ -240,9 +238,27 @@ def check_my_name_exist(my_info) return false end +def check_email_assigned_account(conf_info, my_info_es) + if conf_info['is_update_account'] + return true unless my_info_es.empty? + + message = "The email has not assigned an account yet.\n" + message += "Delete the '-u' option to assign a new account." + else + return true if my_info_es.empty? + + message = "The email has already assigned an account.\n" + message += "You can use option '-u' to update the account." + end + puts message + + return false +end + def send_account(my_info, conf_info, email_info, my_info_es, stdin_info) return unless check_server return unless check_my_email(my_info) + return unless check_email_assigned_account(conf_info, my_info_es) my_info['my_uuid'] = %x(uuidgen).chomp unless conf_info['is_update_account'] build_my_info_from_input(my_info, email_info, my_info_es, stdin_info) -- 2.23.0

1 0

[PATCH v4 compass-ci] mail-robot: add check account exists for email
by Luan Shengde 04 Dec '20

04 Dec '20

avoid re-assign accounts for single email address [why] the mail-robot may assign a new account if user send 'apply account' email with the same email address. and this is a big waste of account resource. [how] when receive request for applying an account, check whether the email already has already been assigned an account. assign a new one if there is none return the old one if there is already one and update info mail robot will update infos if the mail has been set new: - my_name - my_ssh_pubkey Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/apply-account.rb | 34 +++++++++++++++++------ 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/container/mail-robot/lib/apply-account.rb b/container/mail-robot/lib/apply-account.rb index bf40429..99da510 100755 --- a/container/mail-robot/lib/apply-account.rb +++ b/container/mail-robot/lib/apply-account.rb @@ -51,6 +51,8 @@ class ApplyAccount @send_mail_host = %x(/sbin/ip route | awk '/default/ {print $3}').chomp @send_mail_port = SEND_MAIL_PORT @mail_content = mail_content + @es_host = @send_mail_host + @es_port = ES_PORT @my_info = {} end @@ -61,9 +63,8 @@ class ApplyAccount # firstly get my_email before execute parse_mail_content is needed @my_info['my_email'] = @mail_content.from[0] parse_mail_content - acct_info = apply_my_account - - @my_info['my_login_name'] = acct_info['my_login_name'] + my_account_es = check_account_es + apply_my_account(my_account_es) store_account_info send_mail('') @@ -82,15 +83,30 @@ class ApplyAccount @my_info.update parsed_email_info end - def apply_my_account - my_uuid = %x(uuidgen).chomp - - @my_info['my_uuid'] = my_uuid + def apply_my_account(my_account_es) + apply_info = {} + if my_account_es + apply_info.update my_account_es + apply_info.update @my_info + @my_info.update apply_info + apply_info['is_update_account'] = true + else + my_uuid = %x(uuidgen).chomp + @my_info['my_uuid'] = my_uuid + apply_info.update @my_info + end + apply_new_account(apply_info, my_account_es) + end - apply_account = ApplyJumperAccount.new(@my_info) + def apply_new_account(apply_info, my_account_es) + apply_account = ApplyJumperAccount.new(apply_info) acct_info = apply_account.apply_jumper_account - return acct_info + @my_info['my_login_name'] = acct_info['my_login_name'] unless my_account_es + end + + def check_account_es + ESQuery.new(index: 'accounts').query_by_id(@my_info['my_email']) end def store_account_info -- 2.23.0

1 0

[PATCH v3 compass-ci] mail-robot: add check account exists for email
by Luan Shengde 04 Dec '20

04 Dec '20

avoid re-assigned accounts for single email address [why] the mail-robot may assign a new account if user send 'apply account' email with the same email address. and this is a big waste of account resource. [how] when receive request for applying an account, check whether the email already has already been assigned an account. assign a new one if there is none return the old one if there is already one and update info mail robot will update infos if the mail has been set new: - my_name - my_ssh_pubkey Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/apply-account.rb | 36 +++++++++++++++++------ 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/container/mail-robot/lib/apply-account.rb b/container/mail-robot/lib/apply-account.rb index bf40429..0b94ca5 100755 --- a/container/mail-robot/lib/apply-account.rb +++ b/container/mail-robot/lib/apply-account.rb @@ -51,6 +51,8 @@ class ApplyAccount @send_mail_host = %x(/sbin/ip route | awk '/default/ {print $3}').chomp @send_mail_port = SEND_MAIL_PORT @mail_content = mail_content + @es_host = @send_mail_host + @es_port = ES_PORT @my_info = {} end @@ -61,9 +63,8 @@ class ApplyAccount # firstly get my_email before execute parse_mail_content is needed @my_info['my_email'] = @mail_content.from[0] parse_mail_content - acct_info = apply_my_account - - @my_info['my_login_name'] = acct_info['my_login_name'] + my_account_es = check_account_es + apply_my_account(my_account_es) store_account_info send_mail('') @@ -82,15 +83,32 @@ class ApplyAccount @my_info.update parsed_email_info end - def apply_my_account - my_uuid = %x(uuidgen).chomp - - @my_info['my_uuid'] = my_uuid + def apply_my_account(my_account_es) + apply_info = {} + if my_account_es + apply_info.update my_account_es + apply_info.update @my_info + @my_info.update apply_info + apply_info['is_update_account'] = true + else + my_uuid = %x(uuidgen).chomp + @my_info['my_uuid'] = my_uuid + apply_info.update @my_info + end + apply_new_account(apply_info, my_account_es) + end - apply_account = ApplyJumperAccount.new(@my_info) + def apply_new_account(apply_info, my_account_es) + apply_account = ApplyJumperAccount.new(apply_info) acct_info = apply_account.apply_jumper_account - return acct_info + @my_info['my_login_name'] = acct_info['my_login_name'] unless my_account_es + end + + def check_account_es + ESQuery.new(index: 'accounts').query_by_id(@my_info['my_email']) + # my_account_info_str = %x(curl -XGET #{@es_host}:#{@es_port}/accounts/_doc/#{@my_info['my_email']}) + # YAML.safe_load my_account_info_str end def store_account_info -- 2.23.0

1 1

[PATCH v2 compass-ci] mail-robot: add check account exists for email
by Luan Shengde 04 Dec '20

04 Dec '20

avoid multi accounts assignment for single email address [why] the mail-robot may assign a new account if user send 'apply account' email with the same email address. and this is a big waste of account resource. [how] when get the request for applying account, check whether the email already has already been assigned an account. assign a new one if there is none return the old one if there is already one and update info mail robot will update infos if the mail has been set new: - my_name - my_ssh_pubkey Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/apply-account.rb | 36 +++++++++++++++++------ 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/container/mail-robot/lib/apply-account.rb b/container/mail-robot/lib/apply-account.rb index bf40429..d499840 100755 --- a/container/mail-robot/lib/apply-account.rb +++ b/container/mail-robot/lib/apply-account.rb @@ -12,6 +12,7 @@ require_relative 'apply-jumper-account' require_relative 'parse-apply-account-email' SEND_MAIL_PORT = ENV['SEND_MAIL_PORT'] || 49000 +ES_PORT = ENV['ES_PORT'] || 9200 # assign uuid/account for user # when mail-robot listened new email, and the email's subject @@ -51,6 +52,8 @@ class ApplyAccount @send_mail_host = %x(/sbin/ip route | awk '/default/ {print $3}').chomp @send_mail_port = SEND_MAIL_PORT @mail_content = mail_content + @es_host = @send_mail_host + @es_port = ES_PORT @my_info = {} end @@ -61,9 +64,8 @@ class ApplyAccount # firstly get my_email before execute parse_mail_content is needed @my_info['my_email'] = @mail_content.from[0] parse_mail_content - acct_info = apply_my_account - - @my_info['my_login_name'] = acct_info['my_login_name'] + my_account_es = check_account_es + apply_my_account(my_account_es) store_account_info send_mail('') @@ -82,15 +84,31 @@ class ApplyAccount @my_info.update parsed_email_info end - def apply_my_account - my_uuid = %x(uuidgen).chomp - - @my_info['my_uuid'] = my_uuid + def apply_my_account(my_account_es) + apply_info = {} + if my_account_es['found'] + apply_info.update my_account_es['_source'] + apply_info.update @my_info + @my_info.update apply_info + apply_info['is_update_account'] = true + else + my_uuid = %x(uuidgen).chomp + @my_info['my_uuid'] = my_uuid + apply_info.update @my_info + end + apply_new_account(apply_info, my_account_es) + end - apply_account = ApplyJumperAccount.new(@my_info) + def apply_new_account(apply_info, my_account_es) + apply_account = ApplyJumperAccount.new(apply_info) acct_info = apply_account.apply_jumper_account - return acct_info + @my_info['my_login_name'] = acct_info['my_login_name'] unless my_account_es['found'] + end + + def check_account_es + my_account_info_str = %x(curl -XGET #{@es_host}:#{@es_port}/accounts/_doc/#{@my_info['my_email']}) + YAML.safe_load my_account_info_str end def store_account_info -- 2.23.0

2 2

[PATCH v3 lkp-tests] stats/crystal/iozone.cr: keep the changes consistent with iozone
by Lu Kaiyi 04 Dec '20

04 Dec '20

[why] original iozone tool can match the line like: 64 4 1599680 3057153 4274062 4897948 3588436 2892445 2772930 3057153 3958892 3057153 3022727 3363612 4564786 but cannot match the line as below(there are extra whitespace after 2892445): 64 4 1599680 3057153 4274062 4897948 3588436 2892445 Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- stats/crystal/iozone.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stats/crystal/iozone.cr b/stats/crystal/iozone.cr index 7ea6e4b7..2a3ffbe2 100644 --- a/stats/crystal/iozone.cr +++ b/stats/crystal/iozone.cr @@ -8,7 +8,8 @@ per_record = Hash(String | String, Int32).new all_sum = 0 nr_records = 0 while (line = STDIN.gets) - next unless line =~ /^\s*\d+.*\d+$/ + line = line.strip + next unless line =~ /^\d+.*\d+$/ data = line.split data.each_index do |i| -- 2.23.0

1 0

[PATCH lkp-tests] Automatic compilation and configure the variables
by Hu Xuejiao 04 Dec '20

04 Dec '20

[why] Source shell_file based on different shells. [how] First, make and make install is written to the make-env. Second, invoking shell_profile(env.sh) that will distinguish different shell_files. Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- make-env | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 make-env diff --git a/make-env b/make-env new file mode 100644 index 00000000..cab39288 --- /dev/null +++ b/make-env @@ -0,0 +1,6 @@ +#!/bin/bash +. lib/env.sh +make +make install + +source $(shell_profile) -- 2.23.0

3 2

[PATCH compass-ci] git_mirror.rb: disable some unnecessary messages
by Li Yuanchao 04 Dec '20

04 Dec '20

Disable some unnecessary messages, so important logs will not be covered. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index edee331..9f254f5 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -235,7 +235,7 @@ class MirrorMain return if @git_info[git_repo]['is_submodule'] mirror_dir = "/srv/git/#{git_repo}.git" - show_ref_out = %x(git -C #{mirror_dir} show-ref --heads) + show_ref_out = %x(git -C #{mirror_dir} show-ref --heads 2>/dev/null) cur_refs = { heads: {} } show_ref_out.each_line do |line| next if line.start_with? '#' -- 2.23.0

1 0

[PATCH compass-ci] auto_submit.yaml: change testbox of deploy-cci
by Li Yuanchao 04 Dec '20

04 Dec '20

'vm-2p8g' is not enough for self test. It's suggested physical machine is better. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sbin/auto_submit.yaml b/sbin/auto_submit.yaml index e0f52ba..b53b647 100644 --- a/sbin/auto_submit.yaml +++ b/sbin/auto_submit.yaml @@ -1,8 +1,8 @@ a/AvxToNeon/AvxToNeon: - testbox=vm-2p8g os=centos os_version=7.6.1810 os_mount=initramfs os_arch=aarch64 api-avx2neon.yaml c/compass-ci/compass-ci: -- testbox=vm-2p8g os=openeuler os_version=20.03 os_mount=initramfs os_arch=aarch64 deploy-cci.yaml -- testbox=vm-2p8g os=centos os_version=7.6.1810 os_mount=initramfs os_arch=aarch64 deploy-cci.yaml -- testbox=vm-2p8g os=debian os_version=sid os_mount=initramfs os_arch=aarch64 deploy-cci.yaml +- testbox=taishan200-2280-2s64p-256g os=openeuler os_version=20.03 os_mount=initramfs os_arch=aarch64 deploy-cci.yaml +- testbox=taishan200-2280-2s64p-256g os=centos os_version=7.6.1810 os_mount=initramfs os_arch=aarch64 deploy-cci.yaml +- testbox=taishan200-2280-2s64p-256g os=debian os_version=sid os_mount=initramfs os_arch=aarch64 deploy-cci.yaml archlinux: - testbox=vm-2p16g os=openeuler os_version=20.03 os_mount=initramfs os_arch=aarch64 build-pkg.yaml -- 2.23.0

1 0

[PATCH lkp-tests] stats/crystal/iozone.cr: keep the changes consistent with iozone
by Lu Kaiyi 04 Dec '20

04 Dec '20

Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- stats/crystal/iozone.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stats/crystal/iozone.cr b/stats/crystal/iozone.cr index 7ea6e4b7..2a3ffbe2 100644 --- a/stats/crystal/iozone.cr +++ b/stats/crystal/iozone.cr @@ -8,7 +8,8 @@ per_record = Hash(String | String, Int32).new all_sum = 0 nr_records = 0 while (line = STDIN.gets) - next unless line =~ /^\s*\d+.*\d+$/ + line = line.strip + next unless line =~ /^\d+.*\d+$/ data = line.split data.each_index do |i| -- 2.23.0

2 1

[PATCH compass-ci] sparrow: add mail-robot work dir, ENV, permission
by Liu Yinsi 04 Dec '20

04 Dec '20

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/0-package/common | 1 + sparrow/1-storage/permission | 2 ++ sparrow/1-storage/tiny | 3 +++ sparrow/3-code/dev-env | 1 + 4 files changed, 7 insertions(+) diff --git a/sparrow/0-package/common b/sparrow/0-package/common index 74905f1..b381d73 100755 --- a/sparrow/0-package/common +++ b/sparrow/0-package/common @@ -5,6 +5,7 @@ gem install git activesupport rest-client cucumber json faye-websocket elasticsearch grep -q '^lkp:' /etc/passwd || useradd -u 1090 lkp +grep -q '^mailer:' /etc/passwd || useradd -u 1144 mailer grep -q '^team:' /etc/group || groupadd team grep -q '^committer:' /etc/group || groupadd -g 1999 committer diff --git a/sparrow/1-storage/permission b/sparrow/1-storage/permission index f5c1646..d1367cf 100755 --- a/sparrow/1-storage/permission +++ b/sparrow/1-storage/permission @@ -3,6 +3,7 @@ chown lkp:lkp /srv/result chown lkp:lkp /srv/initrd chown -R lkp:lkp /srv/es +chown -R mailer:team /srv/cci/Maildir chown lkp:lkp /srv/cache/netdata_cache chown lkp:lkp /srv/cache/netdata_lib chown lkp:lkp /srv/cci/serial/fluentd-pos @@ -15,6 +16,7 @@ chmod 775 /srv/cache/netdata_cache chmod 775 /srv/cache/netdata_lib chmod 775 /srv/cci/serial/logs chmod 775 /srv/cci/serial/fluentd-pos +chmod -R 775 /srv/cci/Maildir chgrp team /srv/dc chmod g+ws /srv/dc diff --git a/sparrow/1-storage/tiny b/sparrow/1-storage/tiny index 21c2720..e589937 100755 --- a/sparrow/1-storage/tiny +++ b/sparrow/1-storage/tiny @@ -20,6 +20,9 @@ dirs=( /tftpboot /c /etc/qemu + /srv/cci/Maildir/.compass-ci/new + /srv/cci/Maildir/.compass-ci/cur + /srv/cci/Maildir/.compass-ci/sent ) mkdir -p "${dirs[@]}" diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 34cbe17..fb16000 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -38,6 +38,7 @@ LOGGING_ES_PORT: $LOGGING_ES_PORT SEND_MAIL_PORT: $SEND_MAIL_PORT LOCAL_SEND_MAIL_PORT: $LOCAL_SEND_MAIL_PORT LOCAL_ROBOT_EMAIL_ADDRESS: $LOCAL_ROBOT_EMAIL_ADDRESS +MAILDIR: /srv/cci/Maildir/.compass-ci EOF cat > /etc/profile.d/compass.sh <<'EOF' -- 2.23.0

1 0

[PATCH v5 lkp-tests] spec/submit_spec.rb: update submit_job
by Hu Xuejiao 04 Dec '20

04 Dec '20

[why] Originally submit_job is too complicated, it needs to be simplified. Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- .../job_on_fail-3000.yaml => job-3000.yaml} | 0 .../{job_on_fail.yaml => job.yaml} | 1 + ...68000000-1-65535-set_get-never-never.yaml} | 0 .../redis.yaml => link_jobs1/job.yaml} | 0 .../job-1000-2-100000ss-add.yaml} | 0 .../job-10000-2-100000ss-add.yaml} | 0 .../job-20-2-6000ss-cache.yaml} | 0 .../ftq.yaml => link_jobs2/job.yaml} | 0 ...e.yaml => job-1-300-whetstone-double.yaml} | 0 ...-300-shell1.yaml => job-3-300-shell1.yaml} | 0 .../matrix/{unixbench-spec.yaml => job.yaml} | 0 ..._files-defaults.yaml => job-defaults.yaml} | 0 spec/submit/merge_yaml/job.yaml | 1 + .../merge_yaml/merge_included_files.yaml | 1 - ...-add.yaml => job-1000-2-100000ss-add.yaml} | 0 ...add.yaml => job-10000-2-100000ss-add.yaml} | 0 ...-cache.yaml => job-20-2-6000ss-cache.yaml} | 0 .../separate_yaml/{ftq.yaml => job.yaml} | 0 spec/submit_spec.rb | 30 +++++-------------- 19 files changed, 9 insertions(+), 24 deletions(-) rename spec/submit/job_on_fail/{job_on_fail/job_on_fail-3000.yaml => job-3000.yaml} (100%) rename spec/submit/job_on_fail/{job_on_fail.yaml => job.yaml} (97%) rename spec/submit/{link_jobs/redis/redis-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml => link_jobs1/job-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml} (100%) rename spec/submit/{link_jobs/redis.yaml => link_jobs1/job.yaml} (100%) rename spec/submit/{link_jobs/ftq/ftq-1000-2-100000ss-add.yaml => link_jobs2/job-1000-2-100000ss-add.yaml} (100%) rename spec/submit/{link_jobs/ftq/ftq-10000-2-100000ss-add.yaml => link_jobs2/job-10000-2-100000ss-add.yaml} (100%) rename spec/submit/{link_jobs/ftq/ftq-20-2-6000ss-cache.yaml => link_jobs2/job-20-2-6000ss-cache.yaml} (100%) rename spec/submit/{link_jobs/ftq.yaml => link_jobs2/job.yaml} (100%) rename spec/submit/matrix/{unixbench-spec/unixbench-spec-1-300-whetstone-double.yaml => job-1-300-whetstone-double.yaml} (100%) rename spec/submit/matrix/{unixbench-spec/unixbench-spec-3-300-shell1.yaml => job-3-300-shell1.yaml} (100%) rename spec/submit/matrix/{unixbench-spec.yaml => job.yaml} (100%) rename spec/submit/merge_yaml/{merge_included_files/merge_included_files-defaults.yaml => job-defaults.yaml} (100%) create mode 100644 spec/submit/merge_yaml/job.yaml delete mode 100644 spec/submit/merge_yaml/merge_included_files.yaml rename spec/submit/separate_yaml/{ftq/ftq-1000-2-100000ss-add.yaml => job-1000-2-100000ss-add.yaml} (100%) rename spec/submit/separate_yaml/{ftq/ftq-10000-2-100000ss-add.yaml => job-10000-2-100000ss-add.yaml} (100%) rename spec/submit/separate_yaml/{ftq/ftq-20-2-6000ss-cache.yaml => job-20-2-6000ss-cache.yaml} (100%) rename spec/submit/separate_yaml/{ftq.yaml => job.yaml} (100%) diff --git a/spec/submit/job_on_fail/job_on_fail/job_on_fail-3000.yaml b/spec/submit/job_on_fail/job-3000.yaml similarity index 100% rename from spec/submit/job_on_fail/job_on_fail/job_on_fail-3000.yaml rename to spec/submit/job_on_fail/job-3000.yaml diff --git a/spec/submit/job_on_fail/job_on_fail.yaml b/spec/submit/job_on_fail/job.yaml similarity index 97% rename from spec/submit/job_on_fail/job_on_fail.yaml rename to spec/submit/job_on_fail/job.yaml index bf68ff40..5aaedf30 100644 --- a/spec/submit/job_on_fail/job_on_fail.yaml +++ b/spec/submit/job_on_fail/job.yaml @@ -2,4 +2,5 @@ suite: atomic on_fail: sleep: 3000 + atomic: diff --git a/spec/submit/link_jobs/redis/redis-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml b/spec/submit/link_jobs1/job-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml similarity index 100% rename from spec/submit/link_jobs/redis/redis-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml rename to spec/submit/link_jobs1/job-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml diff --git a/spec/submit/link_jobs/redis.yaml b/spec/submit/link_jobs1/job.yaml similarity index 100% rename from spec/submit/link_jobs/redis.yaml rename to spec/submit/link_jobs1/job.yaml diff --git a/spec/submit/link_jobs/ftq/ftq-1000-2-100000ss-add.yaml b/spec/submit/link_jobs2/job-1000-2-100000ss-add.yaml similarity index 100% rename from spec/submit/link_jobs/ftq/ftq-1000-2-100000ss-add.yaml rename to spec/submit/link_jobs2/job-1000-2-100000ss-add.yaml diff --git a/spec/submit/link_jobs/ftq/ftq-10000-2-100000ss-add.yaml b/spec/submit/link_jobs2/job-10000-2-100000ss-add.yaml similarity index 100% rename from spec/submit/link_jobs/ftq/ftq-10000-2-100000ss-add.yaml rename to spec/submit/link_jobs2/job-10000-2-100000ss-add.yaml diff --git a/spec/submit/link_jobs/ftq/ftq-20-2-6000ss-cache.yaml b/spec/submit/link_jobs2/job-20-2-6000ss-cache.yaml similarity index 100% rename from spec/submit/link_jobs/ftq/ftq-20-2-6000ss-cache.yaml rename to spec/submit/link_jobs2/job-20-2-6000ss-cache.yaml diff --git a/spec/submit/link_jobs/ftq.yaml b/spec/submit/link_jobs2/job.yaml similarity index 100% rename from spec/submit/link_jobs/ftq.yaml rename to spec/submit/link_jobs2/job.yaml diff --git a/spec/submit/matrix/unixbench-spec/unixbench-spec-1-300-whetstone-double.yaml b/spec/submit/matrix/job-1-300-whetstone-double.yaml similarity index 100% rename from spec/submit/matrix/unixbench-spec/unixbench-spec-1-300-whetstone-double.yaml rename to spec/submit/matrix/job-1-300-whetstone-double.yaml diff --git a/spec/submit/matrix/unixbench-spec/unixbench-spec-3-300-shell1.yaml b/spec/submit/matrix/job-3-300-shell1.yaml similarity index 100% rename from spec/submit/matrix/unixbench-spec/unixbench-spec-3-300-shell1.yaml rename to spec/submit/matrix/job-3-300-shell1.yaml diff --git a/spec/submit/matrix/unixbench-spec.yaml b/spec/submit/matrix/job.yaml similarity index 100% rename from spec/submit/matrix/unixbench-spec.yaml rename to spec/submit/matrix/job.yaml diff --git a/spec/submit/merge_yaml/merge_included_files/merge_included_files-defaults.yaml b/spec/submit/merge_yaml/job-defaults.yaml similarity index 100% rename from spec/submit/merge_yaml/merge_included_files/merge_included_files-defaults.yaml rename to spec/submit/merge_yaml/job-defaults.yaml diff --git a/spec/submit/merge_yaml/job.yaml b/spec/submit/merge_yaml/job.yaml new file mode 100644 index 00000000..6d64c381 --- /dev/null +++ b/spec/submit/merge_yaml/job.yaml @@ -0,0 +1 @@ +<< : template_yaml diff --git a/spec/submit/merge_yaml/merge_included_files.yaml b/spec/submit/merge_yaml/merge_included_files.yaml deleted file mode 100644 index 8d459a32..00000000 --- a/spec/submit/merge_yaml/merge_included_files.yaml +++ /dev/null @@ -1 +0,0 @@ -<<: template_yaml diff --git a/spec/submit/separate_yaml/ftq/ftq-1000-2-100000ss-add.yaml b/spec/submit/separate_yaml/job-1000-2-100000ss-add.yaml similarity index 100% rename from spec/submit/separate_yaml/ftq/ftq-1000-2-100000ss-add.yaml rename to spec/submit/separate_yaml/job-1000-2-100000ss-add.yaml diff --git a/spec/submit/separate_yaml/ftq/ftq-10000-2-100000ss-add.yaml b/spec/submit/separate_yaml/job-10000-2-100000ss-add.yaml similarity index 100% rename from spec/submit/separate_yaml/ftq/ftq-10000-2-100000ss-add.yaml rename to spec/submit/separate_yaml/job-10000-2-100000ss-add.yaml diff --git a/spec/submit/separate_yaml/ftq/ftq-20-2-6000ss-cache.yaml b/spec/submit/separate_yaml/job-20-2-6000ss-cache.yaml similarity index 100% rename from spec/submit/separate_yaml/ftq/ftq-20-2-6000ss-cache.yaml rename to spec/submit/separate_yaml/job-20-2-6000ss-cache.yaml diff --git a/spec/submit/separate_yaml/ftq.yaml b/spec/submit/separate_yaml/job.yaml similarity index 100% rename from spec/submit/separate_yaml/ftq.yaml rename to spec/submit/separate_yaml/job.yaml diff --git a/spec/submit_spec.rb b/spec/submit_spec.rb index 4096cb73..c8cdcc15 100644 --- a/spec/submit_spec.rb +++ b/spec/submit_spec.rb @@ -16,20 +16,18 @@ end def traverse_file(output_dir) Dir.glob("#{output_dir}/*.yaml").each do |yaml_file| - stable_yaml_file(yaml_file) + stable_yaml_file(yaml_file) unless File.basename(yaml_file) == 'job.yaml' end end -def submit_job(submit_item) - submit_yaml_path = File.join("#{LKP_SRC}/spec/submit", submit_item) - Dir.glob("#{submit_yaml_path}/*.yaml").each do |yaml_file| - output_dir = File.join(submit_yaml_path, File.basename(yaml_file, '.yaml')) +def submit_job() + Dir.glob("#{LKP_SRC}/spec/submit/*").each do |output_dir| submit_cmd = [ "#{LKP_SRC}/sbin/submit", '-o', output_dir, '-s', 'lab: spec_lab', '-s', 'testbox: vm-2p8g--spec_submit', - yaml_file + "#{output_dir}/job.yaml" ] system(*submit_cmd) traverse_file(output_dir) @@ -38,22 +36,8 @@ end describe 'submit job spec' do it 'link jobs spec' do - submit_job('link_jobs') - end - - it 'link matrix' do - submit_job('matrix') - end - - it 'separate yaml spec' do - submit_job('separate_yaml') - end - - it 'job on fail' do - submit_job('job_on_fail') - end - - it 'merge yaml' do - submit_job('merge_yaml') + submit_job() end end + +submit_job -- 2.23.0

1 0

[PATCH compass-ci 2/2] lib/mail_bisect_result.rb: parse and send error_messages
by Cao Xueliang 04 Dec '20

04 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/mail_bisect_result.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/mail_bisect_result.rb b/lib/mail_bisect_result.rb index 2799ce2..5a208c4 100644 --- a/lib/mail_bisect_result.rb +++ b/lib/mail_bisect_result.rb @@ -7,7 +7,7 @@ require 'json' # compose and send email for bisect result class MailBisectResult def initialize(bisect_info) - @error_id = bisect_info['error_id'] + @error_messages = bisect_info['error_messages'] @repo = bisect_info['repo'] @commit_id = bisect_info['commit'] @git_commit = GitCommit.new(@repo, @commit_id) @@ -32,7 +32,8 @@ class MailBisectResult After submitting a job we noticed an error response due to the commit: commit: #{@commit_id[0..11]} ("#{(a)git_commit.subject}") - error_id: #{@error_id} + error_messages: + #{@error_messages} https://gitee.com/openeuler/compass-ci BODY -- 2.23.0

1 0

[PATCH compass-ci 1/2] lib/git_bisect.rb: replace error_id with error_messages
by Cao Xueliang 04 Dec '20

04 Dec '20

The error_messages will display the original error messages. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/git_bisect.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/git_bisect.rb b/lib/git_bisect.rb index 9033db2..e2c5636 100644 --- a/lib/git_bisect.rb +++ b/lib/git_bisect.rb @@ -4,6 +4,7 @@ require 'json' require_relative 'es_query' +require_relative 'error_messages' require_relative 'sched_client' require_relative '../src/delimiter/utils' require_relative "#{ENV['LKP_SRC']}/lib/monitor" @@ -19,6 +20,7 @@ class GitBisect # set object property set_ids set_bad_job + set_build_pkg_dir set_upstream set_work_dir set_good_commit @@ -38,6 +40,10 @@ class GitBisect @bad_job = Utils.init_job_content(@bad_job_id) end + def set_build_pkg_dir + @build_pkg_dir = File.join('/srv', @bad_job['result_root'], 'build-pkg') + end + def set_upstream @upstream_repo = @bad_job['upstream_repo'] @upstream_commit = @bad_job['upstream_commit'] @@ -80,9 +86,10 @@ class GitBisect return nil unless temp[0].include?('is the first bad commit') || temp[-1].include?('bisect run success') first_bad_commit = Utils.parse_first_bad_commit(result) + error_messages = ErrorMessages.new(@build_pkg_dir).obtain_error_messages_by_errorid(@error_id) return Hash['repo' => @upstream_repo, 'commit' => first_bad_commit, - 'job_id' => @bad_job_id, 'error_id' => @error_id] + 'job_id' => @bad_job_id, 'error_messages' => error_messages.join("\n")] end # first search the good commit in db -- 2.23.0

1 0

[PATCH v2 lkp-tests 2/2] stats/crystal/iozone.cr: keep the changes consistent with iozone
by Lu Kaiyi 04 Dec '20

04 Dec '20

Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- stats/crystal/iozone.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stats/crystal/iozone.cr b/stats/crystal/iozone.cr index 7ea6e4b7..2a3ffbe2 100644 --- a/stats/crystal/iozone.cr +++ b/stats/crystal/iozone.cr @@ -8,7 +8,8 @@ per_record = Hash(String | String, Int32).new all_sum = 0 nr_records = 0 while (line = STDIN.gets) - next unless line =~ /^\s*\d+.*\d+$/ + line = line.strip + next unless line =~ /^\d+.*\d+$/ data = line.split data.each_index do |i| -- 2.23.0

1 0

[PATCH v2 lkp-tests 1/2] stats/crystal: sysbench-threads-git to sysbench-threads-git.cr
by Lu Kaiyi 04 Dec '20

04 Dec '20

Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- stats/crystal/sysbench-threads-git.cr | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 stats/crystal/sysbench-threads-git.cr diff --git a/stats/crystal/sysbench-threads-git.cr b/stats/crystal/sysbench-threads-git.cr new file mode 100644 index 00000000..91707e87 --- /dev/null +++ b/stats/crystal/sysbench-threads-git.cr @@ -0,0 +1,14 @@ +#!/usr/bin/env crystal + +# General statistics: +# total time: 0.2671s +# total number of events: 10000 +# total time taken by event execution: 0.3971s +# response time: +# min: 0.04ms + +while (line = STDIN.gets) + next unless line =~ /^\s+total\stime:\s+(\S+)/ + + puts "total time: #{$1}" +end -- 2.23.0

1 0

[PATCH v2 compass-ci] How to add a machine to compass-ci
by Zhang Dewan 04 Dec '20

04 Dec '20

Signed-off-by: Zhang Dewan <513619464(a)qq.com> How a new machine add to compass-ci Signed-off-by: Zhang Dewan <513619464(a)qq.com> --- ...00\350\210\254\350\277\207\347\250\213.md" | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 "doc/manual/\346\267\273\345\212\240\346\234\272\345\231\250\345\210\260compass-ci\347\232\204\344\270\200\350\210\254\350\277\207\347\250\213.md" diff --git "a/doc/manual/\346\267\273\345\212\240\346\234\272\345\231\250\345\210\260compass-ci\347\232\204\344\270\200\350\210\254\350\277\207\347\250\213.md" "b/doc/manual/\346\267\273\345\212\240\346\234\272\345\231\250\345\210\260compass-ci\347\232\204\344\270\200\350\210\254\350\277\207\347\250\213.md" new file mode 100644 index 0000000..340ffea --- /dev/null +++ "b/doc/manual/\346\267\273\345\212\240\346\234\272\345\231\250\345\210\260compass-ci\347\232\204\344\270\200\350\210\254\350\277\207\347\250\213.md" @@ -0,0 +1,54 @@ + +这个过程可以使用“注册--发现”的模型来理解。 + +## 一台机器成为测试机: +- 需要注册到compass-ci +- 需要能够执行调度器（scheduler）给它的任务 + +## 注册 +### 1. 注册信息保存在哪 +测试机需要注册到z9上面的redis-db. + +x.x.x.x:xxxx> `hgetall "sched/mac2host"` # 登录redis, 这个命令可以查看已经注册的测试机信息 + +### 2. 怎么注册 +使用scheduler服务接口注册，模板： +`curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/set_host_mac?hostname=${hostname}&mac=${mac}"` +`curl -X PUT "http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/set_host2queues?host=${hostname}&queues=${api_queues}"` + +可以看到，这两个调用需要三个参数：mac、hostname、api_queues + +#### 2.1 接口参数说明 +##### 2.1.1 mac +当前我们的机器都是局域网内的资源，所以没有比使用mac注册更直接、方便的了。 + +###### 2.1.1.1 如何获取mac地址，这里用TaiShan服务器举例： +- 登录bmc，打开信息--系统信息--网络--网卡--第一张网卡--port1，即可查看。 +- 登录bios，进入Advanced--Network Device List，即可查看。 +- 使用一台和机器的bmc同网段的机器，执行以下命令即可查看： +`ipmitool -I lanplus -H $cur_ip -U $iBMC_user -P $iBMC_passwd raw 0x30 0x90 0x01 0x03 0x0${nic_idx} 0x00|awk '{print $5,$6,$7,$8,$9,$10}' |sed 's/\s/\-/g'` +- 给服务器安装os，登录上去，执行命令，即可查看： +`ip link | grep "^2:" -A 1| grep link |awk '{print $2}` + +##### 2.1.2 hostname +如同每一张网卡的mac是规则且唯一的，hostname作为资源的标识也是如此。 + +###### 2.1.2.1 如何定义hostname，这里还是以Taishan服务器为例： +` taishan200-2280-${cpuNum}s${coreNum}p-${memSize:-256}g--${aN}` +> taishan200-2280 服务器型号 +> cpuNum cpu个数 +> coreNum cpu核数 +> memSize 内存大小 +> aN 该类型机器编号 + +#### 2.1.3 api_queues +作为单台机器的组标识，一般取hostname的前缀。 +比如，taishan200-2280-2s64p-256g--a100, 它的api_queues为：taishan200-2280-2s64p-256g + +## 发现 +### 1. 怎么发现测试机并给它分配任务 +测试机器需要将启动方式设置pxe启动，然后在启动环节向scheduler发起请求，获取启动参数和任务信息。 + +### 2. 设置PXE启动 +- 华为服务器默认pxe启动，无需配置。 +- 其他设备，根据产品类型，按文档操作即可。 -- 2.23.0

2 1

[PATCH lkp-tests] stats/crystal: sysbench-threads-git to sysbench-threads-git.cr
by Lu Kaiyi 04 Dec '20

04 Dec '20

Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- stats/crystal/sysbench-threads-git.cr | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 stats/crystal/sysbench-threads-git.cr diff --git a/stats/crystal/sysbench-threads-git.cr b/stats/crystal/sysbench-threads-git.cr new file mode 100644 index 00000000..5d9dcc10 --- /dev/null +++ b/stats/crystal/sysbench-threads-git.cr @@ -0,0 +1,14 @@ +#!/usr/bin/env crystal + +# General statistics: +# total time: 0.2671s +# total number of events: 10000 +# total time taken by event execution: 0.3971s +# response time: +# min: 0.04ms + +while (line = STDIN.gets) + next unless line =~ /^\s+total time:\s+([0-9.]+)/ + + puts "total time: #{$1.to_f}s" +end -- 2.23.0

2 1

[PATCH v2 compass-ci] user-client/maintain: add '|' for multi args
by Liu Yinsi 04 Dec '20

04 Dec '20

multi args changed from os os_arch os_version to os|os_arch|os_version, and if use os|os_arch|os_version, job yaml must as the first parameter. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- .../maintain/walk-os-test/walk-os-iperf-test | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/user-client/maintain/walk-os-test/walk-os-iperf-test b/user-client/maintain/walk-os-test/walk-os-iperf-test index ed97999..9f75516 100755 --- a/user-client/maintain/walk-os-test/walk-os-iperf-test +++ b/user-client/maintain/walk-os-test/walk-os-iperf-test @@ -101,8 +101,8 @@ def run_qemu end end -def submit_job(os_str, os_mount) - `submit --my-queue 'os os_arch os_version=#{os_str}' 'testbox=#{TESTBOX}' 'os_mount=#{os_mount}' #{TEST_YAML}` +def submit_job(os_args, os_mount) + `submit #{TEST_YAML} --my-queue 'os|os_arch|os_version=#{os_args}' testbox=#{TESTBOX} os_mount=#{os_mount}` end def traversal_os_mount @@ -113,12 +113,13 @@ end def traversal_test_os(os_mount) TEST_OS.each do |os_str| - test_rootfs(os_str, os_mount) + os_args = os_str.gsub(' ', '|') + test_rootfs(os_args, os_mount) end end -def test_rootfs(os_str, os_mount) - message = submit_job(os_str, os_mount).chomp +def test_rootfs(os_args, os_mount) + message = submit_job(os_args, os_mount).chomp job_id = message.split('=')[1] run_qemu @@ -128,7 +129,7 @@ def test_rootfs(os_str, os_mount) find_by_id(job_id) end - report_all = job_id.split(', ') + os_str.split + os_mount.split + job_result + report_all = job_id.split(', ') + os_args.split + os_mount.split + job_result write_report(report_all) end -- 2.23.0

1 0

[PATCH compass-ci] change file mode to be the same with others
by Ren Wen 04 Dec '20

04 Dec '20

Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/lib/json_logger.cr | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/lib/json_logger.cr diff --git a/src/lib/json_logger.cr b/src/lib/json_logger.cr old mode 100755 new mode 100644 -- 2.23.0

1 0

[PATCH compass-ci] scheduler/jobfile_operate.cr: refactor create_job_cpio
by Wu Zhende 04 Dec '20

04 Dec '20

Separate "create_job_cpio" into a new file. Use class Sched, can use Sched's instance variable. Also can simplify jobfile_operate.cr. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 1 + src/scheduler/create_job_cpio.cr | 140 +++++++++++++++++++++++++++++++ src/scheduler/find_job_boot.cr | 2 +- src/scheduler/jobfile_operate.cr | 132 ----------------------------- 4 files changed, 142 insertions(+), 133 deletions(-) create mode 100644 src/scheduler/create_job_cpio.cr diff --git a/src/lib/sched.cr b/src/lib/sched.cr index a9f893f..22e0e4e 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -20,6 +20,7 @@ require "../scheduler/find_next_job_boot" require "../scheduler/close_job" require "../scheduler/request_cluster_state" require "../scheduler/update_job_parameter" +require "../scheduler/create_job_cpio" class Sched property es diff --git a/src/scheduler/create_job_cpio.cr b/src/scheduler/create_job_cpio.cr new file mode 100644 index 0000000..5c1be7e --- /dev/null +++ b/src/scheduler/create_job_cpio.cr @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +# shellwords require from '/c/lkp-tests/lib/' +require "shellwords" +require "file_utils" +require "json" +require "yaml" + +require "./jobfile_operate" + +class Sched + private def prepare_dir(file_path : String) + file_path_dir = File.dirname(file_path) + unless File.exists?(file_path_dir) + FileUtils.mkdir_p(file_path_dir) + end + end + + private def valid_shell_variable?(key) + key =~ /^[a-zA-Z_]+[a-zA-Z0-9_]*$/ + end + + private def create_job_sh(job_sh_content : Array(JSON::Any), path : String) + File.open(path, "w", File::Permissions.new(0o775)) do |file| + file.puts "#!/bin/sh\n\n" + + job_sh_content.each do |line| + if line.as_a? + line.as_a.each { |val| file.puts val } + else + file.puts line + end + end + + file.puts "\"$@\"" + end + end + + private def shell_escape(val) + val = val.join "\n" if val.is_a?(Array) + + if val.nil? || val.empty? + return nil + elsif val =~ /^[+-]?([0-9]*\.?[0-9]+|[0-9]+\.?[0-9]*)([eE][+-]?[0-9]+)?$/ + return val + elsif !val.includes?("'") && !val.includes?("$") + return "'#{val}'" + elsif !val.includes?('"') + return "\"#{val}\"" + else + return Shellwords.shellescape(val) + end + end + + private def parse_one(script_lines, key, val) + return false if val.as_h? || !valid_shell_variable?(key) + + value = shell_escape(val.as_a? || val.to_s) + script_lines << "\texport #{key}=" + value if value + end + + private def sh_export_top_env(job_content : Hash) + script_lines = ["export_top_env()", "{"] of String + + job_content.each { |key, val| parse_one(script_lines, key, val) } + + script_lines << "\n" + script_lines << "\t[ -n \"$LKP_SRC\" ] ||" + script_lines << "\texport LKP_SRC=/lkp/${user:-lkp}/src" + script_lines << "}\n\n" + + script_lines = script_lines.to_s + script_lines = JSON.parse(script_lines) + end + + def create_job_cpio(job_content : JSON::Any, base_dir : String) + job_content = job_content.as_h + + # put job2sh in an array + if job_content.has_key?("job2sh") + tmp_job_sh_content = job_content["job2sh"] + + job_sh_array = [] of JSON::Any + tmp_job_sh_content.as_h.each do |_key, val| + next if val == nil + job_sh_array += val.as_a + end + else + job_sh_array = [] of JSON::Any + end + + # generate job.yaml + temp_yaml = base_dir + "/#{job_content["id"]}/job.yaml" + prepare_dir(temp_yaml) + + # no change to <object> content { "#! jobs/pixz.yaml": null } + # - this will create a <'#! jobs/pixz.yaml':> in the yaml file + # - but the orange is <#! jobs/pixz.yaml> in the user job.yaml + # tested : no effect to job.sh + File.open(temp_yaml, "w") do |file| + YAML.dump(job_content, file) + end + + # generate unbroken job shell content + sh_export_top_env = sh_export_top_env(job_content) + job_sh_content = sh_export_top_env.as_a + job_sh_array + + # generate job.sh + job_sh = base_dir + "/#{job_content["id"]}/job.sh" + create_job_sh(job_sh_content.to_a, job_sh) + + job_dir = base_dir + "/#{job_content["id"]}" + + if job_sh_array.empty? + lkp_src = Jobfile::Operate.prepare_lkp_tests(job_content["lkp_initrd_user"], + job_content["os_arch"]) + + cmd = "#{lkp_src}/sbin/create-job-cpio.sh #{temp_yaml}" + idd = `#{cmd}` + else + cmd = "./create-job-cpio.sh #{job_dir}" + idd = `#{cmd}` + end + + # if the create job cpio failed, what to do? + puts idd if idd.match(/ERROR/) + + # create result dir and copy job.sh, job.yaml and job.cgz to result dir + src_dir = File.dirname(temp_yaml) + dst_dir = File.join("/srv", job_content["result_root"].to_s) + FileUtils.mkdir_p(dst_dir) + + # the job.yaml is not final version + files = ["#{src_dir}/job.sh", + "#{src_dir}/job.yaml", + "#{src_dir}/job.cgz"] + FileUtils.cp(files, dst_dir) + end +end diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index ad48f27..6f51640 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -75,7 +75,7 @@ class Sched job = get_job_from_queues(queues, host) if job - Jobfile::Operate.create_job_cpio(job.dump_to_json_any, Kemal.config.public_folder) + create_job_cpio(job.dump_to_json_any, Kemal.config.public_folder) end return boot_content(job, boot_type) diff --git a/src/scheduler/jobfile_operate.cr b/src/scheduler/jobfile_operate.cr index 98bf011..df79d3b 100644 --- a/src/scheduler/jobfile_operate.cr +++ b/src/scheduler/jobfile_operate.cr @@ -5,139 +5,7 @@ require "file_utils" require "json" require "yaml" -# require from '/c/lkp-tests/lib/' -require "shellwords" - -LKP_PATH = "/c/lkp-tests" -if ENV["LKP_SRC"] != LKP_PATH - raise "ENV LKP_SRC mismatch: #{ENV["LKP_SRC"]} #{LKP_PATH}" -end - module Jobfile::Operate - def self.prepare_dir(file_path : String) - file_path_dir = File.dirname(file_path) - unless File.exists?(file_path_dir) - FileUtils.mkdir_p(file_path_dir) - end - end - - def self.valid_shell_variable?(key) - key =~ /^[a-zA-Z_]+[a-zA-Z0-9_]*$/ - end - - def self.create_job_sh(job_sh_content : Array(JSON::Any), path : String) - File.open(path, "w", File::Permissions.new(0o775)) do |file| - file.puts "#!/bin/sh\n\n" - job_sh_content.each do |line| - if line.as_a? - line.as_a.each { |val| file.puts val } - else - file.puts line - end - end - file.puts "\"$@\"" - end - end - - def self.shell_escape(val) - val = val.join "\n" if val.is_a?(Array) - - if val.nil? || val.empty? - return nil - elsif val =~ /^[+-]?([0-9]*\.?[0-9]+|[0-9]+\.?[0-9]*)([eE][+-]?[0-9]+)?$/ - return val - elsif !val.includes?("'") && !val.includes?("$") - return "'#{val}'" - elsif !val.includes?('"') - return "\"#{val}\"" - else - return Shellwords.shellescape(val) - end - end - - def self.parse_one(script_lines, key, val) - return false if val.as_h? || !valid_shell_variable?(key) - - value = shell_escape(val.as_a? || val.to_s) - script_lines << "\texport #{key}=" + value if value - end - - def self.sh_export_top_env(job_content : Hash) - script_lines = ["export_top_env()", "{"] of String - - job_content.each { |key, val| parse_one(script_lines, key, val) } - - script_lines << "\n" - script_lines << "\t[ -n \"$LKP_SRC\" ] ||" - script_lines << "\texport LKP_SRC=/lkp/${user:-lkp}/src" - script_lines << "}\n\n" - - script_lines = script_lines.to_s - script_lines = JSON.parse(script_lines) - end - - def self.create_job_cpio(job_content : JSON::Any, base_dir : String) - job_content = job_content.as_h - - # put job2sh in an array - if job_content.has_key?("job2sh") - tmp_job_sh_content = job_content["job2sh"] - - job_sh_array = [] of JSON::Any - tmp_job_sh_content.as_h.each do |_key, val| - next if val == nil - job_sh_array += val.as_a - end - else - job_sh_array = [] of JSON::Any - end - - # generate job.yaml - temp_yaml = base_dir + "/#{job_content["id"]}/job.yaml" - prepare_dir(temp_yaml) - - # no change to <object> content { "#! jobs/pixz.yaml": null } - # - this will create a <'#! jobs/pixz.yaml':> in the yaml file - # - but the orange is <#! jobs/pixz.yaml> in the user job.yaml - # tested : no effect to job.sh - File.open(temp_yaml, "w") do |file| - YAML.dump(job_content, file) - end - - # generate unbroken job shell content - sh_export_top_env = sh_export_top_env(job_content) - job_sh_content = sh_export_top_env.as_a + job_sh_array - - # generate job.sh - job_sh = base_dir + "/#{job_content["id"]}/job.sh" - create_job_sh(job_sh_content.to_a, job_sh) - - job_dir = base_dir + "/#{job_content["id"]}" - - if job_sh_array.empty? - lkp_src = prepare_lkp_tests(job_content["lkp_initrd_user"], - job_content["os_arch"]) - - cmd = "#{lkp_src}/sbin/create-job-cpio.sh #{temp_yaml}" - idd = `#{cmd}` - else - cmd = "./create-job-cpio.sh #{job_dir}" - idd = `#{cmd}` - end - - # if the create job cpio failed, what to do? - puts idd if idd.match(/ERROR/) - # create result dir and copy job.sh, job.yaml and job.cgz to result dir - src_dir = File.dirname(temp_yaml) - dst_dir = File.join("/srv", job_content["result_root"].to_s) - FileUtils.mkdir_p(dst_dir) - # the job.yaml is not final version - files = ["#{src_dir}/job.sh", - "#{src_dir}/job.yaml", - "#{src_dir}/job.cgz"] - FileUtils.cp(files, dst_dir) - end - def self.unzip_cgz(source_path : String, target_path : String) FileUtils.mkdir_p(target_path) cmd = "cd #{target_path};gzip -dc #{source_path}|cpio -id" -- 2.23.0

1 0

[PATCH lkp-tests 2/2] lib/common.rb: add build_pkg_error_id function
by Lin Jiaxin 03 Dec '20

03 Dec '20

Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- lib/common.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/common.rb b/lib/common.rb index 91997baa..cc64eb17 100755 --- a/lib/common.rb +++ b/lib/common.rb @@ -319,3 +319,25 @@ end def delete_file_if_exist(file) File.delete(file) if File.exist?(file) end + +def build_pkg_error_id(line) + line = line.chomp + line = line.gsub(/\b[3-9]\.[0-9]+[-a-z0-9.]+/, "#") # linux version: 3.17.0-next-20141008-g099669ed + line = line.gsub(/\b[1-9][0-9]-[A-Z][a-z]+-[0-9]{4}\b/, "#") # Date: 28-Dec-2013 + line = line.gsub(/\b0x[0-9a-f]+\b/, "#") # hex number + line = line.gsub(/\b[a-f0-9]{40}\b/, "#") # SHA-1 + line = line.gsub(/\b[0-9][0-9.]*/, "#") # number + line = line.gsub(/#x\b/, "0x") + line = line.gsub(/[\\"$]/, "~") + line = line.gsub(/[ \t]/, " ") + line = line.gsub(/\ \ +/, " ") + line = line.gsub(/([^a-zA-Z0-9])\ /, "\\1") + line = line.gsub(/\ ([^a-zA-Z])/, "\\1") + line = line.gsub(/^\ /, "") + line = line.gsub(/^-/, "") + line = line.gsub(/\ _/, "_") + line = line.tr(" ", "-") + line = line.gsub(/[-_.,;:#!\[(]+$/, "") + line = line.gsub(/([-_.,;:#!]){3,}/, ":") + line +end -- 2.23.0

1 0