March 2021 - Compass-ci - mailweb.openeuler.org

[PATCH compass-ci] src/lib(scheduer): change function name "*pushlish*" to "*publish*" for mq
by Xu Xijian 16 Mar '21

16 Mar '21

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/lifecycle.cr | 4 ++-- src/lib/mq.cr | 4 ++-- src/lib/sched.cr | 4 ++-- src/scheduler/close_job.cr | 2 +- src/scheduler/find_job_boot.cr | 2 +- src/scheduler/update_job_parameter.cr | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 1eb6264..4bd2ac0 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -79,7 +79,7 @@ class Lifecycle "job_state" => "abnormal", "testbox" => testbox } - @mq.pushlish_confirm("job_mq", msg.to_json) + @mq.publish_confirm("job_mq", msg.to_json) @match[testbox].delete(id) end end @@ -313,7 +313,7 @@ class Lifecycle mq_queue = get_machine_reboot_queue(testbox) machine.as_h.delete("history") machine.as_h["testbox"] = JSON::Any.new(testbox) - @mq.pushlish_confirm(mq_queue, machine.to_json, durable: true) + @mq.publish_confirm(mq_queue, machine.to_json, durable: true) machine["state"] = "rebooting_queue" machine["time"] = Time.local.to_s("%Y-%m-%dT%H:%M:%S+0800") diff --git a/src/lib/mq.cr b/src/lib/mq.cr index 980585d..1c7f4b7 100644 --- a/src/lib/mq.cr +++ b/src/lib/mq.cr @@ -37,7 +37,7 @@ class MQClient Singleton::Of(self).instance end - def pushlish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish_confirm(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) @@ -46,7 +46,7 @@ class MQClient end end - def pushlish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 9528f54..ac677de 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -219,9 +219,9 @@ class Sched return nil end - def mq_pushlish_confirm(queue, msg) + def mq_publish_confirm(queue, msg) 3.times do - @mq.pushlish_confirm(queue, msg) + @mq.publish_confirm(queue, msg) break rescue e res = @mq.reconnect diff --git a/src/scheduler/close_job.cr b/src/scheduler/close_job.cr index e014361..a57b233 100644 --- a/src/scheduler/close_job.cr +++ b/src/scheduler/close_job.cr @@ -45,7 +45,7 @@ class Sched "job_state" => "close", "time" => get_time } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end end diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 4a8ea8a..416f8f4 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -32,7 +32,7 @@ class Sched "time" => get_time, "job_state" => "boot" } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end # auto submit a job to collect the host information. diff --git a/src/scheduler/update_job_parameter.cr b/src/scheduler/update_job_parameter.cr index d81f28b..92b905e 100644 --- a/src/scheduler/update_job_parameter.cr +++ b/src/scheduler/update_job_parameter.cr @@ -44,6 +44,6 @@ class Sched "job_state" => (@env.get?("job_state") || "update").to_s, "time" => @env.get?("time").to_s } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end -- 2.23.0

1 0

[PATCH v3 lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 16 Mar '21

16 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 47 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..d6acad1ab 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,40 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + elsif error_msg.is_a? String + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/[^\w]/, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line.gsub!(/^-|-$/, '') + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

1 0

[PATCH compass-ci] doc: update document
by Luan Shengde 16 Mar '21

16 Mar '21

update document for service new port: send-internet-mail: 49000 => 10001 Signed-off-by: Luan Shengde <shdluan(a)163.com> --- doc/send-mail.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/send-mail.md b/doc/send-mail.md index 5af8d17..ffa5651 100644 --- a/doc/send-mail.md +++ b/doc/send-mail.md @@ -8,7 +8,7 @@ The service is used to send mails with formatted mail data ## send mail port - send intranet mail: 11311 -- send internet mail: 49000 +- send internet mail: 10001 ## send mail host - host the service running on @@ -58,5 +58,5 @@ The service is used to send mails with formatted mail data test msg 1010 " - %x(curl -XPOST 'localhost:49000/send_mail_text' -d "#{data}") + %x(curl -XPOST 'localhost:10001/send_mail_text' -d "#{data}") ``` -- 2.23.0

1 0

[PATCH v3 lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 16 Mar '21

16 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 47 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..9e9e15fab 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,40 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + elsif error_msg.is_a? String + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/[^\w]/, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line.gsub!(/-$/, '') + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

1 1

[PATCH v2 lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 16 Mar '21

16 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 47 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..119dc1d2a 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,40 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + else + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/[^\w]/, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line.gsub!(/-$/, '') + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

2 2

[PATCH compass-ci] lib: update port
by Luan Shengde 16 Mar '21

16 Mar '21

service assigned new port: send-internet-mail: 49000 => 10001 assign-account: 29999 => 10000 Signed-off-by: Luan Shengde <shdluan(a)163.com> --- lib/assign_account_client.rb | 4 ++-- lib/constants.rb | 2 +- lib/mail-post.rb | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/assign_account_client.rb b/lib/assign_account_client.rb index 4831545..f551a9d 100644 --- a/lib/assign_account_client.rb +++ b/lib/assign_account_client.rb @@ -21,9 +21,9 @@ names = Set.new %w[ defaults = relevant_defaults(names) JUMPER_HOST = defaults['JUMPER_HOST'] -JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 +JUMPER_PORT = defaults['JUMPER_PORT'] || 10000 SEND_MAIL_HOST = defaults['SEND_MAIL_HOST'] || 'localhost' -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 49000 +SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 10001 LAB = ENV['lab'] # used for other codes calling to assign account for user diff --git a/lib/constants.rb b/lib/constants.rb index 6c30aaf..03b1e40 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -9,7 +9,7 @@ ES_HOST ||= config['ES_HOST'] || '172.17.0.1' ES_PORT ||= config['ES_PORT'] || 9200 SEND_MAIL_HOST ||= config['SEND_MAIL_HOST'] || '172.17.0.1' -SEND_MAIL_PORT ||= config['SEND_MAIL_PORT'] || 49000 +SEND_MAIL_PORT ||= config['SEND_MAIL_PORT'] || 10001 SRV_HTTP_RESULT_HOST ||= config['SRV_HTTP_RESULT_HOST'] || ENV['SRV_HTTP_RESULT_HOST'] || '172.17.0.1' SRV_HTTP_OS_HOST ||= config['SRV_HTTP_OS_HOST'] || ENV['SRV_HTTP_OS_HOST'] || '172.17.0.1' diff --git a/lib/mail-post.rb b/lib/mail-post.rb index 4eb5800..84fc3c1 100755 --- a/lib/mail-post.rb +++ b/lib/mail-post.rb @@ -76,7 +76,7 @@ def check_send_mail(mail_info) raise 'no/empty email_to address.' if mail_info['to'].nil? || mail_info['to'].empty? raise 'no/empty email content.' if mail_info['body'].nil? || mail_info['body'].empty? - return mail_info unless ENV['SEND_MAIL_PORT'].to_s == '49000' + return mail_info unless ENV['SEND_MAIL_PORT'].to_s == '10001' mail_info = check_email_mapping(mail_info.clone) mail_info = check_email_limit(mail_info.clone) @@ -101,7 +101,7 @@ def send_mail(mail_info) end def check_to_store_email(mail) - return if ENV['SEND_MAIL_PORT'].to_s != '49000' + return if ENV['SEND_MAIL_PORT'].to_s != '10001' return if ENV['HOST_SERVER'] != 'z9' store_email(mail) -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] container/web-backend-nginx: change service port
by Lu Weitao 16 Mar '21

16 Mar '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend-nginx/nginx.conf | 4 ++-- container/web-backend-nginx/start | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/container/web-backend-nginx/nginx.conf b/container/web-backend-nginx/nginx.conf index c9a4ec4..381888a 100644 --- a/container/web-backend-nginx/nginx.conf +++ b/container/web-backend-nginx/nginx.conf @@ -8,7 +8,7 @@ http { default_type application/octet-stream; include /etc/nginx/conf.d/*.conf; server { - listen 11320 ssl; + listen 20003 ssl; server_name api.compass-ci.openeuler.org; ssl_certificate /opt/cert/web-backend.crt; ssl_certificate_key /opt/cert/web-backend.key; @@ -20,7 +20,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forward-For $proxy_add_x_forwarded_for; # web-backend server - proxy_pass http://172.17.0.1:32767; + proxy_pass http://172.17.0.1:10002; } location ~ ^/(result|pub) { diff --git a/container/web-backend-nginx/start b/container/web-backend-nginx/start index 4da4c8b..5360385 100755 --- a/container/web-backend-nginx/start +++ b/container/web-backend-nginx/start @@ -10,7 +10,7 @@ docker_rm web-backend-nginx cmd=( docker run --name web-backend-nginx - -p 11320:11320 + -p 20003:20003 -d -v /etc/ssl/certs:/opt/cert -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] container/web-backend: change service port
by Lu Weitao 16 Mar '21

16 Mar '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend/start | 2 +- container/web-backend/web-backend | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/container/web-backend/start b/container/web-backend/start index 6731d27..e680255 100755 --- a/container/web-backend/start +++ b/container/web-backend/start @@ -18,7 +18,7 @@ cmd=( --name web-backend --restart=always -d - -p 32767:32767 + -p 10002:10002 -e ES_PORT=9200 -e SRV_HTTP_RESULT_HOST=$SRV_HTTP_RESULT_HOST diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 74eda85..33a3fae 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -9,7 +9,7 @@ CCI_SRC = ENV['CCI_SRC'] || '/c/compass-ci' require "#{CCI_SRC}/src/lib/web_backend.rb" set :bind, '0.0.0.0' -set :port, 32767 +set :port, 10002 set :show_exceptions, false set :environment, :production -- 2.23.0

1 0

[PATCH lkp-tests] lib/email.sh: change default send email port
by Wei Jihui 16 Mar '21

16 Mar '21

49000 --> 10001 Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/email.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/email.sh b/lib/email.sh index a0c99c24..63d5a51e 100755 --- a/lib/email.sh +++ b/lib/email.sh @@ -87,5 +87,5 @@ send_email() ${subject}_content - curl -XPOST "${MAIL_HOST:-$LKP_SERVER}:${MAIL_PORT:-49000}/send_mail_text" -d "$email_content" + curl -XPOST "${MAIL_HOST:-$LKP_SERVER}:${MAIL_PORT:-10001}/send_mail_text" -d "$email_content" } -- 2.23.0

1 0

[PATCH v3 lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 16 Mar '21

16 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 81 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 tests/ansible_test diff --git a/tests/ansible_test b/tests/ansible_test new file mode 100755 index 000000000..bf75b19b7 --- /dev/null +++ b/tests/ansible_test @@ -0,0 +1,81 @@ +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + ansible_host=localhost + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] || { + echo "Ansible failed to install." + exit 1 + } + } +} + +download_ansible_role() +{ + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git + if [ $? -eq 0 ]; then + echo "$benchmark is successfully cloned" + else + echo "Please check whether your ansible_repo is correct!" + exit 1 + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: $ansible_host + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook is running failed!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

2 3