- Compass-ci - mailweb.openeuler.org

[PATCH lkp-tests 1/2] job: add monitors to job
by Wei Jihui 21 Oct '20

21 Oct '20

find all monitors and add to job scheduler will use it to get monitors program's cgz. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/job.rb | 7 +++++++ sbin/submit | 1 + 2 files changed, 8 insertions(+) diff --git a/lib/job.rb b/lib/job.rb index caa76c4b..e2416eb8 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -564,6 +564,13 @@ class Job return option_value end + def add_monitors + @job['monitors'] = {} + for_each_in(@job, monitor_params) do |_pk, _h, m_n, _m_args| + @job['monitors'][m_n] = nil + end + end + def add_pp() @job["pp"] = Hash.new() init_program_options() diff --git a/sbin/submit b/sbin/submit index a749ea20..8b46db32 100755 --- a/sbin/submit +++ b/sbin/submit @@ -108,6 +108,7 @@ ARGV.each do |jobfile| raise 'Please configure SCHED_PORT' unless job['SCHED_PORT'] job.add_pp + job.add_monitors job.add_define_files if opt_auto_define_files # get job shell function -- 2.23.0

1 0

[PATCH v6 compass-ci] container/assign-account: answerback-email.rb
by Luan Shengde 21 Oct '20

21 Oct '20

add parameters for answerback-email easier for administrator to manully assign jumper account for user with the parameters -e,--email email_address: add email address -s,--ssh-pubkey pub_key_file: add a ssh login pub_key -f,--raw-email email_file: add email address[, pub_key] via email file Usage: ruby answerback-email.rb -e email_address [-s pub_key_file] ruby answerback-email.rb -f email_file Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/answerback-email.rb | 178 ++++++++----------- 1 file changed, 76 insertions(+), 102 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index b5283ba..bb8e809 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -3,74 +3,89 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true -=begin - -repo_list: - all repos url list from upstream-repos.git - -API: -call graph: -read_mail_content -send_account_request - email_addr - get email address - email_message_id - get email message_id - check_email_available - check email available - pub_key_value - get pub key - account - send apply account request and return account info - build_message - build email message - send_mail - call send_mail to send mail whit build message - -the returned data for account_info like: -{ - "account" => "guest", - "passwd" => "Use pub_key to login", - "jumper_ip" => "10.10.10.10", - "jumper_port" => "10000" -} -=end +# xx --email xxx --login --ssh-pubkey xxx --raw-email email-file +# samba mount +# ssh logshn (huawei, ) (install pubkey / send password) require 'json' require 'mail' require 'set' -require_relative '../defconfig.rb' +require 'optparse' +require_relative '../defconfig' names = Set.new %w[ - JUMPER_IP + JUMPER_HOST JUMPER_PORT - CRYSTAL_INTRANET - SEND_MAIL_PORT + SEND_MAIL_HOST_INTERNET + SEND_MAIL_PORT_INTERNET ] defaults = relevant_defaults(names) -JUMPER_IP = defaults['JUMPER_IP'] -JUMPER_PORT = defaults['JUMPER_PORT'] -CRYSTAL_INTRANET = defaults['CRYSTAL_INTRANET'] -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] +JUMPER_HOST = defaults['JUMPER_HOST'] || 'api.compass-ci.openeuler.org' +JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 +SEND_MAIL_HOST = defaults['SEND_MAIL_HOST_INTERNET'] || 'localhost' +SEND_MAIL_PORT = defaults['SEND_MAIL_PORT_INTERNET'] || 11312 + +$apply_info = { + 'my_email' => nil, + 'my_ssh_pubkey' => nil +} + +def init_info(email_file) + mail_content = Mail.read(email_file) + + $apply_info['my_email'] = mail_content.from[0] + $apply_info['my_ssh_pubkey'] = if mail_content.part[1].filename == 'id_rsa.pub' + mail_content.part[1].body.decoded.gsub(/\r|\n/, '') + end + + $apply_info +end + +options = OptionParser.new do |opts| + opts.banner = "Usage: answerback-mail.rb [--email email] [--ssh-pubkey pub_key_file] [--raw-email email_file]\n" + opts.banner += " -e or -f is required\n" + opts.banner += ' -s is optional when use -e' + + opts.separator '' + opts.separator 'options:' + + opts.on('-e|--email email_address', 'appoint email address') do |email_address| + $apply_info['my_email'] = email_address + end + + opts.on('-s|--ssh-pubkey pub_key_file', 'ssh pub_key file, enable password-less login') do |pub_key_file| + $apply_info['my_ssh_pubkey'] = File.read(pub_key_file) + end + + opts.on('-f|--raw-email email_file', 'email file') do |email_file| + init_info(email_file) + end + + opts.on_tail('-h|--help', 'show this message') do + puts opts + exit + end +end + +options.parse!(ARGV) -def build_message(email, message_id, infos) +def build_message(email, acct_infos) message = <<~EMAIL_MESSAGE To: #{email} - Message-ID: #{message_id} Subject: jumper account is ready - Dear #{email} + Dear user: Thank you for joining us. You can use the following command to login the jumper server: login command: - ssh -p #{infos['jumper_port']} #{infos['account']}@#{infos['jumper_ip']} + ssh -p #{acct_infos['jumper_port']} #{acct_infos['account']}@#{acct_infos['jumper_ip']} - account passwd: - account_password: #{infos['passwd']} + account password: + #{acct_infos['passwd']} regards compass-ci @@ -79,67 +94,26 @@ def build_message(email, message_id, infos) return message end -def email_addr(mail_content) - msg = 'not an applying account email' - - raise msg unless mail_content.subject =~ /apply ssh account/i - - email = mail_content.from.join(',') - - return email -end - -# def check_email_available(mail_content, email) -# oos_list = File.read('/c/upstream-repos/repo_list').split(/\n/) -# url = mail_content.body.decoded.split(/\n/).find { |line| line =~ /https?:\/\// } -# base_url = url.split('/')[0,5].join('/') -# message = 'The url is not in upstream repo_list' -# -# raise message unless oos_list.include? base_url -# -# url_fdback = %x(curl #{url}) -# email_index = url_fdback.index email -# -# message = 'No commit info found from the url for the email' -# raise message unless email_index -# end - -def email_message_id(mail_content) - message_id = mail_content.message_id - return message_id -end - -def pub_key_value(mail_content) - pub_key = mail_content.body.decoded.split(/\n/).find { |line| line =~ /ssh-rsa/ } - return pub_key -end - def account_info(pub_key) - account_info_str = %x(curl -XGET '#{JUMPER_IP}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") - account_info = JSON.parse account_info_str - - return account_info + account_info_str = if pub_key.nil? + %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account') + else + %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") + end + JSON.parse account_info_str end -def send_account(mail_content) - email = email_addr(mail_content) - message_id = email_message_id(mail_content) - # check_email_available(mail_content, email) +def send_account + message = "No email address specified\n" + message += "use -e email_address add a email address\n" + message += 'or use -f to add a email file' + raise message if $apply_info['my_email'].nil? - pub_key = pub_key_value(mail_content) - acct_info = account_info(pub_key) - - message = build_message(email, message_id, acct_info) - - %x(curl -XPOST '#{CRYSTAL_INTRANET}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") -end + acct_info = account_info($apply_info['my_ssh_pubkey']) -def read_mail_content(mail_file) - mail_content = Mail.read(mail_file) + message = build_message($apply_info['my_email'], acct_info) - return mail_content + %x(curl -XPOST '#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end -mail_file = ARGV[0] -mail_content = read_mail_content(mail_file) -send_account(mail_content) +send_account -- 2.23.0

1 0

[PATCH compass-ci] sched:add monitors depends to job
by Cao Xueliang 21 Oct '20

21 Oct '20

[why] It may add monitors field in job yaml, sched will parse the monitors field and add depends to the initrd string. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/lib/job.cr | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 12d3980..00b5e68 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -396,12 +396,42 @@ class Job initrd_pkg_arr = Array(String).new initrd_http_prefix = "http://#{INITRD_HTTP_HOST}:#{INITRD_HTTP_PORT}" mount_type = os_mount == "cifs" ? "nfs" : os_mount.dup + + get_monitors_initrd(mount_type, initrd_http_prefix, initrd_deps_arr, initrd_pkg_arr) + get_pp_initrd(mount_type, initrd_http_prefix, initrd_deps_arr, initrd_pkg_arr) + + self["initrd_deps"] = initrd_deps_arr.uniq.join(" ") + self["initrd_pkg"] = initrd_pkg_arr.uniq.join(" ") + end + + private def get_monitors_initrd(mount_type, initrd_http_prefix, initrd_deps_arr, initrd_pkg_arr) + if @hash["monitors"]? + program_params = @hash["monitors"].as_h + program_params.keys.each do |program| + deps_dest_file = "#{SRV_INITRD}/deps/#{mount_type}/#{os_dir}/#{program}.cgz" + pkg_dest_file = "#{SRV_INITRD}/pkg/#{mount_type}/#{os_dir}/#{program}.cgz" + + if File.exists?("#{user_lkp_src}/monitors/#{program}") + if File.exists?(deps_dest_file) + initrd_deps_arr << "#{initrd_http_prefix}" + JobHelper.service_path(deps_dest_file) + end + + if File.exists?(pkg_dest_file) + initrd_pkg_arr << "#{initrd_http_prefix}" + JobHelper.service_path(pkg_dest_file) + end + end + end + end + end + + private def get_pp_initrd(mount_type, initrd_http_prefix, initrd_deps_arr, initrd_pkg_arr) if @hash["pp"]? program_params = @hash["pp"].as_h program_params.keys.each do |program| if program =~ /^(.*)-\d+$/ program = $1 end + deps_dest_file = "#{SRV_INITRD}/deps/#{mount_type}/#{os_dir}/#{program}.cgz" pkg_dest_file = "#{SRV_INITRD}/pkg/#{mount_type}/#{os_dir}/#{program}.cgz" @@ -414,9 +444,6 @@ class Job end end end - - self["initrd_deps"] = initrd_deps_arr.uniq.join(" ") - self["initrd_pkg"] = initrd_pkg_arr.uniq.join(" ") end def update_tbox_group(tbox_group) -- 2.23.0

2 3

[PATCH lkp-tests 2/2] feat(iso2rootfs): add step: submit rootfs test job after rootfs is made
by Yu Chuan 21 Oct '20

21 Oct '20

[Why] The produced dailybuild rootfs needs further testing by submit a job. [How] 1. Configure lkp env by {LKP_SRC}/sbin/install-dependencies.sh 2. Generate global compass-ci yaml. - The path of yaml: - /etc/compass-ci/defaults/sparrow.yaml - The yaml have two necessary fields: - SCHED_HOST - SCHED_PORT - The value of these two fields are consistent with the value in job.yaml of iso2rootfs. 3. Modify job.yaml of rootfs test job. - Add the required fields of job.yaml of rootfs test job to job.yaml of iso2rootfs; - os - os_arch - os_version - os_mount - testbox - Replace/insert them before submit the rootfs test job. 4. Submit the job.yaml of rootfs test job. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 6 ++++ tests/iso2rootfs | 72 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index f413c9cee1f0..644c64be06b4 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -11,3 +11,9 @@ iso2rootfs: rootfs_path: initrd_http_host: initrd_http_port: + test_yaml: + test_os: + test_os_arch: + test_os_version: + test_os_mount: + test_testbox: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index bf1f995e78df..5f56f5c21db9 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -53,6 +53,12 @@ check_yaml_vars() "rootfs_path" "initrd_http_host" "initrd_http_port" + "test_yaml" + "test_os" + "test_os_arch" + "test_os_version" + "test_os_mount" + "test_testbox" ) local yaml_t @@ -404,6 +410,69 @@ test_rootfs() EOF } +############ submit test job yaml ############ +generate_global_yaml() +{ + local config_yaml="/etc/compass-ci/defaults/sparrow.yaml" + + mkdir -p "$(dirname "${config_yaml}")" + echo "SCHED_HOST: ${SCHED_HOST}" >> "${config_yaml}" + echo "SCHED_PORT: ${SCHED_PORT}" >> "${config_yaml}" +} + +assign_var() +{ + local var_name=$1 + local var_value=$2 + + if grep -qw "${var_name}:" "${test_yaml}" + then + sed -i "s/^$ *${var_name}:$.*/\1 ${var_value}/g" "${test_yaml}" + else + echo "${var_name}: ${var_value}" >> "${test_yaml}" + fi +} + +modify_test_job_yaml() +{ + log_info "starting prepare ${test_yaml}" + + [ -f "${test_yaml}" ] || die "test yaml$${test_yaml}$ exist in {LKP_SRC}/jobs !!!" + + local wait_add_vars=( + "os" + "os_arch" + "os_version" + "os_mount" + "testbox" + ) + for vt in "${wait_add_vars[@]}" + do + assign_var "$vt" "$(eval echo "\$test_${vt}")" + done +} + +submit_test_job_yaml() +{ + log_info "starting submit ${test_yaml} ..." + + [ -d "${LKP_SRC}" ] || die "no LKP_SRC !!!" + + cd "${LKP_SRC}" && { + chmod +x ./sbin/install-dependencies.sh + ./sbin/install-dependencies.sh + + generate_global_yaml + } + + cd "${LKP_SRC}/jobs" && { + modify_test_job_yaml + "${LKP_SRC}/sbin/submit" "${test_yaml}" + } + + log_info "submit ${test_yaml} finished" +} + ############ post works ############ post_works() { @@ -414,7 +483,7 @@ post_works() local soft_link="${iso_version}-dailybuild" cd "${ROOTFS_LOCAL_PATH}" && rm -f "${soft_link}" && - ln -s "$(basename ${ROOTFS_DES_DIR})" "${soft_link}" + ln -s "$(basename "${ROOTFS_DES_DIR}")" "${soft_link}" cd / && umount "${ROOTFS_LOCAL_PATH}" @@ -433,6 +502,7 @@ main() run_qcow2rootfs test_rootfs + submit_test_job_yaml post_works } -- 2.23.0

4 5

[PATCH compass-ci 1/2] src/lib: find depends_initrd for multitask
by Li Ping 21 Oct '20

21 Oct '20

[why] when the job.yaml like multi-qemu.yaml: multi-qemu-0: multi-qemu-1: the code need multi-qemu, not multi-qemu-x, and scheduler can not find the depends_initrd multi-qemu-x.cgz. Signed-off-by: Li Ping <15396232681(a)163.com> --- src/lib/job.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 12d3980..f0a9007 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -399,7 +399,7 @@ class Job if @hash["pp"]? program_params = @hash["pp"].as_h program_params.keys.each do |program| - if program =~ /^(.*)-\d+$/ + if program =~ /(.*)-\d+$/ program = $1 end deps_dest_file = "#{SRV_INITRD}/deps/#{mount_type}/#{os_dir}/#{program}.cgz" -- 2.23.0

1 1

[PATCH compass-ci 2/2] job.cr: change the default values of os and os_version
by Li Ping 21 Oct '20

21 Oct '20

--- src/lib/job.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index f0a9007..54d0862 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -39,10 +39,10 @@ class Job getter hash : Hash(String, JSON::Any) INIT_FIELD = { - os: "debian", + os: "openeuler", lab: LAB, os_arch: "aarch64", - os_version: "sid", + os_version: "20.03", lkp_initrd_user: "latest", docker_image: "centos:7", } -- 2.23.0

1 0

[PATCH lkp-tests 1/2] fix(install-dependencies.sh): fix some problems when configure lkp env
by Yu Chuan 21 Oct '20

21 Oct '20

[Why] 1. param: '-y' + ./sbin/install-dependencies.sh ...Omit... Total download size: 3.7 M Installed size: 18 M Is this ok [y/N]: 2. pkgs: gcc, gcc-c++, make ERROR: Error installing rest-client: ERROR: Failed to build gem native extension. current directory: /usr/local/share/gems/gems/unf_ext-0.0.7.7/ext/unf_ext /usr/bin/ruby -r ./siteconf20201020-5195-h45kpd.rb extconf.rb checking for -lstdc++... *** extconf.rb failed *** Could not create Makefile due to some reason, probably lack of necessary libraries and/or headers. Check the mkmf.log file for more details. You may need configuration options. ...Omit... /usr/share/ruby/mkmf.rb:456:in `try_do': The compiler failed to generate an executable file. (RuntimeError) You have to install development tools first. ...Omit... 3. pkg: ruby-devel ERROR: Error installing rest-client: ERROR: Failed to build gem native extension. current directory: /usr/local/share/gems/gems/unf_ext-0.0.7.7/ext/unf_ext /usr/bin/ruby -r ./siteconf20201020-5371-1gqrhar.rb extconf.rb mkmf.rb can't find header files for ruby at /usr/share/include/ruby.h extconf failed, exit code 1 Gem files will remain installed in /usr/local/share/gems/gems/unf_ext-0.0.7.7 for inspection. Results logged to /usr/local/lib64/gems/ruby/unf_ext-0.0.7.7/gem_make.out 4. pkg: git + {LKP_SRC}/sbin/submit test-tools.yaml /usr/local/share/gems/gems/git-1.7.0/lib/git/lib.rb:989:in `command': git '-c' 'color.ui=false' version 2>&1:sh: git: command not found (Git::GitExecuteError) Signed-off-by: Yu Chuan <13186087857(a)163.com> --- sbin/install-dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbin/install-dependencies.sh b/sbin/install-dependencies.sh index 48457d4c61cc..5673f7a773a3 100644 --- a/sbin/install-dependencies.sh +++ b/sbin/install-dependencies.sh @@ -14,9 +14,9 @@ ubuntu|debian) ;; openEuler|fedora|rhel|centos) if type dnf > /dev/null 2>&1; then - dnf install --skip-broken ruby rubygems + dnf install -y --skip-broken ruby rubygems gcc gcc-c++ make ruby-devel git else - yum install --skip-broken ruby rubygems + yum install -y --skip-broken ruby rubygems gcc gcc-c++ make ruby-devel git fi gem install -f git activesupport rest-client ;; -- 2.23.0

3 5

[PATCH v2 compass-ci 2/2] fix: scheduler: refactor match_tbox_group
by Tong Qunfeng 21 Oct '20

21 Oct '20

[why] simplify string match operate. Signed-off-by: Tong Qunfeng <taxcom(a)tom.com> --- src/lib/job.cr | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 12d3980..8dad205 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -22,9 +22,8 @@ end module JobHelper def self.match_tbox_group(testbox : String) tbox_group = testbox - find = testbox.match(/(.*)(\-\d{1,}$)/) - if find != nil - tbox_group = find.not_nil![1] + if testbox =~ /(.*)-\d+$/ + tbox_group = $1 end return tbox_group end -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] fix: scheduler: cut down log
by Tong Qunfeng 21 Oct '20

21 Oct '20

[why] when consume with "No job now", scheduler output 14 lines log. when submit 1 job and consume it, scheduler output 78 lines log. it is a bit of heavy for log collect and some logs are redundancy. [how] 1.merge 4 lines to 1 line (form to json) for debug_message. 2.remove output "Done", it's same with kemal-frame meesage "200 GET/PUT..." 3.only 0 | 1 extra message for 1 API: if there has an active message, do not call debug_message as default, there has 1 kemal-frame message for each API 2020-10-20 09:40:28 UTC 200 GET /boot.ipxe/mac/0a-4c-34-18-0a-17 7.18ms <time in> <response status> <----request API----> <response time span> [now] this patch cut 92 lines log to 26 line, interactive with 15 API. examples: --- <4 to 1> and <remove single "Done"> >> 172.168.131.113:29720 << Done 2020-10... 200 PUT /set_host_mac?hostname=vm... 289.2µs --> 2020-10... 200 PUT /set_host_mac?hostname=vm... 977.56µs --- <0 | 1 extra message> >> 172.18.222.81:44210 << Done {"job_state":"running","job_id":"crystal.105070"} 2020-10... 200 GET /~lkp/cgi-bin/lkp-jobfile-append-var?... 2.36ms --> {"job_state":"running","job_id":"crystal.105071"} 2020-10... UTC 200 GET /~lkp/cgi-bin/lkp-jobfile-append-var?... 2.8ms Signed-off-by: Tong Qunfeng <taxcom(a)tom.com> --- src/lib/sched.cr | 9 +++------ src/scheduler/scheduler.cr | 30 ++++++++++-------------------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index e6f353a..7215df5 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -317,7 +317,6 @@ class Sched JobHelper.service_path("#{SRV_INITRD}/lkp/#{job.lkp_initrd_user}/lkp-#{job.arch}.cgz") response["job"] = "http://#{SCHED_HOST}:#{SCHED_PORT}/job_initrd_tmpfs/#{job.id}/job.cgz" - puts %({"job_id": "#{job.id}", "job_state": "boot"}) return response.to_json end @@ -339,7 +338,6 @@ class Sched response += "boot\n" - puts %({"job_id": "#{job.id}", "job_state": "boot"}) return response end @@ -481,7 +479,6 @@ class Sched response += job.kernel_params response += "\nboot\n" - puts %({"job_id": "#{job.id}", "job_state": "boot"}) return response end @@ -512,7 +509,7 @@ class Sched # json log log = job_content.dup log["job_id"] = log.delete("id").not_nil! - puts log.to_json + return log.to_json end def update_tbox_wtmp(env : HTTP::Server::Context) @@ -541,7 +538,7 @@ class Sched # json log hash["testbox"] = testbox - puts hash.to_json + return hash.to_json end def report_ssh_port(testbox : String, ssh_port : String) @@ -572,6 +569,6 @@ class Sched @redis.remove_finished_job(job_id) - puts %({"job_id": "#{job_id}", "job_state": "complete"}) + return %({"job_id": "#{job_id}", "job_state": "complete"}) end end diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr index e38d15c..5befcf9 100644 --- a/src/scheduler/scheduler.cr +++ b/src/scheduler/scheduler.cr @@ -34,9 +34,7 @@ module Scheduler # for debug (maybe kemal debug|logger does better) def self.debug_message(env, response) - puts "\n\n" - puts ">> #{env.request.remote_address}" - puts "<< #{response}" + puts %({"from": "#{env.request.remote_address}", "response": #{response.to_json}}) end # echo alive @@ -53,7 +51,8 @@ module Scheduler get "/boot.:boot_type/:parameter/:value" do |env| response = sched.find_job_boot(env) - debug_message(env, response) + job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? + puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id response end @@ -62,7 +61,8 @@ module Scheduler get "/~lkp/cgi-bin/gpxelinux.cgi" do |env| response = sched.find_next_job_boot(env) - debug_message(env, response) + job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? + puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id response end @@ -87,7 +87,6 @@ module Scheduler file_path = ::File.join [Kemal.config.public_folder, job_id, job_package] puts %({"job_id": "#{job_id}", "job_state": "download"}) - debug_message(env, file_path) send_file env, file_path end @@ -102,8 +101,6 @@ module Scheduler if (client_hostname = env.params.query["hostname"]?) && (client_mac = env.params.query["mac"]?) sched.set_host_mac(client_mac, client_hostname) - debug_message(env, "Done") - "Done" else "No yet!" @@ -115,8 +112,6 @@ module Scheduler if client_mac = env.params.query["mac"]? sched.del_host_mac(client_mac) - debug_message(env, "Done") - "Done" else "No yet!" @@ -129,10 +124,8 @@ module Scheduler # ?job_file=/lkp/scheduled/job.yaml&job_state=post_run&job_id=10 # ?job_file=/lkp/scheduled/job.yaml&loadavg=0.28 0.82 0.49 1/105 3389&start_time=1587725398&end_time=1587725698&job_id=10 get "/~lkp/cgi-bin/lkp-jobfile-append-var" do |env| - # get job_id from request - debug_message(env, "Done") + puts sched.update_job_parameter(env) - sched.update_job_parameter(env) "Done" end @@ -168,17 +161,15 @@ module Scheduler # get job_id from request job_id = env.params.query["job_id"]? if job_id - debug_message(env, "Done") - - sched.close_job(job_id) + puts sched.close_job(job_id) end + "Done" end get "/~lkp/cgi-bin/lkp-wtmp" do |env| - debug_message(env, "Done") + puts sched.update_tbox_wtmp(env) - sched.update_tbox_wtmp(env) "Done" end @@ -188,12 +179,11 @@ module Scheduler job_id = env.params.query["job_id"].to_s if testbox && ssh_port - debug_message(env, "Done") - sched.report_ssh_port(testbox, ssh_port) end puts %({"job_id": "#{job_id}", "state": "set ssh port", "ssh_port": "#{ssh_port}", "tbox_name": "#{testbox}"}) + "Done" end end -- 2.23.0

1 0

[PATCH v4 compass-ci] /src/lib/web_backend.rb: fix "/compare_candidates" tbox_group regex bug
by Zhang Yuhang 21 Oct '20

21 Oct '20

Error Flow: 1. /\d+$/ will be matched, such as "xxx123". 2. `index = "xxx123".index('--') || "xxx123".rindex('-')` - index equal nil. 3. `r = r[0, nil]` - raise a TypeError. Solution: Fill a couple brackets, so will just match "vm-abc-0abc-abc" or "vm-abc-0abc-012". Signed-off-by: Zhang Yuhang <zhangyuhang25(a)huawei.com> --- src/lib/web_backend.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 48cfadb..fda01ca 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -47,7 +47,7 @@ end def filter_tbox_group(es_result) result = Set.new es_result.each do |r| - if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-([a-zA-Z]+)|(\d+)$)/ + if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-(([a-zA-Z]+)|(\d+))$)/ index = r.index('--') || r.rindex('-') r = r[0, index] end -- 2.23.0

2 3

[PATCH lkp-tests] fix: daemon: don't run 'netserver' foreground
by Ren Wen 21 Oct '20

21 Oct '20

[Why] "-D : Do not daemonize" Option '-D' will run 'netserver' foreground. Need to get exit status of executed 'netserver' when running cluster test, so remove '-D' [Ref] [Running 'netserver' as a standalone Daemon] (http://www.cs.kent.edu/~farrell/dist/ref/Netperf.html#0.2.2Z141Z1.SUJSTF.7R…) Signed-off-by: Ren Wen <15991987063(a)163.com> --- daemon/netserver | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/netserver b/daemon/netserver index 0302dd35..d6eff6da 100755 --- a/daemon/netserver +++ b/daemon/netserver @@ -9,5 +9,5 @@ opt_ip= [ "$ip" = 'ipv6' ] && opt_ip='-6' # start netserver -log_echo netserver $opt_ip -D -exec netserver $opt_ip -D +log_echo netserver $opt_ip +exec netserver $opt_ip -- 2.23.0

2 2

[PATCH lab-z9] hosts: add hostfile for a95
by Bai Jing 21 Oct '20

21 Oct '20

Signed-off-by: Bai Jing <799286817(a)qq.com> --- hosts/taishan200-2280-2s48p-256g--a95 | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 hosts/taishan200-2280-2s48p-256g--a95 diff --git a/hosts/taishan200-2280-2s48p-256g--a95 b/hosts/taishan200-2280-2s48p-256g--a95 new file mode 100644 index 0000000..3fd8124 --- /dev/null +++ b/hosts/taishan200-2280-2s48p-256g--a95 @@ -0,0 +1,19 @@ +nr_node: 4 +nr_cpu: 96 +memory: 512G +nr_hdd_partitions: 4 +hdd_partitions: + - /dev/disk/by-id/scsi-35000039988792e31 + - /dev/disk/by-id/scsi-35000039988793705 + - /dev/disk/by-id/scsi-35000039988792aa5 + - /dev/disk/by-id/scsi-350000399887932fd +mac_addr: + - cc:05:77:fd:fb:61 + - cc:05:77:fd:fb:62 + - cc:05:77:fd:fb:63 + - cc:05:77:fd:fb:64 + - cc:05:77:fd:fb:65 + - cc:05:77:fd:fb:66 + - cc:05:77:fd:fb:67 + - cc:05:77:fd:fb:68 +model_name: Kunpeng-920 -- 2.23.0

1 0

[PATCH lkp-tests] lib: fix btrfs-progs is not installed when os_mount=initramfs
by Hu Xuejiao 21 Oct '20

21 Oct '20

When job.yaml container "fs: **", and /srv/initrd/deps/**/**/fs.cgz(btrfs-progs), it will execute mkfs.btrfs -f $rootfs_partition. but job.yaml don't container "fs: **" to appear warning: mkfs.btrfs command not found Signed-off-by: Hu XueJiao <huxuejiao1(a)huawei.com> --- lib/bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index a7776d68..16789c5c 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -456,6 +456,7 @@ mount_rootfs() ROOTFS_DIR=/opt/rootfs mkdir -p $ROOTFS_DIR mount $rootfs_partition $ROOTFS_DIR || { + command -v mkfs.btrfs >/dev/null || return mkfs.btrfs -f $rootfs_partition mount $rootfs_partition $ROOTFS_DIR } -- 2.23.0

2 1

[PATCH compass-ci] auto_submit: add upstream_tag if the commit is tagged
by Li Yuanchao 21 Oct '20

21 Oct '20

sometimes the testcase needs a specific version software, add upstream_tag can help find the specific version. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sbin/auto_submit b/sbin/auto_submit index 1f55701..88ec560 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -28,12 +28,25 @@ class AutoSubmit return pkgbuild_repo end + def if_tagged(tag_list, commit_id) + tag_list.each_line do |line| + if line.include?(commit_id) + tag = line.split + return tag[1].delete_prefix('refs/tags/') + end + end + return nil + end + def submit(newrefs_info, submit_argv) + tag_list = %x(git -C /srv/git/#{newrefs_info['git_repo']}.git show-ref --tags) newrefs_info['new_refs']['heads'].each do |branch, commit_id| real_argvs = Array.new(submit_argv) commit_date = `git -C /srv/git/#{newrefs_info['git_repo']}.git log --format=%ct -1 #{commit_id}` real_argvs.push("upstream_branch=#{branch.delete_prefix('refs/heads/')}") real_argvs.push("upstream_commit=#{commit_id}") + tag = if_tagged(tag_list, commit_id) + real_argvs.push("upstream_tag=#{tag}") if tag real_argvs.push("commit_date=#{commit_date}") system(real_argvs.join(' ')) -- 2.23.0

7 17

[PATCH v5 lkp-tests] tests: add get_ipmi_ip function to host-info
by Zhang Yu 21 Oct '20

21 Oct '20

Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- tests/host-info | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/host-info b/tests/host-info index a57855dc..99b14083 100755 --- a/tests/host-info +++ b/tests/host-info @@ -80,6 +80,25 @@ get_partitions() fi } +get_ipmi_ip() +{ + # root@localhost ~# ipmitool lan print + # Set in Progress : Set Complete + # IP Address Source : Static Address + # IP Address : 9.3.4.1 + # Subnet Mask : 255.255.0.0 + # MAC Address : e0:00:84:2b:51:7b + # SNMP Community String : TrapAdmin12#$ + # IP Header : TTL=0x40 Flags=0x40 Precedence=0x00 TOS=0x10 + # Default Gateway IP : 9.3.0.1 + # 802.1q VLAN ID : Disabled + # RMCP+ Cipher Suites : 0,1,2,3,17 + # Cipher Suite Priv Max : XuuaXXXXXXXXXXX + imp_ip=$(ipmitool lan print |awk 'NR==3{print $4}') + [ $? -eq 0 ] && echo "ipmi_ip:" $imp_ip + +} + main() { get_node @@ -87,6 +106,7 @@ main() get_partitions get_mac get_model_name + get_ipmi_ip } main -- 2.23.0

3 2

[PATCH compass-ci] fix: scheduler: cut down log
by Tong Qunfeng 21 Oct '20

21 Oct '20

[why] when consume with "No job now", scheduler output 14 lines log. when submit 1 job and consume it, scheduler output 78 lines log. it is a bit of heavy for log collect and some logs are redundancy. [how] 1.merge 4 lines to 1 line (form to json) for debug_message. 2.remove output "Done", it's same with kemal-frame meesage "200 GET/PUT..." 3.only 0 | 1 extra message for 1 API: if there has an active message, do not call debug_message as default, there has 1 kemal-frame message for each API 2020-10-20 09:40:28 UTC 200 GET /boot.ipxe/mac/0a-4c-34-18-0a-17 7.18ms <time in> <response status> <----request API----> <response time span> [now] this patch cut 92 lines log to 26 line, interactive with 15 API. examples: --- <4 to 1> and <remove single "Done"> >> 172.168.131.113:29720 << Done 2020-10... 200 PUT /set_host_mac?hostname=vm... 289.2µs --> 2020-10... 200 PUT /set_host_mac?hostname=vm... 977.56µs --- <0 | 1 extra message> >> 172.18.222.81:44210 << Done {"job_state":"running","job_id":"crystal.105070"} 2020-10... 200 GET /~lkp/cgi-bin/lkp-jobfile-append-var?... 2.36ms --> {"job_state":"running","job_id":"crystal.105071"} 2020-10... UTC 200 GET /~lkp/cgi-bin/lkp-jobfile-append-var?... 2.8ms Signed-off-by: Tong Qunfeng <taxcom(a)tom.com> --- src/lib/sched.cr | 9 +++------ src/scheduler/scheduler.cr | 30 ++++++++++-------------------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index e6f353a..7215df5 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -317,7 +317,6 @@ class Sched JobHelper.service_path("#{SRV_INITRD}/lkp/#{job.lkp_initrd_user}/lkp-#{job.arch}.cgz") response["job"] = "http://#{SCHED_HOST}:#{SCHED_PORT}/job_initrd_tmpfs/#{job.id}/job.cgz" - puts %({"job_id": "#{job.id}", "job_state": "boot"}) return response.to_json end @@ -339,7 +338,6 @@ class Sched response += "boot\n" - puts %({"job_id": "#{job.id}", "job_state": "boot"}) return response end @@ -481,7 +479,6 @@ class Sched response += job.kernel_params response += "\nboot\n" - puts %({"job_id": "#{job.id}", "job_state": "boot"}) return response end @@ -512,7 +509,7 @@ class Sched # json log log = job_content.dup log["job_id"] = log.delete("id").not_nil! - puts log.to_json + return log.to_json end def update_tbox_wtmp(env : HTTP::Server::Context) @@ -541,7 +538,7 @@ class Sched # json log hash["testbox"] = testbox - puts hash.to_json + return hash.to_json end def report_ssh_port(testbox : String, ssh_port : String) @@ -572,6 +569,6 @@ class Sched @redis.remove_finished_job(job_id) - puts %({"job_id": "#{job_id}", "job_state": "complete"}) + return %({"job_id": "#{job_id}", "job_state": "complete"}) end end diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr index e38d15c..5befcf9 100644 --- a/src/scheduler/scheduler.cr +++ b/src/scheduler/scheduler.cr @@ -34,9 +34,7 @@ module Scheduler # for debug (maybe kemal debug|logger does better) def self.debug_message(env, response) - puts "\n\n" - puts ">> #{env.request.remote_address}" - puts "<< #{response}" + puts %({"from": "#{env.request.remote_address}", "response": #{response.to_json}}) end # echo alive @@ -53,7 +51,8 @@ module Scheduler get "/boot.:boot_type/:parameter/:value" do |env| response = sched.find_job_boot(env) - debug_message(env, response) + job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? + puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id response end @@ -62,7 +61,8 @@ module Scheduler get "/~lkp/cgi-bin/gpxelinux.cgi" do |env| response = sched.find_next_job_boot(env) - debug_message(env, response) + job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? + puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id response end @@ -87,7 +87,6 @@ module Scheduler file_path = ::File.join [Kemal.config.public_folder, job_id, job_package] puts %({"job_id": "#{job_id}", "job_state": "download"}) - debug_message(env, file_path) send_file env, file_path end @@ -102,8 +101,6 @@ module Scheduler if (client_hostname = env.params.query["hostname"]?) && (client_mac = env.params.query["mac"]?) sched.set_host_mac(client_mac, client_hostname) - debug_message(env, "Done") - "Done" else "No yet!" @@ -115,8 +112,6 @@ module Scheduler if client_mac = env.params.query["mac"]? sched.del_host_mac(client_mac) - debug_message(env, "Done") - "Done" else "No yet!" @@ -129,10 +124,8 @@ module Scheduler # ?job_file=/lkp/scheduled/job.yaml&job_state=post_run&job_id=10 # ?job_file=/lkp/scheduled/job.yaml&loadavg=0.28 0.82 0.49 1/105 3389&start_time=1587725398&end_time=1587725698&job_id=10 get "/~lkp/cgi-bin/lkp-jobfile-append-var" do |env| - # get job_id from request - debug_message(env, "Done") + puts sched.update_job_parameter(env) - sched.update_job_parameter(env) "Done" end @@ -168,17 +161,15 @@ module Scheduler # get job_id from request job_id = env.params.query["job_id"]? if job_id - debug_message(env, "Done") - - sched.close_job(job_id) + puts sched.close_job(job_id) end + "Done" end get "/~lkp/cgi-bin/lkp-wtmp" do |env| - debug_message(env, "Done") + puts sched.update_tbox_wtmp(env) - sched.update_tbox_wtmp(env) "Done" end @@ -188,12 +179,11 @@ module Scheduler job_id = env.params.query["job_id"].to_s if testbox && ssh_port - debug_message(env, "Done") - sched.report_ssh_port(testbox, ssh_port) end puts %({"job_id": "#{job_id}", "state": "set ssh port", "ssh_port": "#{ssh_port}", "tbox_name": "#{testbox}"}) + "Done" end end -- 2.23.0

1 0

[PATCH lkp-tests] job: add monitors to job
by Wei Jihui 21 Oct '20

21 Oct '20

find all monitors and add to job scheduler will use it to get monitors program's cgz. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/job.rb | 7 +++++++ sbin/submit | 1 + 2 files changed, 8 insertions(+) diff --git a/lib/job.rb b/lib/job.rb index caa76c4b..e2416eb8 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -564,6 +564,13 @@ class Job return option_value end + def add_monitors + @job['monitors'] = {} + for_each_in(@job, monitor_params) do |_pk, _h, m_n, _m_args| + @job['monitors'][m_n] = nil + end + end + def add_pp() @job["pp"] = Hash.new() init_program_options() diff --git a/sbin/submit b/sbin/submit index a749ea20..8b46db32 100755 --- a/sbin/submit +++ b/sbin/submit @@ -108,6 +108,7 @@ ARGV.each do |jobfile| raise 'Please configure SCHED_PORT' unless job['SCHED_PORT'] job.add_pp + job.add_monitors job.add_define_files if opt_auto_define_files # get job shell function -- 2.23.0

2 1

[PATCH v4 compass-ci 1/2] container/assign-account: answerback-email.rb
by Luan Shengde 21 Oct '20

21 Oct '20

add parameters for answerback-email -e,--email email_addr: add email address -s,--ssh-pubkey pub_key: add a ssh login pub_key -f,--raw-email mail_file: add email address[, pub_key] via email file why: easier for administrator to manully assign jumper account for user with: - user's email - user's email and a pub_key - user's email_file Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/answerback-email.rb | 173 ++++++++----------- 1 file changed, 74 insertions(+), 99 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index b5283ba..5a5234c 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -3,46 +3,18 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true -=begin - -repo_list: - all repos url list from upstream-repos.git - -API: -call graph: -read_mail_content -send_account_request - email_addr - get email address - email_message_id - get email message_id - check_email_available - check email available - pub_key_value - get pub key - account - send apply account request and return account info - build_message - build email message - send_mail - call send_mail to send mail whit build message - -the returned data for account_info like: -{ - "account" => "guest", - "passwd" => "Use pub_key to login", - "jumper_ip" => "10.10.10.10", - "jumper_port" => "10000" -} -=end +# xx --email xxx --login --ssh-pubkey xxx --raw-email email-file +# samba mount +# ssh logshn (huawei, ) (install pubkey / send password) require 'json' require 'mail' require 'set' -require_relative '../defconfig.rb' +require 'optparse' +require_relative '../defconfig' names = Set.new %w[ - JUMPER_IP + JUMPER_HOST JUMPER_PORT CRYSTAL_INTRANET SEND_MAIL_PORT @@ -50,27 +22,71 @@ names = Set.new %w[ defaults = relevant_defaults(names) -JUMPER_IP = defaults['JUMPER_IP'] -JUMPER_PORT = defaults['JUMPER_PORT'] -CRYSTAL_INTRANET = defaults['CRYSTAL_INTRANET'] -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] +JUMPER_HOST = defaults['JUMPER_HOST'] || '183.134.196.212' +JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 +CRYSTAL_INTRANET = defaults['CRYSTAL_INTRANET'] || localhost +SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 11312 + +apply_info = { + 'email_addr' => nil, + 'pub_key' => nil +} + +def init_info(email_file, apply_info) + mail_content = Mail.read(email_file) + + email_addr = mail_content.from[0] + pub_key = mail_content.part[1].body.decoded.gsub(/\r|\n/, '') if mail_content.part[1].filename == 'id_rsa.pub' + + apply_info['email_addr'] = email_addr + apply_info['pub_key'] = pub_key + + apply_info +end + +options = OptionParser.new do |opts| + opts.banner = "Usage: answerback-mail.rb [--email email] [--ssh-pubkey pub_key] [--raw-email email_file]\n" + opts.banner += " -e or -f is required\n" + opts.banner += ' -s is optional when use -e' + + opts.separator '' + opts.separator 'options:' + + opts.on('-e email_addr', '--email email_addr', 'appoint email address') do |email_addr| + apply_info['email_addr'] = email_addr + end + + opts.on('-s pub_key', '--ssh-pubkey pub_key', 'ssh pub_key to enable keyless login') do |pub_key| + apply_info['pub_key'] = pub_key + end + + opts.on('-f email_file', '--raw-email email_file', 'email file') do |email_file| + init_info(email_file, apply_info) + end + + opts.on_tail('-h', '--help', 'show this message') do + puts opts + exit + end +end -def build_message(email, message_id, infos) +options.parse!(ARGV) + +def build_message(email, acct_infos) message = <<~EMAIL_MESSAGE To: #{email} - Message-ID: #{message_id} Subject: jumper account is ready - Dear #{email} + Dear user: Thank you for joining us. You can use the following command to login the jumper server: login command: - ssh -p #{infos['jumper_port']} #{infos['account']}@#{infos['jumper_ip']} + ssh -p #{acct_infos['jumper_port']} #{acct_infos['account']}@#{acct_infos['jumper_ip']} - account passwd: - account_password: #{infos['passwd']} + account password: + #{acct_infos['passwd']} regards compass-ci @@ -79,67 +95,26 @@ def build_message(email, message_id, infos) return message end -def email_addr(mail_content) - msg = 'not an applying account email' - - raise msg unless mail_content.subject =~ /apply ssh account/i - - email = mail_content.from.join(',') - - return email -end - -# def check_email_available(mail_content, email) -# oos_list = File.read('/c/upstream-repos/repo_list').split(/\n/) -# url = mail_content.body.decoded.split(/\n/).find { |line| line =~ /https?:\/\// } -# base_url = url.split('/')[0,5].join('/') -# message = 'The url is not in upstream repo_list' -# -# raise message unless oos_list.include? base_url -# -# url_fdback = %x(curl #{url}) -# email_index = url_fdback.index email -# -# message = 'No commit info found from the url for the email' -# raise message unless email_index -# end - -def email_message_id(mail_content) - message_id = mail_content.message_id - return message_id -end - -def pub_key_value(mail_content) - pub_key = mail_content.body.decoded.split(/\n/).find { |line| line =~ /ssh-rsa/ } - return pub_key -end - def account_info(pub_key) - account_info_str = %x(curl -XGET '#{JUMPER_IP}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") - account_info = JSON.parse account_info_str - - return account_info + account_info_str = if pub_key.nil? + %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account') + else + %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") + end + JSON.parse account_info_str end -def send_account(mail_content) - email = email_addr(mail_content) - message_id = email_message_id(mail_content) - # check_email_available(mail_content, email) +def send_account(apply_info) + message = "No email address specified\n" + message += "use -e email_addr add a email address\n" + message += 'or use -f to add a mail file' + raise message if apply_info['email_addr'].nil? - pub_key = pub_key_value(mail_content) - acct_info = account_info(pub_key) + acct_info = account_info(apply_info['pub_key']) - message = build_message(email, message_id, acct_info) + message = build_message(apply_info['email_addr'], acct_info) %x(curl -XPOST '#{CRYSTAL_INTRANET}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end -def read_mail_content(mail_file) - mail_content = Mail.read(mail_file) - - return mail_content -end - -mail_file = ARGV[0] -mail_content = read_mail_content(mail_file) -send_account(mail_content) +send_account(apply_info) -- 2.23.0

2 3

[PATCH v3 compass-ci] /src/lib/web_backend.rb: fix "/compare_candidates" tbox_group regex bug
by Zhang Yuhang 21 Oct '20

21 Oct '20

Info: "/compare_canditates" error, due to fault regular expression. Detail: Miss a couple brackets, so will match /\d+$/. Error Flow: 1. like "xxx123" will be matched 2. index = "xxx123".index('--') || "xxx123".rindex('-') 3. index equal "nil" 4. r = r[0, nil] will raise an error "TypeError (no implicit conversion from nil to integer)" Solution: Fill a couple brackets, so will match "vm-abc-0abc-abc" or "vm-abc-0abc-012". Signed-off-by: Zhang Yuhang <zhangyuhang25(a)huawei.com> --- src/lib/web_backend.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 48cfadb..fda01ca 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -47,7 +47,7 @@ end def filter_tbox_group(es_result) result = Set.new es_result.each do |r| - if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-([a-zA-Z]+)|(\d+)$)/ + if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-(([a-zA-Z]+)|(\d+))$)/ index = r.index('--') || r.rindex('-') r = r[0, index] end -- 2.23.0

1 1

[PATCH v4 compass-ci 2/2] container/assign-account: answerback-email.rb
by Luan Shengde 21 Oct '20

21 Oct '20

add parameters for answerback-email -e,--email email_addr: add email address -s,--ssh-pubkey pub_key: add a ssh login pub_key -f,--raw-email mail_file: add email address[, pub_key] via email file why: easier for administrator to manully assign jumper account for user with the parameters Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/answerback-email.rb | 42 ++++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index 5a5234c..a3108cb 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -16,48 +16,48 @@ require_relative '../defconfig' names = Set.new %w[ JUMPER_HOST JUMPER_PORT - CRYSTAL_INTRANET - SEND_MAIL_PORT + SEND_MAIL_HOST + SEND_INTERNET_MAIL_PORT ] defaults = relevant_defaults(names) JUMPER_HOST = defaults['JUMPER_HOST'] || '183.134.196.212' JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 -CRYSTAL_INTRANET = defaults['CRYSTAL_INTRANET'] || localhost -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 11312 +SEND_MAIL_HOST = defaults['SEND_MAIL_HOST'] || localhost +SEND_MAIL_PORT = defaults['SEND_INTERNET_MAIL_PORT'] || 11312 apply_info = { - 'email_addr' => nil, - 'pub_key' => nil + 'my_email' => nil, + 'my_pub_key' => nil } def init_info(email_file, apply_info) mail_content = Mail.read(email_file) - email_addr = mail_content.from[0] - pub_key = mail_content.part[1].body.decoded.gsub(/\r|\n/, '') if mail_content.part[1].filename == 'id_rsa.pub' + my_email = mail_content.from[0] + my_pub_key = mail_content.part[1].body.decoded.gsub(/\r|\n/, '') if mail_content.part[1].filename == 'id_rsa.pub' - apply_info['email_addr'] = email_addr - apply_info['pub_key'] = pub_key + apply_info['my_email'] = my_email + apply_info['my_pub_key'] = my_pub_key apply_info end options = OptionParser.new do |opts| - opts.banner = "Usage: answerback-mail.rb [--email email] [--ssh-pubkey pub_key] [--raw-email email_file]\n" + opts.banner = "Usage: answerback-mail.rb [--email email] [--ssh-pubkey pub_key_file] [--raw-email email_file]\n" opts.banner += " -e or -f is required\n" opts.banner += ' -s is optional when use -e' opts.separator '' opts.separator 'options:' - opts.on('-e email_addr', '--email email_addr', 'appoint email address') do |email_addr| - apply_info['email_addr'] = email_addr + opts.on('-e email_address', '--email email_address', 'appoint email address') do |email_address| + apply_info['my_email'] = email_address end - opts.on('-s pub_key', '--ssh-pubkey pub_key', 'ssh pub_key to enable keyless login') do |pub_key| - apply_info['pub_key'] = pub_key + opts.on('-s pub_key_file', '--ssh-pubkey pub_key_file', 'ssh pub_key file, enable keyless login') do |pub_key_file| + apply_info['my_pub_key'] = File.read(pub_key_file) end opts.on('-f email_file', '--raw-email email_file', 'email file') do |email_file| @@ -106,15 +106,15 @@ end def send_account(apply_info) message = "No email address specified\n" - message += "use -e email_addr add a email address\n" - message += 'or use -f to add a mail file' - raise message if apply_info['email_addr'].nil? + message += "use -e email_address add a email address\n" + message += 'or use -f to add a email file' + raise message if apply_info['my_email'].nil? - acct_info = account_info(apply_info['pub_key']) + acct_info = account_info(apply_info['my_pub_key']) - message = build_message(apply_info['email_addr'], acct_info) + message = build_message(apply_info['my_email'], acct_info) - %x(curl -XPOST '#{CRYSTAL_INTRANET}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") + %x(curl -XPOST '#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end send_account(apply_info) -- 2.23.0

1 1

[PATCH v4 compass-ci 2/6] container/send-internet-mail: internet-smtp.rb
by Luan Shengde 21 Oct '20

21 Oct '20

setup smtp for email server Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- lib/internet-smtp.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 lib/internet-smtp.rb diff --git a/lib/internet-smtp.rb b/lib/internet-smtp.rb new file mode 100644 index 0000000..3f181ba --- /dev/null +++ b/lib/internet-smtp.rb @@ -0,0 +1,20 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'mail' + +def setup_smtp + smtp = { + address: 'smtp.qq.com', + port: 25, + domain: 'qq.com', + user_name: ENV['ROBOT_EMAIL_ADDRESS'], + password: ENV['ROBOT_EMAIL_PASSWORD'], + openssl_verify_mode: 'none', + enable_starttls_auto: true + } + + Mail.defaults { delivery_method :smtp, smtp } +end -- 2.23.0

2 2

[PATCH v2 compass-ci] /src/lib/web_backend.rb: fix "/compare_candidates" tbox_group regex bug
by Zhang Yuhang 21 Oct '20

21 Oct '20

info: "/compare_canditates" error, due to fault regular expression before: /^vm-.*-\d\w*-([a-zA-Z]+)|(\d+)$/ afeter: /^vm-.*-\d\w*-(([a-zA-Z]+)|(\d+))$/ before miss a couple brackets, so will match /\d+$/. It will cause an exception. Signed-off-by: Zhang Yuhang <zhangyuhang25(a)huawei.com> --- src/lib/web_backend.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 48cfadb..fda01ca 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -47,7 +47,7 @@ end def filter_tbox_group(es_result) result = Set.new es_result.each do |r| - if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-([a-zA-Z]+)|(\d+)$)/ + if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-(([a-zA-Z]+)|(\d+))$)/ index = r.index('--') || r.rindex('-') r = r[0, index] end -- 2.23.0

2 2

【Meeting Notice】compass-ci fluentd 日志处理系统 Time: 2020-10-22 18:30-20:30
by Meeting Book 21 Oct '20

21 Oct '20

1 0

【Meeting Notice】compass-ci fluentd 日志处理系统 Time: 2020-10-22 10:00-11:00
by Meeting Book 21 Oct '20

21 Oct '20

1 0

[PATCH v2 compass-ci] fix: sched: alter conditions when submitting cluster and normal jobs
by Ren Wen 21 Oct '20

21 Oct '20

[Why] I change the rules to judge the job is whether a cluster job. according to codes in '$LKP_SRC/lib/job.sh': ``` should_wait_cluster() ... [ "$cluster" = "cs-localhost" ] && return 1 ... ``` [Result] When the field 'cluster' values doesn't starts with 'cs-localhost', do the cluster job; normal job else. Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/lib/sched.cr | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index a4ddcec..a18bdee 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -139,22 +139,12 @@ class Sched return node_state end - # EXAMPLE: - # cluster_file: "cs-lkp-hsw-ep5" - # return: Hash(YAML::Any, YAML::Any) | Nil, 0 | <hosts_size> - # {"lkp-hsw-ep5" => {"roles" => ["server"], "macs" => ["ec:f4:bb:cb:7b:92"]}, - # "lkp-hsw-ep2" => {"roles" => ["client"], "macs" => ["ec:f4:bb:cb:54:92"]}}, 2 + # get cluster config using own lkp_src cluster file, + # a hash type will be returned def get_cluster_config(cluster_file, lkp_initrd_user, os_arch) lkp_src = Jobfile::Operate.prepare_lkp_tests(lkp_initrd_user, os_arch) cluster_file_path = Path.new(lkp_src, "cluster", cluster_file) - - if File.file?(cluster_file_path) - cluster_config = YAML.parse(File.read(cluster_file_path)).as_h - hosts_size = cluster_config.values.size - return cluster_config, hosts_size - end - - return nil, 0 + return YAML.parse(File.read(cluster_file_path)).as_h end def get_commit_date(job) @@ -177,16 +167,17 @@ class Sched job = Job.new(job_content, job_content["id"]?) job["commit_date"] = get_commit_date(job) + # it is not a cluster job if cluster field is empty or + # field's prefix is 'cs-localhost' cluster_file = job["cluster"] - if cluster_file != "" - cluster_config, hosts_size = get_cluster_config( - cluster_file, job.lkp_initrd_user, job.os_arch) - - return submit_cluster_job( - job, cluster_config.not_nil!) if hosts_size >= 2 + if cluster_file.empty? || cluster_file.starts_with?("cs-localhost") + return submit_single_job(job) + else + cluster_config = get_cluster_config(cluster_file, + job.lkp_initrd_user, + job.os_arch) + return submit_cluster_job(job, cluster_config) end - - return submit_single_job(job) rescue ex puts ex.inspect_with_backtrace return [{ -- 2.23.0

2 2