- Compass-ci - mailweb.openeuler.org

[PATCH v4 compass-ci] src/lib/job.cr: optimize the get_pkg_common_dir function
by Li Ping 10 Mar '21

10 Mar '21

[why] 1. rpm has no much to do with os_mount. add a judgment 2. optimize the code style Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/job.cr | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0b4bee4..25d8462 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -258,23 +258,28 @@ class Job end def get_pkg_common_dir - pkg_style = nil + tmp_style = nil ["cci-makepkg", "cci-depends", "build-pkg", "rpmbuild-pkg"].each do |item| - pkg_style = @hash[item]? - break if pkg_style + tmp_style = @hash[item]? + break if tmp_style end - return nil unless pkg_style + return nil unless tmp_style + pkg_style = Hash(String, JSON::Any).new {|h, k| h[k] = JSON::Any.new(nil)} + pkg_style.merge!(tmp.as_h? || Hash(String, JSON::Any).new) - pkg_style = JSON.parse("{}") if pkg_style == nil + tmp_os = pkg_style["os"].as_s? || "#{os}" + tmp_os_arch = pkg_style["os_arch"].as_s? || "#{os_arch}" + tmp_os_version = pkg_style["os_version"].as_s? || "#{os_version}" - tmp_os = pkg_style["os"]? == nil ? "#{os}" : pkg_style["os"] - tmp_os_arch = pkg_style["os_arch"]? == nil ? "#{os_arch}" : pkg_style["os_arch"] - tmp_os_version = pkg_style["os_version"]? == nil ? "#{os_version}" : pkg_style["os_version"] + mount_type = pkg_style["os_mount"].as_s? || "#{os_mount}" + # same usage for client + mount_type = "nfs" if mount_type == "cifs" - tmp_os_mount = pkg_style["os_mount"]? == nil ? "#{os_mount}" : pkg_style["os_mount"] - mount_type = tmp_os_mount == "cifs" ? "nfs" : tmp_os_mount.dup - - common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + if @hash["rpmbuild-pkg"]? + common_dir = "#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + else + common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + end return common_dir end -- 2.23.0

1 1

[PATCH v3 compass-ci 2/2] lib/dump-stat.rb: refactor lkp-tests/sbin/dump-stat
by Lu Weitao 10 Mar '21

10 Mar '21

[Why] support handle hash from lkp-tests/stats/$script.rb Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/dump_stat.rb | 194 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 lib/dump_stat.rb diff --git a/lib/dump_stat.rb b/lib/dump_stat.rb new file mode 100644 index 0000000..1c0695e --- /dev/null +++ b/lib/dump_stat.rb @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: GPL-2.0-only + +LKP_SRC ||= ENV['LKP_SRC'] || '/c/lkp-tests' + +require "#{LKP_SRC}/lib/statistics" +require "#{LKP_SRC}/lib/bounds" +require "#{LKP_SRC}/lib/yaml" +require "#{LKP_SRC}/lib/job" +require "#{LKP_SRC}/lib/string_ext" +require "#{LKP_SRC}/lib/log" +require 'set' + +UNSTRUCTURED_MONITORS = %w[ftrace].to_set + +def warn_stat(msg, monitor) + log_warn msg + log_warn "check #{RESULT_ROOT}/#{monitor}" +end + +# dump stat which input by lkp-tests/stats/$script return +# input: +# eg-1: +# { +# "pgfree" => [275506, 280018], +# ... +# } +# eg-2: +# { +# "iperf.tcp.sender.bps" => 34804801216.197174, +# "iperf.tcp.receiver.bps" => "34804762215.18231" +# } +module DumpStat + def self.dump_stat(monitor, stat_result) + @result = {} + @invalid_records = [] + @record_index = 0 + @monitor = monitor + + stat_result.each do |key, value| + key = key.resolve_invalid_bytes + next if key[0] == '#' + next if value.empty? || value == 0 + next if monitor =~ /^(dmesg|kmsg)$/ && key =~ /^(message|pattern):/ + + if key =~ /[ \t]/ + @invalid_records.push @record_index + warn_stat "whitespace in stats name: #{key}", @monitor + return nil # for exit current stats/script dump-stat + end + next if assign_log_message(key, value) + + k = @monitor + '.' + key + @result[k] ||= [] + fill_zero(k) + if value.is_a?(String) + value = check_string_value(k, value, @monitor) + next unless value + return nil unless number?(value, @invalid_records) + + value = value.index('.') ? value.to_f : value.to_i + elsif value.is_a?(Array) + (0..value.size - 1).each do |i| + next unless value[i].is_a?(String) + + value[i] = check_string_value(k, value[i], @monitor) + next unless value[i] + return nil unless number?(value[i], @invalid_records) + + value[i] = value[i].index('.') ? value[i].to_f : value[i].to_i + valid_stats_verification(k, value[i]) + end + @result[k] = value + next + end + valid_stats_verification(k, value) + @result[k].push value + end + return nil if @result.empty? + + remove_zero_stats + delete_invalid_number(@result, @invalid_records, @monitor) + cols_verifation + return nil unless useful_result?(@result) + + save_json(@result, "#{RESULT_ROOT}/#{(a)monitor}.json", @result.size * @min_cols > 1000) + end + + # keep message | log line which key end with .message|.log + def self.assign_log_message(key, value) + if key.end_with?('.message', '.log') + k = @monitor + '.' + key + @result[k] = value + return true + end + + false + end + + def self.fill_zero(key) + size = @result[key].size + if @record_index < size + @record_index = size + elsif (@record_index - size).positive? + # fill 0 for missing values + @result[key].concat([0] * (@record_index - size)) + end + end + + def self.valid_stats_verification(key, value) + return nil if valid_stats_range? key, value + + @invalid_records.push @record_index + puts "outside valid range: #{value} in #{key} #{RESULT_ROOT}" + end + + def self.remove_zero_stats + @max_cols = 0 + @min_cols = Float::INFINITY + @min_cols_stat = '' + @max_cols_stat = '' + zero_stats = [] + @result.each do |key, val| + next if key.end_with?('.message', '.log') + + if @max_cols < val.size + @max_cols = val.size + @max_cols_stat = key + end + if @min_cols > val.size + @min_cols = val.size + @min_cols_stat = key + end + next if val[0] != 0 + next if val[-1] != 0 + next if val.sum != 0 + + zero_stats << key + end + zero_stats.each { |x| @result.delete x } + end + + def self.cols_verifation + return nil unless @min_cols < @max_cols && !UNSTRUCTURED_MONITORS.include?(@monitor) + + if @min_cols == @max_cols - 1 + @result.each { |_k, y| y.pop if y.size == @max_cols } + puts "Last record seems incomplete. Truncated #{RESULT_ROOT}/#{(a)monitor}.json" + else + warn_stat "Not a matrix: value size is different - #{@min_cols_stat}: #{@min_cols} != #{@max_cols_stat}: #{@max_cols}: #{RESULT_ROOT}/#{(a)monitor}.json", @monitor + end + end +end + +def check_string_value(key, value, monitor) + # value terminator is expected. If not, throw out an error warning. + warn_stat "no line terminator in stats value: #{value}", monitor if value.chomp!.nil? + + value.strip! + if value.empty? + warn_stat "empty stat value of #{key}", monitor + return nil + end + + return value +end + +# only number is valid +def number?(value, invalid_records) + unless value.numeric? + invalid_records.push record_index + warn_stat "invalid stats value: #{value}", monitor + return nil + end + + true +end + +def delete_invalid_number(result, invalid_records, monitor) + return nil if monitor == 'ftrace' + + invalid_records.reverse_each do |index| + result.each do |_k, value| + value.delete_at index + end + end +end + +def useful_result?(result) + return nil if result.empty? + return nil if result.values[0].size.zero? + return nil if result.values[-1].size.zero? + + true +end -- 2.23.0

1 0

[PATCH v4 lkp-tests 2/2] sbin/makepkg: fix whether to change domain of submodules
by Liu Shaofei 09 Mar '21

09 Mar '21

the param "--rewritegitserver" indicates that all of submodules will be changed domian name, and downloaded from Z9/Crystal server. Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- sbin/makepkg | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/sbin/makepkg b/sbin/makepkg index 7a92ea71b..cb0516413 100755 --- a/sbin/makepkg +++ b/sbin/makepkg @@ -89,6 +89,7 @@ SIGNPKG='' SPLITPKG=0 SOURCEONLY=0 VERIFYSOURCE=0 +REWRITEGITSERVER=0 # Forces the pkgver of the current PKGBUILD. Used by the fakeroot call # when dealing with svn/cvs/etc PKGBUILDs. @@ -587,6 +588,16 @@ download_git() { fi } +# modify submodules address to local server address. +# (( REWRITEGITSERVER )) is true, replace submodules domain, and download it from local end. +modify_gitmodules() { + (( REWRITEGITSERVER )) || return + + if [[ -f ".gitmodules" ]] && [[ -n ${GIT_SERVER} ]]; then + git config --system url."git://${GIT_SERVER}/".insteadOf https:// + fi +} + extract_git() { local netfile=$1 @@ -623,10 +634,7 @@ extract_git() { cd_safe "${dir##*/}" - # modify submodules address to local server address. - if [[ -f ".gitmodules" ]] && [[ -n ${LKP_SERVER} ]]; then - echo -e "[url \"git://${LKP_SERVER}/\"]\n\tinsteadOf = https://" >> /etc/gitconfig - fi + modify_gitmodules local ref=origin/HEAD if [[ -n $fragment ]]; then @@ -3421,6 +3429,7 @@ usage() { printf -- "$(gettext " --skipinteg Do not perform any verification checks on source files")\n" printf -- "$(gettext " --skippgpcheck Do not verify source files with PGP signatures")\n" printf -- "$(gettext " --verifysource Download source files (if needed) and perform integrity checks")\n" + printf -- "$(gettext " --rewritegitserver Change domain name when downloading submodules repository")\n" echo printf -- "$(gettext "These options can be passed to %s:")\n" "pacman" echo @@ -3462,7 +3471,7 @@ OPT_LONG=('allsource' 'check' 'clean' 'cleanbuild' 'config:' 'force' 'geninteg' 'help' 'holdver' 'ignorearch' 'install' 'key:' 'log' 'noarchive' 'nobuild' 'nocolor' 'nocheck' 'nodeps' 'noextract' 'noprepare' 'nosign' 'pkg:' 'repackage' 'rmdeps' 'sign' 'skipchecksums' 'skipinteg' 'skippgpcheck' 'source' 'syncdeps' - 'verifysource' 'version') + 'verifysource' 'version' 'rewritegitserver') # Pacman Options OPT_LONG+=('asdeps' 'noconfirm' 'needed' 'noprogressbar') @@ -3514,6 +3523,7 @@ while true; do -s|--syncdeps) DEP_BIN=1 ;; -S|--source) SOURCEONLY=1 ;; --verifysource) VERIFYSOURCE=1 ;; + --rewritegitserver) REWRITEGITSERVER=1 ;; -h|--help) usage; exit 0 ;; # E_OK -V|--version) version; exit 0 ;; # E_OK -- 2.23.0

1 0

[PATCH v4 lkp-tests 1/2] tests/build-pkg: download submodules from local end
by Liu Shaofei 09 Mar '21

09 Mar '21

[why] When building software package by build-pkg, the source codes repository contains submodules. By default, the submodules can be downloaded from remote end during building packages. [how] By using --rewritegitserver param, submodules can be downloaded from Z9/Crystal. Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- tests/build-pkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build-pkg b/tests/build-pkg index 6a083a9b3..439328651 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -93,7 +93,7 @@ create_cgz_name() cgz_path="$PKG_MNT/${pack_to}/${pkgname}" cgz_name="${cgz_path}/${upstream_commit}.cgz" - pkg_args="-A --check --skippgpcheck" + pkg_args="-A --check --skippgpcheck --rewritegitserver" [ "${os}" == "archlinux" ] && pkg_args="${pkg_args} -s --needed --noconfirm" [ -n "$git_tag" ] && { -- 2.23.0

1 0

[PATCH compass-ci] lib/git_mirror.rb: don't count new ref if it is not new
by Li Yuanchao 09 Mar '21

09 Mar '21

When repos are added, new_refs_count will increase. But many repos are not updated that day, that cause new_refs_count increase much more than real new refs number. Now only count those really new. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 1fa8c9e..f711747 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -203,7 +203,7 @@ class MirrorMain feedback_info[:new_refs] = new_refs send_message(feedback_info) - new_refs_log(git_repo, new_refs[:heads].length) + new_refs_log(git_repo, new_refs[:heads].length) if last_commit_new?(git_repo) end def do_push(fork_key) @@ -336,7 +336,7 @@ class MirrorMain def update_fork_stat(git_repo, possible_new_refs) update_stat_fetch(git_repo) - update_stat_new_refs(git_repo) if possible_new_refs + update_stat_new_refs(git_repo) if possible_new_refs && last_commit_new?(git_repo) es_repo_update(git_repo) end end @@ -530,4 +530,11 @@ class MirrorMain nr_new_branch: nr_new_branch }.to_json) end + + def last_commit_new?(git_repo) + inactive_time = %x(git -C /srv/git/#{@git_info[git_repo]['belong']}/#{git_repo}.git log --pretty=format:"%cr" -1) + return false if inactive_time =~ /(day|week|month|year)/ + + return true + end end -- 2.23.0

1 0

[PATCH compass-ci] lib/lifecycle: persistent restart queue
by Wu Zhende 09 Mar '21

09 Mar '21

After the service restarted, the queue information is not lost. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/lifecycle.cr | 2 +- src/lib/mq.cr | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 16830d0..3dab6e8 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -311,7 +311,7 @@ class Lifecycle return if Time.local < deadline mq_queue = get_machine_reboot_queue(testbox) - @mq.pushlish_confirm(mq_queue, machine.to_json) + @mq.pushlish_confirm(mq_queue, machine.to_json, durable: true) machine["state"] = "rebooting_queue" machine["time"] = Time.local.to_s("%Y-%m-%dT%H:%M:%S+0800") diff --git a/src/lib/mq.cr b/src/lib/mq.cr index 011d732..87c67e9 100644 --- a/src/lib/mq.cr +++ b/src/lib/mq.cr @@ -3,6 +3,7 @@ require "singleton" require "amqp-client" +require "amq-protocol" class MQClient getter ch : AMQP::Client::Channel @@ -19,14 +20,22 @@ class MQClient Singleton::Of(self).instance end - def pushlish_confirm(queue, msg) - q = @ch.queue(queue) - q.publish_confirm msg + def pushlish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + q = @ch.queue(queue, passive, durable, exclusive, auto_delete) + if durable + q.publish_confirm(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) + else + q.publish_confirm msg + end end - def pushlish(queue, msg) - q = @ch.queue(queue) - q.publish msg + def pushlish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + q = @ch.queue(queue, passive, durable, exclusive, auto_delete) + if durable + q.publish(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) + else + q.publish(msg) + end end def get(queue) -- 2.23.0

1 0

[PATCH v3 lkp-tests 1/2] jobs/iso2rootfs.yaml: change some field to secrets field.
by Li Ping 09 Mar '21

09 Mar '21

[why] the related fields when submit test.yaml should be add to secrets field. submit_name: submit_email: submit_token: and the required fields when submit test.yaml not have much to do with the iso2rootfs be changed to level-1 parameter. Signed-off-by: Li Ping <1477412247(a)qq.com> --- jobs/iso2rootfs.yaml | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 55c4345fa..5df2cdf61 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -51,23 +51,24 @@ iso2rootfs: # http://1.1.1.1//dailybuilds/1970-01-01-00-00-00/openEuler-20.03-LTS-aarch64… dailybuild_iso_url_file: - ################# - # submit test yaml related fields - ################# +################# +# submit test yaml related fields +################# - # three required fields when submit test yaml - submit_name: +# three required fields when submit test yaml +secrets: + submit_name: submit_email: submit_token: - # submit target tests yaml - ## 1. You can add as many jobs as you like. - ## 2. The following three fields is required for every test job. - test1_yaml: - test1_os_mount: - test1_testbox: - ## the following three fields is required when your job.yaml and - ## script for this test are from the internet. - test1_git_url: - test1_git_yaml: - test1_git_script: +# submit target tests yaml +## 1. You can add as many jobs as you like. +## 2. The following three fields is required for every test job. +test1_yaml: +test1_os_mount: +test1_testbox: +## the following three fields is required when your job.yaml and +## script for this test are from the internet. +test1_git_url: +test1_git_yaml: +test1_git_script: -- 2.23.0

2 2

[PATCH v2 lkp-tests 2/2] tests/rpmbuild-pkg: delete os_mount field in sync_dest dir.
by Li Ping 09 Mar '21

09 Mar '21

[why] rpm has no much to do with os_mount Signed-off-by: Li Ping <1477412247(a)qq.com> --- tests/rpmbuild-pkg | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index 38de16e66..607bb772d 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -6,8 +6,7 @@ [ -n "$rpm_repo" ] || die "rpm_repo is empty" package_name=${rpm_repo##*/} -[[ "$os_mount" = "cifs" ]] && os_mount="nfs" -sync_dest="/initrd/rpmbuild-pkg/${os_mount}/${os}/${os_arch}/${os_version}/${package_name}" +sync_dest="/initrd/rpmbuild-pkg/${os}/${os_arch}/${os_version}/${package_name}" init_workspace() { @@ -25,7 +24,7 @@ download_rpm_repo() { local filelist=$(curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-ls-files", "."]}') - for pkgfile in "${filelist[@]}" + for pkgfile in ${filelist[@]} do local dir="SOURCES" echo $pkgfile | egrep "\.spec$" && dir="SPECS" -- 2.23.0

1 0

[PATCH v2 lkp-tests 1/2] sbin/submit: optimize submit -r
by Li Ping 09 Mar '21

09 Mar '21

[why] submit -m -r iperf.yaml submit /home/liping/lkp-tests/jobs/iperf.yaml, got job id=z9.1183160 submit /home/liping/lkp-tests/jobs/iperf.yaml, got job id=z9.1183161 query=>{"job_id":["z9.1183160","z9.1183161"]} connect to ws://172.168.131.2:11310/filter {"level_num":1,"level":"INFO","time":"2021-03-05T11:31:36.649+0800","job_id":"z9.1183161","message":"","job_state":"submit","result_root":"/srv/result/iperf/2021-03-05/vm-2p8g/openeuler-20.03-aarch64/udp-300/z9.1183161","status_code":200,"method":"POST","resource":"/submit_job","elapsed":"24.68ms"} {"level_num":1,"level":"INFO","time":"2021-03-05T11:31:39.699+0800","job_id":"z9.1183160","result_root":"/srv/result/iperf/2021-03-05/vm-2p8g/openeuler-20.03-aarch64/tcp-300/z9.1183160","job_state":"set result root","status_code":200,"method":"GET","resource":"/boot.ipxe/mac/0a-9f-90-36-ae-6b","elapsed":"8.75ms"} {"level_num":1,"level":"INFO","time":"2021-03-05T11:31:39.735+0800","job_id":"z9.1183160","job_state":"boot","status_code":200,"method":"GET","resource":"/boot.ipxe/mac/0a-9f-90-36-ae-6b","elapsed":"44.08ms"} ==> only get one result [how] get result_root directly from the msg not from fluentd data Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/monitor.rb | 36 ++++++++++-------------------------- sbin/monitor | 8 ++++++++ sbin/submit | 4 +++- 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/lib/monitor.rb b/lib/monitor.rb index 78582be6a..24191fa8d 100755 --- a/lib/monitor.rb +++ b/lib/monitor.rb @@ -11,7 +11,7 @@ require 'eventmachine' require 'json' class Monitor - attr_accessor :monitor_url, :query, :overrides, :action, :job, :result, :stop_query + attr_accessor :monitor_url, :query, :overrides, :action, :job, :result, :result_roots, :stop_query def initialize(monitor_url = '', query = {}, action = {}) @monitor_url = monitor_url @@ -25,9 +25,9 @@ class Monitor @exit_status_code = 0 @defaults = {} @result = [] + @result_roots = [] @stop_query = {} @reason = nil - @result_root = nil end def load_default @@ -95,29 +95,13 @@ class Monitor exec cmd end - def set_result_root(data) - return unless @action['mirror_result'] - return unless data['log'] - - data = JSON.parse(data['log']) - return unless data['result_root'] - - @result_root = data['result_root'] - end - - def lftp_mirror - @result_root.delete_prefix!('/srv') - srv_http_host = job['SRV_HTTP_HOST'] || 'api.compass-ci.openeuler.org' - srv_http_port = job['SRV_HTTP_PORT'] || '11300' - url = "http://#{srv_http_host}:#{srv_http_port}#{@result_root}" - system "lftp -c mirror #{url} >/dev/null 2>&1" - end - - def mirror_result(data) - if @result_root - lftp_mirror - else - set_result_root(data) + def mirror_result + @result_roots.each do |res| + res.to_s.delete_prefix!('/srv') + srv_http_host = job['SRV_HTTP_HOST'] || 'api.compass-ci.openeuler.org' + srv_http_port = job['SRV_HTTP_PORT'] || '11300' + url = "http://#{srv_http_host}:#{srv_http_port}#{res}" + system "lftp -c mirror #{url} >/dev/null 2>&1" end end @@ -163,7 +147,7 @@ class Monitor output(data) connect(data, ws) - mirror_result(data) + mirror_result stop(data, ws) if @action['stop'] end diff --git a/sbin/monitor b/sbin/monitor index def47bf6a..a2e677fc0 100755 --- a/sbin/monitor +++ b/sbin/monitor @@ -12,6 +12,7 @@ require 'yaml' opt_set_key_value = {} opt_action = {'output' => true} opt_job = {} +opt_results = [] opt_monitor_url = nil opt_timeout = 0 opt_stop_query = {} @@ -35,6 +36,12 @@ options = OptionParser.new do |opts| opt_action[action] = true end end + + opts.on('-r result_roots', '--results results', 'set monitor result_roots') do |result_roots| + result_roots.split(',').each do |res| + opt_results << res + end + end opts.separator ' action: output, output log' opts.separator ' action: connect, connect to the host when get ip' opts.separator ' action: stop, exit monitor client when get log' @@ -79,6 +86,7 @@ monitor = Monitor.new monitor.monitor_url = opt_monitor_url if opt_monitor_url monitor.overrides = opt_set_key_value monitor.action = opt_action +monitor.result_roots = opt_results monitor.job = opt_job monitor.stop_query = opt_stop_query diff --git a/sbin/submit b/sbin/submit index e5e41828c..c2af7254d 100755 --- a/sbin/submit +++ b/sbin/submit @@ -19,6 +19,7 @@ opt_monitor = false opt_monitor_query = {} opt_my_queue = false actions = ['output', 'stop'] +result_roots = [] options = OptionParser.new do |opts| opts.banner = 'Usage: submit [options] job1.yaml job2.yaml ...' @@ -199,6 +200,7 @@ ARGV.each do |jobfile| messages = scheduler_client.submit_job(job_json) JSON.parse(messages).each do |msg| if msg['message'].empty? + result_roots << msg['result_root'] job_ids << msg['job_id'].to_s puts("submit #{jobfile}, got job id=#{msg['job_id']}") else @@ -212,6 +214,6 @@ end if opt_monitor job_hash_list[0].delete('define_files') opt_monitor_query.merge!({'job_id' => job_ids}) - cmd = "#{LKP_SRC}/sbin/monitor -f '#{opt_monitor_query.to_json}' -j '#{job_hash_list[0].to_json}' -a #{actions.join(',')} -s 'job_state: extract_finished'" + cmd = "#{LKP_SRC}/sbin/monitor -f '#{opt_monitor_query.to_json}' -j '#{job_hash_list[0].to_json}' -a #{actions.join(',')} -r #{result_roots.join(',')} -s 'job_state: extract_finished'" exec cmd end -- 2.23.0

1 0

Re: [PATCH lkp-tests 1/2] sbin/makepkg: fix whether to change domain of submodules
by Wu Fengguang 09 Mar '21

09 Mar '21

On Wed, Feb 10, 2021 at 02:36:57PM +0800, Liu Shaofei wrote: >the param "--norevisedomain" indicates that all of submodules will not >be changed domian name, and downloaded from remote end. spell: domian => domain >Signed-off-by: Liu Shaofei <370072077(a)qq.com> >--- > sbin/makepkg | 19 ++++++++++++++----- > 1 file changed, 14 insertions(+), 5 deletions(-) > >diff --git a/sbin/makepkg b/sbin/makepkg >index 7a92ea71b..41e19a513 100755 >--- a/sbin/makepkg >+++ b/sbin/makepkg >@@ -89,6 +89,7 @@ SIGNPKG='' > SPLITPKG=0 > SOURCEONLY=0 > VERIFYSOURCE=0 >+NOREVISEDOMAIN=0 You are adding an option to makepkg. To keep compatibility, if the option is not given, it should do nothing. So I suggest change it to REWRITEGITSERVER=0 And use option name rewritegitserver Thanks, Fengguang > # Forces the pkgver of the current PKGBUILD. Used by the fakeroot call > # when dealing with svn/cvs/etc PKGBUILDs. >@@ -587,6 +588,16 @@ download_git() { > fi > } > >+# modify submodules address to local server address. >+# (( NOREVISEDOMAIN )) is true, not replace submodules domain, and download it from remote end. >+modify_gitmodules() { >+ (( NOREVISEDOMAIN )) && return >+ >+ if [[ -f ".gitmodules" ]] && [[ -n ${GIT_SERVER} ]]; then >+ git config --system url."git://${GIT_SERVER}/".insteadOf https:// >+ fi >+} >+ > extract_git() { > local netfile=$1 > >@@ -623,10 +634,7 @@ extract_git() { > > cd_safe "${dir##*/}" > >- # modify submodules address to local server address. >- if [[ -f ".gitmodules" ]] && [[ -n ${LKP_SERVER} ]]; then >- echo -e "[url \"git://${LKP_SERVER}/\"]\n\tinsteadOf = https://" >> /etc/gitconfig >- fi >+ modify_gitmodules > > local ref=origin/HEAD > if [[ -n $fragment ]]; then >@@ -3462,7 +3470,7 @@ OPT_LONG=('allsource' 'check' 'clean' 'cleanbuild' 'config:' 'force' 'geninteg' > 'help' 'holdver' 'ignorearch' 'install' 'key:' 'log' 'noarchive' 'nobuild' > 'nocolor' 'nocheck' 'nodeps' 'noextract' 'noprepare' 'nosign' 'pkg:' 'repackage' > 'rmdeps' 'sign' 'skipchecksums' 'skipinteg' 'skippgpcheck' 'source' 'syncdeps' >- 'verifysource' 'version') >+ 'verifysource' 'version' 'norevisedomain') > > # Pacman Options > OPT_LONG+=('asdeps' 'noconfirm' 'needed' 'noprogressbar') >@@ -3514,6 +3522,7 @@ while true; do > -s|--syncdeps) DEP_BIN=1 ;; > -S|--source) SOURCEONLY=1 ;; > --verifysource) VERIFYSOURCE=1 ;; >+ --norevisedomain) NOREVISEDOMAIN=1 ;; > > -h|--help) usage; exit 0 ;; # E_OK > -V|--version) version; exit 0 ;; # E_OK >-- >2.23.0 >

2 1