On Wed, Jan 20, 2021 at 04:53:17PM +0800, Wei Jihui wrote:

submit -i is for add other yamls content to job.yaml

[usage]

how about this: case 1: submit -i 1.yaml -i 2.yaml job.yaml priority: job > 2 > 1 case 2: xxx may be better understanding Thanks, Shenwei

1. submit -i 1.yaml -i 2.yaml job.yaml vars in job.yaml can overwrite 1.yaml and 2.yaml 2. submit -i 1.yaml job.yaml -i 2.yaml vars in job.yaml can only overwrite 1.yaml vars in 2.yaml can overwrite job.yaml and 1.yaml 3. submit -i 1.yaml job1.yaml -i 2.yaml job2.yaml -i 3.yaml vars in job1.yaml can only overwrite 1.yaml vars in job2.yaml can overwrite 1.yaml and 2.yaml vars in 2.yaml can overwrite job1.yaml and 1.yaml vars in 3.yaml can overwrite 1.yaml,job1.yaml,2.yaml,job2.yaml

Signed-off-by: Wei Jihui <weijihuiall@163.com> --- lib/job.rb | 17 +++++++++++++---- sbin/submit | 30 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/lib/job.rb b/lib/job.rb index 631ebfc9..640b25a5 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -215,6 +215,8 @@ class Job attr_accessor :overrides attr_accessor :defaults attr_accessor :cmdline_defaults + attr_accessor :default_yamls + attr_accessor :override_yamls

def initialize(job = {}, defaults = {}, overrides = {}, cmdline_defaults={}) @job = job @@ -222,6 +224,8 @@ class Job @overrides = overrides # from command line @cmdline_defaults = cmdline_defaults # from command line @available_programs = {} + @default_yamls = [] + @override_yamls = [] end

def source_file_symkey(file) @@ -254,8 +258,11 @@ class Job @overrides.merge!(hash['override']){ |_key, a, _b| a} hash.delete('override') end - hash.merge!(@overrides) - hash.merge!(fault_reproduction) if hash.has_key?("id") + + revise_hash(hash, load_include_yamls(@default_yamls), false) unless @default_yamls.empty? + revise_hash(hash, load_include_yamls(@override_yamls), true) unless @override_yamls.empty? + revise_hash(hash, @overrides, true) unless @overrides.empty? + @jobs.concat(multi_args(hash)) # return [hash] or [h1,h2] end rescue StandardError => e @@ -286,8 +293,10 @@ class Job @jobfile = jobfile end

- def fault_reproduction - wait = load_yaml("#{lkp_src}/jobs/fault_reproduction.yaml", {}) + def load_include_yamls(include_yamls) + include_hash = {} + include_hash[comment_to_symbol("#{include_yamls.join(' ')}")] = nil + include_hash.merge!(load_yaml_merge(include_yamls, {})) end

def multi_args(hash) diff --git a/sbin/submit b/sbin/submit index 39cdcf57..e8b17648 100755 --- a/sbin/submit +++ b/sbin/submit @@ -14,6 +14,7 @@ opt_set_key_value = {} opt_cmdline_defaults = {} opt_output_dir = nil opt_auto_define_files = false +opt_include_yamls = [] opt_monitor = false opt_monitor_query = {} opt_my_queue = false @@ -43,6 +44,10 @@ options = OptionParser.new do |opts| opt_auto_define_files = true end

+ opts.on('-i include.yaml', '--include include.yaml', 'include other job yamls') do |yaml| + opt_include_yamls << yaml + end + opts.on('-c', '--connect', 'auto connect to the host') do actions << 'connect' end @@ -66,6 +71,7 @@ options = OptionParser.new do |opts|

end

+ARGV_CLONE = ARGV.clone options.parse!(ARGV)

seen_job_yaml = false @@ -91,6 +97,19 @@ end job_ids = [] job_hash_list = []

+def prioritize_include_yaml(include_yamls, jobfile) + default_yamls, override_yamls = [], [] + jobfile_index = ARGV_CLONE.index(jobfile) + include_yamls.each do |yaml| + if ARGV_CLONE.index(yaml) < jobfile_index + default_yamls << yaml + else + override_yamls << yaml + end + end + return default_yamls, override_yamls +end + def find_jobfile(jobfile) return jobfile if File.exist?(jobfile)

@@ -103,14 +122,25 @@ def find_jobfile(jobfile) exit 1 end

+def find_jobfiles(jobfile_list) + search_jobfile_list = [] + jobfile_list.each do |jobfile| + search_jobfile_list << find_jobfile(jobfile) + end + return search_jobfile_list +end + submit_id = %x(uuidgen).chomp puts "submit_id=#{submit_id}"

ARGV.each do |jobfile| + default_yamls, override_yamls = prioritize_include_yaml(opt_include_yamls, jobfile) jobfile = find_jobfile(jobfile) jobs = Job2sh.new jobs.cmdline_defaults = opt_cmdline_defaults jobs.overrides = opt_set_key_value + jobs.default_yamls = find_jobfiles(default_yamls) + jobs.override_yamls = find_jobfiles(override_yamls) jobs.load(jobfile, true) || next jobs[:expand_params] = true jobs['testbox'] = opt_set_key_value['testbox'] if opt_set_key_value['testbox'] -- 2.23.0