submit -i is for add other yamls content to job.yaml
[usage] 1. submit -i 1.yaml -i 2.yaml job.yaml vars in job.yaml can overwrite 1.yaml and 2.yaml 2. submit -i 1.yaml job.yaml -i 2.yaml vars in job.yaml can only overwrite 1.yaml vars in 2.yaml can overwrite job.yaml and 1.yaml 3. submit -i 1.yaml job1.yaml -i 2.yaml job2.yaml -i 3.yaml vars in job1.yaml can only overwrite 1.yaml vars in job2.yaml can overwrite 1.yaml and 2.yaml vars in 2.yaml can overwrite job1.yaml and 1.yaml vars in 3.yaml can overwrite 1.yaml,job1.yaml,2.yaml,job2.yaml
Signed-off-by: Wei Jihui weijihuiall@163.com --- lib/job.rb | 17 +++++++++++++---- sbin/submit | 30 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-)
diff --git a/lib/job.rb b/lib/job.rb index 631ebfc9..640b25a5 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -215,6 +215,8 @@ class Job attr_accessor :overrides attr_accessor :defaults attr_accessor :cmdline_defaults + attr_accessor :default_yamls + attr_accessor :override_yamls
def initialize(job = {}, defaults = {}, overrides = {}, cmdline_defaults={}) @job = job @@ -222,6 +224,8 @@ class Job @overrides = overrides # from command line @cmdline_defaults = cmdline_defaults # from command line @available_programs = {} + @default_yamls = [] + @override_yamls = [] end
def source_file_symkey(file) @@ -254,8 +258,11 @@ class Job @overrides.merge!(hash['override']){ |_key, a, _b| a} hash.delete('override') end - hash.merge!(@overrides) - hash.merge!(fault_reproduction) if hash.has_key?("id") + + revise_hash(hash, load_include_yamls(@default_yamls), false) unless @default_yamls.empty? + revise_hash(hash, load_include_yamls(@override_yamls), true) unless @override_yamls.empty? + revise_hash(hash, @overrides, true) unless @overrides.empty? + @jobs.concat(multi_args(hash)) # return [hash] or [h1,h2] end rescue StandardError => e @@ -286,8 +293,10 @@ class Job @jobfile = jobfile end
- def fault_reproduction - wait = load_yaml("#{lkp_src}/jobs/fault_reproduction.yaml", {}) + def load_include_yamls(include_yamls) + include_hash = {} + include_hash[comment_to_symbol("#{include_yamls.join(' ')}")] = nil + include_hash.merge!(load_yaml_merge(include_yamls, {})) end
def multi_args(hash) diff --git a/sbin/submit b/sbin/submit index 39cdcf57..e8b17648 100755 --- a/sbin/submit +++ b/sbin/submit @@ -14,6 +14,7 @@ opt_set_key_value = {} opt_cmdline_defaults = {} opt_output_dir = nil opt_auto_define_files = false +opt_include_yamls = [] opt_monitor = false opt_monitor_query = {} opt_my_queue = false @@ -43,6 +44,10 @@ options = OptionParser.new do |opts| opt_auto_define_files = true end
+ opts.on('-i include.yaml', '--include include.yaml', 'include other job yamls') do |yaml| + opt_include_yamls << yaml + end + opts.on('-c', '--connect', 'auto connect to the host') do actions << 'connect' end @@ -66,6 +71,7 @@ options = OptionParser.new do |opts|
end
+ARGV_CLONE = ARGV.clone options.parse!(ARGV)
seen_job_yaml = false @@ -91,6 +97,19 @@ end job_ids = [] job_hash_list = []
+def prioritize_include_yaml(include_yamls, jobfile) + default_yamls, override_yamls = [], [] + jobfile_index = ARGV_CLONE.index(jobfile) + include_yamls.each do |yaml| + if ARGV_CLONE.index(yaml) < jobfile_index + default_yamls << yaml + else + override_yamls << yaml + end + end + return default_yamls, override_yamls +end + def find_jobfile(jobfile) return jobfile if File.exist?(jobfile)
@@ -103,14 +122,25 @@ def find_jobfile(jobfile) exit 1 end
+def find_jobfiles(jobfile_list) + search_jobfile_list = [] + jobfile_list.each do |jobfile| + search_jobfile_list << find_jobfile(jobfile) + end + return search_jobfile_list +end + submit_id = %x(uuidgen).chomp puts "submit_id=#{submit_id}"
ARGV.each do |jobfile| + default_yamls, override_yamls = prioritize_include_yaml(opt_include_yamls, jobfile) jobfile = find_jobfile(jobfile) jobs = Job2sh.new jobs.cmdline_defaults = opt_cmdline_defaults jobs.overrides = opt_set_key_value + jobs.default_yamls = find_jobfiles(default_yamls) + jobs.override_yamls = find_jobfiles(override_yamls) jobs.load(jobfile, true) || next jobs[:expand_params] = true jobs['testbox'] = opt_set_key_value['testbox'] if opt_set_key_value['testbox']