[PATCH v3 lkp-tests] sbin/submit: add -f option for uploading libvirt config
An option for 'submit': -f, --upload-file FILE upload an XML template of libvirt FILE's extension should be '.xml'. Example: case1: submit -f existent_file.xml => existent_file/<md5sum>.xml case2: submit -f nonexistent_file.xml => Find no file: nonexistent_file.xml Signed-off-by: Ren Wen <15991987063@163.com> --- lib/job.rb | 11 +++++++++++ sbin/submit | 31 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/lib/job.rb b/lib/job.rb index 631ebfc98..812f8c299 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -16,6 +16,17 @@ require 'set' require 'pp' require 'English' +# copied from compass-ci/container/defconfig.rb +def cci_defaults + hash = {} + Dir.glob(['/etc/compass-ci/defaults/*.yaml', + '/etc/compass-ci/accounts/*.yaml', + "#{ENV['HOME']}/.config/compass-ci/defaults/*.yaml"]).each do |file| + hash.update YAML.load_file(file) || {} + end + hash +end + def restore(ah, copy) if ah.class == Hash ah.clear.merge!(copy) diff --git a/sbin/submit b/sbin/submit index 39cdcf576..1d56369ad 100755 --- a/sbin/submit +++ b/sbin/submit @@ -17,6 +17,7 @@ opt_auto_define_files = false opt_monitor = false opt_monitor_query = {} opt_my_queue = false +opt_upload_files = [] actions = ['output', 'stop'] options = OptionParser.new do |opts| @@ -64,10 +65,40 @@ options = OptionParser.new do |opts| opt_my_queue = true end + opts.on('-f FILE', '--upload-file FILE', "upload an XML template of libvirt. File: *.xml") do |file| + unless File.exist? file + puts "Find no file: #{file}" + exit 1 + end + + opt_upload_files << file + end + end options.parse!(ARGV) +def upload_configs(files) + defaults = cci_defaults() + upload_host = defaults['LKP_SERVER'] + upload_port = defaults['UPLOAD_LIBVIRT_PORT'] || 3081 + + files.each do |file| + response = `curl -sSf -T #{file} http://#{upload_host}:#{upload_port}/` + if $? == 0 + puts response + else + # TODO: how to deal with uploaded file when one failing + puts response + exit 1 + end + end + + exit 0 +end + +upload_configs(opt_upload_files) unless opt_upload_files.empty? + seen_job_yaml = false ARGV.delete_if do |arg| if arg.index '=' -- 2.23.0
Can we auto test-upload the libvirt xml file included in the job? A separate upload looks not user friendly and error prone. Thanks, Fengguang On Thu, Jan 21, 2021 at 11:29:19AM +0800, Ren Wen wrote:
An option for 'submit': -f, --upload-file FILE upload an XML template of libvirt
FILE's extension should be '.xml'.
Example:
case1: submit -f existent_file.xml => existent_file/<md5sum>.xml case2: submit -f nonexistent_file.xml => Find no file: nonexistent_file.xml
Signed-off-by: Ren Wen <15991987063@163.com> --- lib/job.rb | 11 +++++++++++ sbin/submit | 31 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+)
diff --git a/lib/job.rb b/lib/job.rb index 631ebfc98..812f8c299 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -16,6 +16,17 @@ require 'set' require 'pp' require 'English'
+# copied from compass-ci/container/defconfig.rb +def cci_defaults + hash = {} + Dir.glob(['/etc/compass-ci/defaults/*.yaml', + '/etc/compass-ci/accounts/*.yaml', + "#{ENV['HOME']}/.config/compass-ci/defaults/*.yaml"]).each do |file| + hash.update YAML.load_file(file) || {} + end + hash +end + def restore(ah, copy) if ah.class == Hash ah.clear.merge!(copy) diff --git a/sbin/submit b/sbin/submit index 39cdcf576..1d56369ad 100755 --- a/sbin/submit +++ b/sbin/submit @@ -17,6 +17,7 @@ opt_auto_define_files = false opt_monitor = false opt_monitor_query = {} opt_my_queue = false +opt_upload_files = [] actions = ['output', 'stop']
options = OptionParser.new do |opts| @@ -64,10 +65,40 @@ options = OptionParser.new do |opts| opt_my_queue = true end
+ opts.on('-f FILE', '--upload-file FILE', "upload an XML template of libvirt. File: *.xml") do |file| + unless File.exist? file + puts "Find no file: #{file}" + exit 1 + end + + opt_upload_files << file + end + end
options.parse!(ARGV)
+def upload_configs(files) + defaults = cci_defaults() + upload_host = defaults['LKP_SERVER'] + upload_port = defaults['UPLOAD_LIBVIRT_PORT'] || 3081 + + files.each do |file| + response = `curl -sSf -T #{file} http://#{upload_host}:#{upload_port}/` + if $? == 0 + puts response + else + # TODO: how to deal with uploaded file when one failing + puts response + exit 1 + end + end + + exit 0 +end + +upload_configs(opt_upload_files) unless opt_upload_files.empty? + seen_job_yaml = false ARGV.delete_if do |arg| if arg.index '=' -- 2.23.0
On Thu, Jan 21, 2021 at 12:28:44PM +0800, Wu Fengguang wrote:
Can we auto test-upload the libvirt xml file included in the job? A separate upload looks not user friendly and error prone.
可以做到. 但上传和提交libvirt-job是针对libvirt-job的2个不同的功能, 不是一个功能的2部分. To @xsw Thanks, RenWen
Thanks, Fengguang
On Thu, Jan 21, 2021 at 11:29:19AM +0800, Ren Wen wrote:
An option for 'submit': -f, --upload-file FILE upload an XML template of libvirt
FILE's extension should be '.xml'.
Example:
case1: submit -f existent_file.xml => existent_file/<md5sum>.xml case2: submit -f nonexistent_file.xml => Find no file: nonexistent_file.xml
Signed-off-by: Ren Wen <15991987063@163.com> --- lib/job.rb | 11 +++++++++++ sbin/submit | 31 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+)
diff --git a/lib/job.rb b/lib/job.rb index 631ebfc98..812f8c299 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -16,6 +16,17 @@ require 'set' require 'pp' require 'English'
+# copied from compass-ci/container/defconfig.rb +def cci_defaults + hash = {} + Dir.glob(['/etc/compass-ci/defaults/*.yaml', + '/etc/compass-ci/accounts/*.yaml', + "#{ENV['HOME']}/.config/compass-ci/defaults/*.yaml"]).each do |file| + hash.update YAML.load_file(file) || {} + end + hash +end + def restore(ah, copy) if ah.class == Hash ah.clear.merge!(copy) diff --git a/sbin/submit b/sbin/submit index 39cdcf576..1d56369ad 100755 --- a/sbin/submit +++ b/sbin/submit @@ -17,6 +17,7 @@ opt_auto_define_files = false opt_monitor = false opt_monitor_query = {} opt_my_queue = false +opt_upload_files = [] actions = ['output', 'stop']
options = OptionParser.new do |opts| @@ -64,10 +65,40 @@ options = OptionParser.new do |opts| opt_my_queue = true end
+ opts.on('-f FILE', '--upload-file FILE', "upload an XML template of libvirt. File: *.xml") do |file| + unless File.exist? file + puts "Find no file: #{file}" + exit 1 + end + + opt_upload_files << file + end + end
options.parse!(ARGV)
+def upload_configs(files) + defaults = cci_defaults() + upload_host = defaults['LKP_SERVER'] + upload_port = defaults['UPLOAD_LIBVIRT_PORT'] || 3081 + + files.each do |file| + response = `curl -sSf -T #{file} http://#{upload_host}:#{upload_port}/` + if $? == 0 + puts response + else + # TODO: how to deal with uploaded file when one failing + puts response + exit 1 + end + end + + exit 0 +end + +upload_configs(opt_upload_files) unless opt_upload_files.empty? + seen_job_yaml = false ARGV.delete_if do |arg| if arg.index '=' -- 2.23.0
On Thu, Jan 21, 2021 at 12:49:12PM +0800, Ren Wen wrote:
On Thu, Jan 21, 2021 at 12:28:44PM +0800, Wu Fengguang wrote:
Can we auto test-upload the libvirt xml file included in the job? A separate upload looks not user friendly and error prone.
可以做到. 但上传和提交libvirt-job是针对libvirt-job的2个不同的功能, 不是一个功能的2部分.
To @xsw
Thanks, RenWen
Thanks, Fengguang
On Thu, Jan 21, 2021 at 11:29:19AM +0800, Ren Wen wrote:
An option for 'submit': -f, --upload-file FILE upload an XML template of libvirt
FILE's extension should be '.xml'.
Example:
case1: submit -f existent_file.xml => existent_file/<md5sum>.xml case2: submit -f nonexistent_file.xml => Find no file: nonexistent_file.xml
Signed-off-by: Ren Wen <15991987063@163.com> --- lib/job.rb | 11 +++++++++++ sbin/submit | 31 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+)
diff --git a/lib/job.rb b/lib/job.rb index 631ebfc98..812f8c299 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -16,6 +16,17 @@ require 'set' require 'pp' require 'English'
+# copied from compass-ci/container/defconfig.rb +def cci_defaults + hash = {} + Dir.glob(['/etc/compass-ci/defaults/*.yaml', + '/etc/compass-ci/accounts/*.yaml', + "#{ENV['HOME']}/.config/compass-ci/defaults/*.yaml"]).each do |file| + hash.update YAML.load_file(file) || {} + end + hash +end + def restore(ah, copy) if ah.class == Hash ah.clear.merge!(copy) diff --git a/sbin/submit b/sbin/submit index 39cdcf576..1d56369ad 100755 --- a/sbin/submit +++ b/sbin/submit @@ -17,6 +17,7 @@ opt_auto_define_files = false opt_monitor = false opt_monitor_query = {} opt_my_queue = false +opt_upload_files = [] actions = ['output', 'stop']
options = OptionParser.new do |opts| @@ -64,10 +65,40 @@ options = OptionParser.new do |opts| opt_my_queue = true end
+ opts.on('-f FILE', '--upload-file FILE', "upload an XML template of libvirt. File: *.xml") do |file| + unless File.exist? file + puts "Find no file: #{file}" + exit 1 + end + + opt_upload_files << file + end + end
options.parse!(ARGV)
+def upload_configs(files) + defaults = cci_defaults() + upload_host = defaults['LKP_SERVER'] + upload_port = defaults['UPLOAD_LIBVIRT_PORT'] || 3081 + + files.each do |file| + response = `curl -sSf -T #{file} http://#{upload_host}:#{upload_port}/` + if $? == 0 + puts response + else + # TODO: how to deal with uploaded file when one failing + puts response + exit 1 + end + end + + exit 0 +end + +upload_configs(opt_upload_files) unless opt_upload_files.empty? + seen_job_yaml = false ARGV.delete_if do |arg| if arg.index '=' -- 2.23.0
On Thu, Jan 21, 2021 at 12:28:44PM +0800, Wu Fengguang wrote:
Can we auto test-upload the libvirt xml file included in the job? A separate upload looks not user friendly and error prone.
ideally, submit -f xsw-domain.xml testbox=vt-2p16g iperfA.yaml submit testbox=vt-2p16g iperfA.yaml submit steps: - create final job - iterate over vt.*.file
create job field: [md5sum for all xml files] submit modify job to use the final xsw-domain/xxxx.xml
scheduler check with server side scheduler return json with incomplete fail, with a field tell submit to upload missing xml files submit job again, with job fields whose value is xml file content
job.yaml submit first try: vt: domain: xsw-domain.xml file-md5sum: filename: md5sum xsw-domain.xml: md5sum submit 2nd try, add field: file-base64: filename: base64 submit result_root/job.yaml vt: domain: xsw-domain.xml/xxx a/b/xsw-domain.xml => xsw-domain.xml/md5 :x
Thanks, Fengguang
On Thu, Jan 21, 2021 at 11:29:19AM +0800, Ren Wen wrote:
An option for 'submit': -f, --upload-file FILE upload an XML template of libvirt
FILE's extension should be '.xml'.
Example:
case1: submit -f existent_file.xml => existent_file/<md5sum>.xml case2: submit -f nonexistent_file.xml => Find no file: nonexistent_file.xml
Signed-off-by: Ren Wen <15991987063@163.com> --- lib/job.rb | 11 +++++++++++ sbin/submit | 31 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+)
diff --git a/lib/job.rb b/lib/job.rb index 631ebfc98..812f8c299 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -16,6 +16,17 @@ require 'set' require 'pp' require 'English'
+# copied from compass-ci/container/defconfig.rb +def cci_defaults + hash = {} + Dir.glob(['/etc/compass-ci/defaults/*.yaml', + '/etc/compass-ci/accounts/*.yaml', + "#{ENV['HOME']}/.config/compass-ci/defaults/*.yaml"]).each do |file| + hash.update YAML.load_file(file) || {} + end + hash +end + def restore(ah, copy) if ah.class == Hash ah.clear.merge!(copy) diff --git a/sbin/submit b/sbin/submit index 39cdcf576..1d56369ad 100755 --- a/sbin/submit +++ b/sbin/submit @@ -17,6 +17,7 @@ opt_auto_define_files = false opt_monitor = false opt_monitor_query = {} opt_my_queue = false +opt_upload_files = [] actions = ['output', 'stop']
options = OptionParser.new do |opts| @@ -64,10 +65,40 @@ options = OptionParser.new do |opts| opt_my_queue = true end
+ opts.on('-f FILE', '--upload-file FILE', "upload an XML template of libvirt. File: *.xml") do |file| + unless File.exist? file + puts "Find no file: #{file}" + exit 1 + end + + opt_upload_files << file + end + end
options.parse!(ARGV)
+def upload_configs(files) + defaults = cci_defaults() + upload_host = defaults['LKP_SERVER'] + upload_port = defaults['UPLOAD_LIBVIRT_PORT'] || 3081 + + files.each do |file| + response = `curl -sSf -T #{file} http://#{upload_host}:#{upload_port}/` + if $? == 0 + puts response + else + # TODO: how to deal with uploaded file when one failing + puts response + exit 1 + end + end + + exit 0 +end + +upload_configs(opt_upload_files) unless opt_upload_files.empty? + seen_job_yaml = false ARGV.delete_if do |arg| if arg.index '=' -- 2.23.0
participants (3)
-
Ren Wen -
Wu Fengguang -
Xiao Shenwei