[PATCH v1 compass-ci] improve multi-qemu code to support queue parameter
Usage: multi-qemu [-n] [-c] [-q] -n, --name HOSTNAME_PREFIX specify used hostname_prefix -c, --count count how much VM do you need -q, --queues queues queues from which tasks are obtained -h, --help show this message example: ./multi-qemu -n vm-2p8g.xsw -c 2 -q vm-2p8g~xsw,vm-2p8g.aarch64 Signed-off-by: Xiao Shenwei <xiaoshenwei96@163.com> --- providers/multi-qemu | 98 +++++++++++++++++++++++++++++++++----------- 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/providers/multi-qemu b/providers/multi-qemu index 23e0451..03564f7 100755 --- a/providers/multi-qemu +++ b/providers/multi-qemu @@ -4,41 +4,91 @@ # frozen_string_literal: true require 'fileutils' +require 'optparse' +require 'time' -PWD = Dir.pwd +opt = {} +options = OptionParser.new do |opts| + opts.banner = 'Usage: multi-qemu [-n] [-c] [-q]' + + opts.separator '' + opts.on('-n HOSTNAME_PREFIX', '--name HOSTNAME_PREFIX', 'specify used hostname_prefix') do |name| + opt['hostname_prefix'] = name + end + + opts.on('-c count', '--count count', 'how much VM do you need') do |num| + opt['nr_vm'] = num + end + + opts.on('-q queues', '--queues queues', 'requested queues, use "," to separate more than 2 values') do |queues| + opt['queues'] = queues + end + + opts.on_tail('-h', '--help', 'show this message') do + puts opts + exit + end +end + +if ARGV.size.zero? + puts options + exit +end + +options.parse!(ARGV) # Run multiple QEMU in parallel -HOSTNAME = ARGV[0] || "vm-2p8g--#{ENV['USER']}" -NR_VM = ARGV[1] || 1 +PWD = Dir.pwd +HOSTNAME = opt['hostname_prefix'] || "vm-2p8g.#{ENV['USER']}" +NR_VM = opt['nr_vm'] || 1 +QUEUES = opt['queues'] || "vm-2p8g.#{RUBY_PLATFORM.split('-')[0]}" +LOG_DIR = '/srv/cci/serial/logs' -def run(seqno) - loop do - start_time = Time.new - hostname = "#{HOSTNAME}-#{seqno}" - log_file = "/srv/cci/serial/logs/#{hostname}" +def main(hostname) + start_time = Time.new.strftime('%Y-%m-%d %H:%M:%S') + log_file = "#{LOG_DIR}/#{hostname}" + record_runtime(log_file, start_time) + run_vm(hostname) + duration = ((Time.new - Time.parse(start_time)) / 60).round(2) + record_runtime(log_file, duration, is_start: false) +end +def record_runtime(log_file, message, is_start: true) + if is_start File.open(log_file, 'w') do |f| # fluentd refresh time is 1s # let fluentd to monitor this file first sleep(2) - f.puts "\n#{start_time.strftime('%Y-%m-%d %H:%M:%S')} starting QEMU" + f.puts "\n#{message} starting QEMU" end + return + end + File.open(log_file, 'a') do |f| + f.puts "\nTotal QEMU duration: #{message} minutes" + end +end - pwd_hostname = File.join(PWD, hostname) - FileUtils.mkdir_p(pwd_hostname) unless File.exist?(pwd_hostname) - FileUtils.cd(pwd_hostname) - system( - { 'hostname' => hostname }, - ENV['CCI_SRC'] + '/providers/qemu.sh' - ) - - duration = ((Time.new - start_time) / 60).round(2) - File.open(log_file, 'a') do |f| - f.puts "\nTotal QEMU duration: #{duration} minutes" - end +def run_vm(hostname) + pwd_hostname = File.join(PWD, hostname) + FileUtils.mkdir_p(pwd_hostname) unless File.exist?(pwd_hostname) + FileUtils.cd(pwd_hostname) + system( + { 'hostname' => hostname, 'queues' => QUEUES }, + ENV['CCI_SRC'] + '/providers/qemu.sh' + ) +end - # sleep 5s is for fluentd to collect it's log - sleep(5) +def loop_main(hostname) + loop do + begin + main(hostname) + rescue StandardError => e + puts e.backtrace + # if an exception happend, request the next time after 30 seconds + sleep 25 + ensure + sleep 5 + end end end @@ -55,7 +105,7 @@ def multiqemu pids = [] NR_VM.to_i.times do |i| pid = Process.fork do - run i + loop_main("#{HOSTNAME}-#{i}") end pids << pid end -- 2.23.0
On Wed, Nov 04, 2020 at 10:37:28AM +0800, Xiao Shenwei wrote:
Usage: multi-qemu [-n] [-c] [-q] -n, --name HOSTNAME_PREFIX specify used hostname_prefix -c, --count count how much VM do you need -q, --queues queues queues from which tasks are obtained -h, --help show this message
example: ./multi-qemu -n vm-2p8g.xsw -c 2 -q vm-2p8g~xsw,vm-2p8g.aarch64
Signed-off-by: Xiao Shenwei <xiaoshenwei96@163.com> --- providers/multi-qemu | 98 +++++++++++++++++++++++++++++++++----------- 1 file changed, 74 insertions(+), 24 deletions(-)
diff --git a/providers/multi-qemu b/providers/multi-qemu index 23e0451..03564f7 100755 --- a/providers/multi-qemu +++ b/providers/multi-qemu @@ -4,41 +4,91 @@ # frozen_string_literal: true
require 'fileutils' +require 'optparse' +require 'time'
-PWD = Dir.pwd +opt = {} +options = OptionParser.new do |opts| + opts.banner = 'Usage: multi-qemu [-n] [-c] [-q]' + + opts.separator '' + opts.on('-n HOSTNAME_PREFIX', '--name HOSTNAME_PREFIX', 'specify used hostname_prefix') do |name| + opt['hostname_prefix'] = name + end + + opts.on('-c count', '--count count', 'how much VM do you need') do |num| + opt['nr_vm'] = num + end + + opts.on('-q queues', '--queues queues', 'requested queues, use "," to separate more than 2 values') do |queues| + opt['queues'] = queues + end + + opts.on_tail('-h', '--help', 'show this message') do + puts opts + exit + end +end + +if ARGV.size.zero? + puts options + exit +end + +options.parse!(ARGV)
# Run multiple QEMU in parallel -HOSTNAME = ARGV[0] || "vm-2p8g--#{ENV['USER']}" -NR_VM = ARGV[1] || 1 +PWD = Dir.pwd +HOSTNAME = opt['hostname_prefix'] || "vm-2p8g.#{ENV['USER']}" +NR_VM = opt['nr_vm'] || 1 +QUEUES = opt['queues'] || "vm-2p8g.#{RUBY_PLATFORM.split('-')[0]}"
hao about set default value for opt, and no matter add -c, -n -q, the opt will still has the ele opt['hostname_prefix'], opt['nr_vm'], and opt['queues'], you do not need || here, and no need to check ARGV.size.zero? Thanks Luan Shengde
+LOG_DIR = '/srv/cci/serial/logs'
-def run(seqno) - loop do - start_time = Time.new - hostname = "#{HOSTNAME}-#{seqno}" - log_file = "/srv/cci/serial/logs/#{hostname}" +def main(hostname) + start_time = Time.new.strftime('%Y-%m-%d %H:%M:%S') + log_file = "#{LOG_DIR}/#{hostname}" + record_runtime(log_file, start_time) + run_vm(hostname) + duration = ((Time.new - Time.parse(start_time)) / 60).round(2) + record_runtime(log_file, duration, is_start: false) +end
+def record_runtime(log_file, message, is_start: true) + if is_start File.open(log_file, 'w') do |f| # fluentd refresh time is 1s # let fluentd to monitor this file first sleep(2) - f.puts "\n#{start_time.strftime('%Y-%m-%d %H:%M:%S')} starting QEMU" + f.puts "\n#{message} starting QEMU" end + return + end + File.open(log_file, 'a') do |f| + f.puts "\nTotal QEMU duration: #{message} minutes" + end +end
- pwd_hostname = File.join(PWD, hostname) - FileUtils.mkdir_p(pwd_hostname) unless File.exist?(pwd_hostname) - FileUtils.cd(pwd_hostname) - system( - { 'hostname' => hostname }, - ENV['CCI_SRC'] + '/providers/qemu.sh' - ) - - duration = ((Time.new - start_time) / 60).round(2) - File.open(log_file, 'a') do |f| - f.puts "\nTotal QEMU duration: #{duration} minutes" - end +def run_vm(hostname) + pwd_hostname = File.join(PWD, hostname) + FileUtils.mkdir_p(pwd_hostname) unless File.exist?(pwd_hostname) + FileUtils.cd(pwd_hostname) + system( + { 'hostname' => hostname, 'queues' => QUEUES }, + ENV['CCI_SRC'] + '/providers/qemu.sh' + ) +end
- # sleep 5s is for fluentd to collect it's log - sleep(5) +def loop_main(hostname) + loop do + begin + main(hostname) + rescue StandardError => e + puts e.backtrace + # if an exception happend, request the next time after 30 seconds + sleep 25 + ensure + sleep 5 + end end end
@@ -55,7 +105,7 @@ def multiqemu pids = [] NR_VM.to_i.times do |i| pid = Process.fork do - run i + loop_main("#{HOSTNAME}-#{i}") end pids << pid end -- 2.23.0
+NR_VM = opt['nr_vm'] || 1 +QUEUES = opt['queues'] || "vm-2p8g.#{RUBY_PLATFORM.split('-')[0]}"
hao about set default value for opt, and no matter add -c, -n -q, the opt will still has the ele opt['hostname_prefix'], opt['nr_vm'], and opt['queues'], you do not need || here, and no need to check ARGV.size.zero?
understand, but i need this case: if run ./multi-qemu need show help message because multi-qemu need sepcifies diff parameter for every user, if use default may be not suitable Thanks, Shenwei
Thanks Luan Shengde
+LOG_DIR = '/srv/cci/serial/logs'
-def run(seqno) - loop do - start_time = Time.new - hostname = "#{HOSTNAME}-#{seqno}" - log_file = "/srv/cci/serial/logs/#{hostname}" +def main(hostname) + start_time = Time.new.strftime('%Y-%m-%d %H:%M:%S') + log_file = "#{LOG_DIR}/#{hostname}" + record_runtime(log_file, start_time) + run_vm(hostname) + duration = ((Time.new - Time.parse(start_time)) / 60).round(2) + record_runtime(log_file, duration, is_start: false) +end
+def record_runtime(log_file, message, is_start: true) + if is_start File.open(log_file, 'w') do |f| # fluentd refresh time is 1s # let fluentd to monitor this file first sleep(2) - f.puts "\n#{start_time.strftime('%Y-%m-%d %H:%M:%S')} starting QEMU" + f.puts "\n#{message} starting QEMU" end + return + end + File.open(log_file, 'a') do |f| + f.puts "\nTotal QEMU duration: #{message} minutes" + end +end
- pwd_hostname = File.join(PWD, hostname) - FileUtils.mkdir_p(pwd_hostname) unless File.exist?(pwd_hostname) - FileUtils.cd(pwd_hostname) - system( - { 'hostname' => hostname }, - ENV['CCI_SRC'] + '/providers/qemu.sh' - ) - - duration = ((Time.new - start_time) / 60).round(2) - File.open(log_file, 'a') do |f| - f.puts "\nTotal QEMU duration: #{duration} minutes" - end +def run_vm(hostname) + pwd_hostname = File.join(PWD, hostname) + FileUtils.mkdir_p(pwd_hostname) unless File.exist?(pwd_hostname) + FileUtils.cd(pwd_hostname) + system( + { 'hostname' => hostname, 'queues' => QUEUES }, + ENV['CCI_SRC'] + '/providers/qemu.sh' + ) +end
- # sleep 5s is for fluentd to collect it's log - sleep(5) +def loop_main(hostname) + loop do + begin + main(hostname) + rescue StandardError => e + puts e.backtrace + # if an exception happend, request the next time after 30 seconds + sleep 25 + ensure + sleep 5 + end end end
@@ -55,7 +105,7 @@ def multiqemu pids = [] NR_VM.to_i.times do |i| pid = Process.fork do - run i + loop_main("#{HOSTNAME}-#{i}") end pids << pid end -- 2.23.0
On Wed, Nov 04, 2020 at 02:10:04PM +0800, Xiao Shenwei wrote:
+NR_VM = opt['nr_vm'] || 1 +QUEUES = opt['queues'] || "vm-2p8g.#{RUBY_PLATFORM.split('-')[0]}"
hao about set default value for opt, and no matter add -c, -n -q, the opt will still has the ele opt['hostname_prefix'], opt['nr_vm'], and opt['queues'], you do not need || here, and no need to check ARGV.size.zero?
understand, but i need this case: if run ./multi-qemu need show help message
because multi-qemu need sepcifies diff parameter for every user, if use default may be not suitable
I mean you can set default value, if need use user defined value, use the argument to set new values, I people don't add the arguments. just use default value Thanks Luan Shengde
Thanks, Shenwei
Thanks Luan Shengde
+LOG_DIR = '/srv/cci/serial/logs'
-def run(seqno) - loop do - start_time = Time.new - hostname = "#{HOSTNAME}-#{seqno}" - log_file = "/srv/cci/serial/logs/#{hostname}" +def main(hostname) + start_time = Time.new.strftime('%Y-%m-%d %H:%M:%S') + log_file = "#{LOG_DIR}/#{hostname}" + record_runtime(log_file, start_time) + run_vm(hostname) + duration = ((Time.new - Time.parse(start_time)) / 60).round(2) + record_runtime(log_file, duration, is_start: false) +end
+def record_runtime(log_file, message, is_start: true) + if is_start File.open(log_file, 'w') do |f| # fluentd refresh time is 1s # let fluentd to monitor this file first sleep(2) - f.puts "\n#{start_time.strftime('%Y-%m-%d %H:%M:%S')} starting QEMU" + f.puts "\n#{message} starting QEMU" end + return + end + File.open(log_file, 'a') do |f| + f.puts "\nTotal QEMU duration: #{message} minutes" + end +end
- pwd_hostname = File.join(PWD, hostname) - FileUtils.mkdir_p(pwd_hostname) unless File.exist?(pwd_hostname) - FileUtils.cd(pwd_hostname) - system( - { 'hostname' => hostname }, - ENV['CCI_SRC'] + '/providers/qemu.sh' - ) - - duration = ((Time.new - start_time) / 60).round(2) - File.open(log_file, 'a') do |f| - f.puts "\nTotal QEMU duration: #{duration} minutes" - end +def run_vm(hostname) + pwd_hostname = File.join(PWD, hostname) + FileUtils.mkdir_p(pwd_hostname) unless File.exist?(pwd_hostname) + FileUtils.cd(pwd_hostname) + system( + { 'hostname' => hostname, 'queues' => QUEUES }, + ENV['CCI_SRC'] + '/providers/qemu.sh' + ) +end
- # sleep 5s is for fluentd to collect it's log - sleep(5) +def loop_main(hostname) + loop do + begin + main(hostname) + rescue StandardError => e + puts e.backtrace + # if an exception happend, request the next time after 30 seconds + sleep 25 + ensure + sleep 5 + end end end
@@ -55,7 +105,7 @@ def multiqemu pids = [] NR_VM.to_i.times do |i| pid = Process.fork do - run i + loop_main("#{HOSTNAME}-#{i}") end pids << pid end -- 2.23.0
participants (2)
-
Luan Shengde -
Xiao Shenwei