Separate "create_job_cpio" into a new file.
Use class Sched, can use Sched's instance variable.
Also can simplify jobfile_operate.cr.
Signed-off-by: Wu Zhende <wuzhende666(a)163.com>
---
src/lib/sched.cr | 1 +
src/scheduler/create_job_cpio.cr | 140 +++++++++++++++++++++++++++++++
src/scheduler/find_job_boot.cr | 2 +-
src/scheduler/jobfile_operate.cr | 132 -----------------------------
4 files changed, 142 insertions(+), 133 deletions(-)
create mode 100644 src/scheduler/create_job_cpio.cr
diff --git a/src/lib/sched.cr b/src/lib/sched.cr
index a9f893f..22e0e4e 100644
--- a/src/lib/sched.cr
+++ b/src/lib/sched.cr
@@ -20,6 +20,7 @@ require "../scheduler/find_next_job_boot"
require "../scheduler/close_job"
require "../scheduler/request_cluster_state"
require "../scheduler/update_job_parameter"
+require "../scheduler/create_job_cpio"
class Sched
property es
diff --git a/src/scheduler/create_job_cpio.cr b/src/scheduler/create_job_cpio.cr
new file mode 100644
index 0000000..5c1be7e
--- /dev/null
+++ b/src/scheduler/create_job_cpio.cr
@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: MulanPSL-2.0+
+# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
+
+# shellwords require from '/c/lkp-tests/lib/'
+require "shellwords"
+require "file_utils"
+require "json"
+require "yaml"
+
+require "./jobfile_operate"
+
+class Sched
+ private def prepare_dir(file_path : String)
+ file_path_dir = File.dirname(file_path)
+ unless File.exists?(file_path_dir)
+ FileUtils.mkdir_p(file_path_dir)
+ end
+ end
+
+ private def valid_shell_variable?(key)
+ key =~ /^[a-zA-Z_]+[a-zA-Z0-9_]*$/
+ end
+
+ private def create_job_sh(job_sh_content : Array(JSON::Any), path : String)
+ File.open(path, "w", File::Permissions.new(0o775)) do |file|
+ file.puts "#!/bin/sh\n\n"
+
+ job_sh_content.each do |line|
+ if line.as_a?
+ line.as_a.each { |val| file.puts val }
+ else
+ file.puts line
+ end
+ end
+
+ file.puts "\"$@\""
+ end
+ end
+
+ private def shell_escape(val)
+ val = val.join "\n" if val.is_a?(Array)
+
+ if val.nil? || val.empty?
+ return nil
+ elsif val =~ /^[+-]?([0-9]*\.?[0-9]+|[0-9]+\.?[0-9]*)([eE][+-]?[0-9]+)?$/
+ return val
+ elsif !val.includes?("'") && !val.includes?("$")
+ return "'#{val}'"
+ elsif !val.includes?('"')
+ return "\"#{val}\""
+ else
+ return Shellwords.shellescape(val)
+ end
+ end
+
+ private def parse_one(script_lines, key, val)
+ return false if val.as_h? || !valid_shell_variable?(key)
+
+ value = shell_escape(val.as_a? || val.to_s)
+ script_lines << "\texport #{key}=" + value if value
+ end
+
+ private def sh_export_top_env(job_content : Hash)
+ script_lines = ["export_top_env()", "{"] of String
+
+ job_content.each { |key, val| parse_one(script_lines, key, val) }
+
+ script_lines << "\n"
+ script_lines << "\t[ -n \"$LKP_SRC\" ] ||"
+ script_lines << "\texport LKP_SRC=/lkp/${user:-lkp}/src"
+ script_lines << "}\n\n"
+
+ script_lines = script_lines.to_s
+ script_lines = JSON.parse(script_lines)
+ end
+
+ def create_job_cpio(job_content : JSON::Any, base_dir : String)
+ job_content = job_content.as_h
+
+ # put job2sh in an array
+ if job_content.has_key?("job2sh")
+ tmp_job_sh_content = job_content["job2sh"]
+
+ job_sh_array = [] of JSON::Any
+ tmp_job_sh_content.as_h.each do |_key, val|
+ next if val == nil
+ job_sh_array += val.as_a
+ end
+ else
+ job_sh_array = [] of JSON::Any
+ end
+
+ # generate job.yaml
+ temp_yaml = base_dir + "/#{job_content["id"]}/job.yaml"
+ prepare_dir(temp_yaml)
+
+ # no change to <object> content { "#! jobs/pixz.yaml": null }
+ # - this will create a <'#! jobs/pixz.yaml':> in the yaml file
+ # - but the orange is <#! jobs/pixz.yaml> in the user job.yaml
+ # tested : no effect to job.sh
+ File.open(temp_yaml, "w") do |file|
+ YAML.dump(job_content, file)
+ end
+
+ # generate unbroken job shell content
+ sh_export_top_env = sh_export_top_env(job_content)
+ job_sh_content = sh_export_top_env.as_a + job_sh_array
+
+ # generate job.sh
+ job_sh = base_dir + "/#{job_content["id"]}/job.sh"
+ create_job_sh(job_sh_content.to_a, job_sh)
+
+ job_dir = base_dir + "/#{job_content["id"]}"
+
+ if job_sh_array.empty?
+ lkp_src = Jobfile::Operate.prepare_lkp_tests(job_content["lkp_initrd_user"],
+ job_content["os_arch"])
+
+ cmd = "#{lkp_src}/sbin/create-job-cpio.sh #{temp_yaml}"
+ idd = `#{cmd}`
+ else
+ cmd = "./create-job-cpio.sh #{job_dir}"
+ idd = `#{cmd}`
+ end
+
+ # if the create job cpio failed, what to do?
+ puts idd if idd.match(/ERROR/)
+
+ # create result dir and copy job.sh, job.yaml and job.cgz to result dir
+ src_dir = File.dirname(temp_yaml)
+ dst_dir = File.join("/srv", job_content["result_root"].to_s)
+ FileUtils.mkdir_p(dst_dir)
+
+ # the job.yaml is not final version
+ files = ["#{src_dir}/job.sh",
+ "#{src_dir}/job.yaml",
+ "#{src_dir}/job.cgz"]
+ FileUtils.cp(files, dst_dir)
+ end
+end
diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr
index ad48f27..6f51640 100644
--- a/src/scheduler/find_job_boot.cr
+++ b/src/scheduler/find_job_boot.cr
@@ -75,7 +75,7 @@ class Sched
job = get_job_from_queues(queues, host)
if job
- Jobfile::Operate.create_job_cpio(job.dump_to_json_any, Kemal.config.public_folder)
+ create_job_cpio(job.dump_to_json_any, Kemal.config.public_folder)
end
return boot_content(job, boot_type)
diff --git a/src/scheduler/jobfile_operate.cr b/src/scheduler/jobfile_operate.cr
index 98bf011..df79d3b 100644
--- a/src/scheduler/jobfile_operate.cr
+++ b/src/scheduler/jobfile_operate.cr
@@ -5,139 +5,7 @@ require "file_utils"
require "json"
require "yaml"
-# require from '/c/lkp-tests/lib/'
-require "shellwords"
-
-LKP_PATH = "/c/lkp-tests"
-if ENV["LKP_SRC"] != LKP_PATH
- raise "ENV LKP_SRC mismatch: #{ENV["LKP_SRC"]} #{LKP_PATH}"
-end
-
module Jobfile::Operate
- def self.prepare_dir(file_path : String)
- file_path_dir = File.dirname(file_path)
- unless File.exists?(file_path_dir)
- FileUtils.mkdir_p(file_path_dir)
- end
- end
-
- def self.valid_shell_variable?(key)
- key =~ /^[a-zA-Z_]+[a-zA-Z0-9_]*$/
- end
-
- def self.create_job_sh(job_sh_content : Array(JSON::Any), path : String)
- File.open(path, "w", File::Permissions.new(0o775)) do |file|
- file.puts "#!/bin/sh\n\n"
- job_sh_content.each do |line|
- if line.as_a?
- line.as_a.each { |val| file.puts val }
- else
- file.puts line
- end
- end
- file.puts "\"$@\""
- end
- end
-
- def self.shell_escape(val)
- val = val.join "\n" if val.is_a?(Array)
-
- if val.nil? || val.empty?
- return nil
- elsif val =~ /^[+-]?([0-9]*\.?[0-9]+|[0-9]+\.?[0-9]*)([eE][+-]?[0-9]+)?$/
- return val
- elsif !val.includes?("'") && !val.includes?("$")
- return "'#{val}'"
- elsif !val.includes?('"')
- return "\"#{val}\""
- else
- return Shellwords.shellescape(val)
- end
- end
-
- def self.parse_one(script_lines, key, val)
- return false if val.as_h? || !valid_shell_variable?(key)
-
- value = shell_escape(val.as_a? || val.to_s)
- script_lines << "\texport #{key}=" + value if value
- end
-
- def self.sh_export_top_env(job_content : Hash)
- script_lines = ["export_top_env()", "{"] of String
-
- job_content.each { |key, val| parse_one(script_lines, key, val) }
-
- script_lines << "\n"
- script_lines << "\t[ -n \"$LKP_SRC\" ] ||"
- script_lines << "\texport LKP_SRC=/lkp/${user:-lkp}/src"
- script_lines << "}\n\n"
-
- script_lines = script_lines.to_s
- script_lines = JSON.parse(script_lines)
- end
-
- def self.create_job_cpio(job_content : JSON::Any, base_dir : String)
- job_content = job_content.as_h
-
- # put job2sh in an array
- if job_content.has_key?("job2sh")
- tmp_job_sh_content = job_content["job2sh"]
-
- job_sh_array = [] of JSON::Any
- tmp_job_sh_content.as_h.each do |_key, val|
- next if val == nil
- job_sh_array += val.as_a
- end
- else
- job_sh_array = [] of JSON::Any
- end
-
- # generate job.yaml
- temp_yaml = base_dir + "/#{job_content["id"]}/job.yaml"
- prepare_dir(temp_yaml)
-
- # no change to <object> content { "#! jobs/pixz.yaml": null }
- # - this will create a <'#! jobs/pixz.yaml':> in the yaml file
- # - but the orange is <#! jobs/pixz.yaml> in the user job.yaml
- # tested : no effect to job.sh
- File.open(temp_yaml, "w") do |file|
- YAML.dump(job_content, file)
- end
-
- # generate unbroken job shell content
- sh_export_top_env = sh_export_top_env(job_content)
- job_sh_content = sh_export_top_env.as_a + job_sh_array
-
- # generate job.sh
- job_sh = base_dir + "/#{job_content["id"]}/job.sh"
- create_job_sh(job_sh_content.to_a, job_sh)
-
- job_dir = base_dir + "/#{job_content["id"]}"
-
- if job_sh_array.empty?
- lkp_src = prepare_lkp_tests(job_content["lkp_initrd_user"],
- job_content["os_arch"])
-
- cmd = "#{lkp_src}/sbin/create-job-cpio.sh #{temp_yaml}"
- idd = `#{cmd}`
- else
- cmd = "./create-job-cpio.sh #{job_dir}"
- idd = `#{cmd}`
- end
-
- # if the create job cpio failed, what to do?
- puts idd if idd.match(/ERROR/)
- # create result dir and copy job.sh, job.yaml and job.cgz to result dir
- src_dir = File.dirname(temp_yaml)
- dst_dir = File.join("/srv", job_content["result_root"].to_s)
- FileUtils.mkdir_p(dst_dir)
- # the job.yaml is not final version
- files = ["#{src_dir}/job.sh",
- "#{src_dir}/job.yaml",
- "#{src_dir}/job.cgz"]
- FileUtils.cp(files, dst_dir)
- end
-
def self.unzip_cgz(source_path : String, target_path : String)
FileUtils.mkdir_p(target_path)
cmd = "cd #{target_path};gzip -dc #{source_path}|cpio -id"
--
2.23.0