Separate "create_job_cpio" into a new file. Use class Sched, can use Sched's instance variable. Also can simplify jobfile_operate.cr. Signed-off-by: Wu Zhende <wuzhende666@163.com> --- src/lib/sched.cr | 1 + src/scheduler/create_job_cpio.cr | 140 +++++++++++++++++++++++++++++++ src/scheduler/find_job_boot.cr | 2 +- src/scheduler/jobfile_operate.cr | 132 ----------------------------- 4 files changed, 142 insertions(+), 133 deletions(-) create mode 100644 src/scheduler/create_job_cpio.cr diff --git a/src/lib/sched.cr b/src/lib/sched.cr index a9f893f..22e0e4e 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -20,6 +20,7 @@ require "../scheduler/find_next_job_boot" require "../scheduler/close_job" require "../scheduler/request_cluster_state" require "../scheduler/update_job_parameter" +require "../scheduler/create_job_cpio" class Sched property es diff --git a/src/scheduler/create_job_cpio.cr b/src/scheduler/create_job_cpio.cr new file mode 100644 index 0000000..5c1be7e --- /dev/null +++ b/src/scheduler/create_job_cpio.cr @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +# shellwords require from '/c/lkp-tests/lib/' +require "shellwords" +require "file_utils" +require "json" +require "yaml" + +require "./jobfile_operate" + +class Sched + private def prepare_dir(file_path : String) + file_path_dir = File.dirname(file_path) + unless File.exists?(file_path_dir) + FileUtils.mkdir_p(file_path_dir) + end + end + + private def valid_shell_variable?(key) + key =~ /^[a-zA-Z_]+[a-zA-Z0-9_]*$/ + end + + private def create_job_sh(job_sh_content : Array(JSON::Any), path : String) + File.open(path, "w", File::Permissions.new(0o775)) do |file| + file.puts "#!/bin/sh\n\n" + + job_sh_content.each do |line| + if line.as_a? + line.as_a.each { |val| file.puts val } + else + file.puts line + end + end + + file.puts "\"$@\"" + end + end + + private def shell_escape(val) + val = val.join "\n" if val.is_a?(Array) + + if val.nil? || val.empty? + return nil + elsif val =~ /^[+-]?([0-9]*\.?[0-9]+|[0-9]+\.?[0-9]*)([eE][+-]?[0-9]+)?$/ + return val + elsif !val.includes?("'") && !val.includes?("$") + return "'#{val}'" + elsif !val.includes?('"') + return "\"#{val}\"" + else + return Shellwords.shellescape(val) + end + end + + private def parse_one(script_lines, key, val) + return false if val.as_h? || !valid_shell_variable?(key) + + value = shell_escape(val.as_a? || val.to_s) + script_lines << "\texport #{key}=" + value if value + end + + private def sh_export_top_env(job_content : Hash) + script_lines = ["export_top_env()", "{"] of String + + job_content.each { |key, val| parse_one(script_lines, key, val) } + + script_lines << "\n" + script_lines << "\t[ -n \"$LKP_SRC\" ] ||" + script_lines << "\texport LKP_SRC=/lkp/${user:-lkp}/src" + script_lines << "}\n\n" + + script_lines = script_lines.to_s + script_lines = JSON.parse(script_lines) + end + + def create_job_cpio(job_content : JSON::Any, base_dir : String) + job_content = job_content.as_h + + # put job2sh in an array + if job_content.has_key?("job2sh") + tmp_job_sh_content = job_content["job2sh"] + + job_sh_array = [] of JSON::Any + tmp_job_sh_content.as_h.each do |_key, val| + next if val == nil + job_sh_array += val.as_a + end + else + job_sh_array = [] of JSON::Any + end + + # generate job.yaml + temp_yaml = base_dir + "/#{job_content["id"]}/job.yaml" + prepare_dir(temp_yaml) + + # no change to <object> content { "#! jobs/pixz.yaml": null } + # - this will create a <'#! jobs/pixz.yaml':> in the yaml file + # - but the orange is <#! jobs/pixz.yaml> in the user job.yaml + # tested : no effect to job.sh + File.open(temp_yaml, "w") do |file| + YAML.dump(job_content, file) + end + + # generate unbroken job shell content + sh_export_top_env = sh_export_top_env(job_content) + job_sh_content = sh_export_top_env.as_a + job_sh_array + + # generate job.sh + job_sh = base_dir + "/#{job_content["id"]}/job.sh" + create_job_sh(job_sh_content.to_a, job_sh) + + job_dir = base_dir + "/#{job_content["id"]}" + + if job_sh_array.empty? + lkp_src = Jobfile::Operate.prepare_lkp_tests(job_content["lkp_initrd_user"], + job_content["os_arch"]) + + cmd = "#{lkp_src}/sbin/create-job-cpio.sh #{temp_yaml}" + idd = `#{cmd}` + else + cmd = "./create-job-cpio.sh #{job_dir}" + idd = `#{cmd}` + end + + # if the create job cpio failed, what to do? + puts idd if idd.match(/ERROR/) + + # create result dir and copy job.sh, job.yaml and job.cgz to result dir + src_dir = File.dirname(temp_yaml) + dst_dir = File.join("/srv", job_content["result_root"].to_s) + FileUtils.mkdir_p(dst_dir) + + # the job.yaml is not final version + files = ["#{src_dir}/job.sh", + "#{src_dir}/job.yaml", + "#{src_dir}/job.cgz"] + FileUtils.cp(files, dst_dir) + end +end diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index ad48f27..6f51640 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -75,7 +75,7 @@ class Sched job = get_job_from_queues(queues, host) if job - Jobfile::Operate.create_job_cpio(job.dump_to_json_any, Kemal.config.public_folder) + create_job_cpio(job.dump_to_json_any, Kemal.config.public_folder) end return boot_content(job, boot_type) diff --git a/src/scheduler/jobfile_operate.cr b/src/scheduler/jobfile_operate.cr index 98bf011..df79d3b 100644 --- a/src/scheduler/jobfile_operate.cr +++ b/src/scheduler/jobfile_operate.cr @@ -5,139 +5,7 @@ require "file_utils" require "json" require "yaml" -# require from '/c/lkp-tests/lib/' -require "shellwords" - -LKP_PATH = "/c/lkp-tests" -if ENV["LKP_SRC"] != LKP_PATH - raise "ENV LKP_SRC mismatch: #{ENV["LKP_SRC"]} #{LKP_PATH}" -end - module Jobfile::Operate - def self.prepare_dir(file_path : String) - file_path_dir = File.dirname(file_path) - unless File.exists?(file_path_dir) - FileUtils.mkdir_p(file_path_dir) - end - end - - def self.valid_shell_variable?(key) - key =~ /^[a-zA-Z_]+[a-zA-Z0-9_]*$/ - end - - def self.create_job_sh(job_sh_content : Array(JSON::Any), path : String) - File.open(path, "w", File::Permissions.new(0o775)) do |file| - file.puts "#!/bin/sh\n\n" - job_sh_content.each do |line| - if line.as_a? - line.as_a.each { |val| file.puts val } - else - file.puts line - end - end - file.puts "\"$@\"" - end - end - - def self.shell_escape(val) - val = val.join "\n" if val.is_a?(Array) - - if val.nil? || val.empty? - return nil - elsif val =~ /^[+-]?([0-9]*\.?[0-9]+|[0-9]+\.?[0-9]*)([eE][+-]?[0-9]+)?$/ - return val - elsif !val.includes?("'") && !val.includes?("$") - return "'#{val}'" - elsif !val.includes?('"') - return "\"#{val}\"" - else - return Shellwords.shellescape(val) - end - end - - def self.parse_one(script_lines, key, val) - return false if val.as_h? || !valid_shell_variable?(key) - - value = shell_escape(val.as_a? || val.to_s) - script_lines << "\texport #{key}=" + value if value - end - - def self.sh_export_top_env(job_content : Hash) - script_lines = ["export_top_env()", "{"] of String - - job_content.each { |key, val| parse_one(script_lines, key, val) } - - script_lines << "\n" - script_lines << "\t[ -n \"$LKP_SRC\" ] ||" - script_lines << "\texport LKP_SRC=/lkp/${user:-lkp}/src" - script_lines << "}\n\n" - - script_lines = script_lines.to_s - script_lines = JSON.parse(script_lines) - end - - def self.create_job_cpio(job_content : JSON::Any, base_dir : String) - job_content = job_content.as_h - - # put job2sh in an array - if job_content.has_key?("job2sh") - tmp_job_sh_content = job_content["job2sh"] - - job_sh_array = [] of JSON::Any - tmp_job_sh_content.as_h.each do |_key, val| - next if val == nil - job_sh_array += val.as_a - end - else - job_sh_array = [] of JSON::Any - end - - # generate job.yaml - temp_yaml = base_dir + "/#{job_content["id"]}/job.yaml" - prepare_dir(temp_yaml) - - # no change to <object> content { "#! jobs/pixz.yaml": null } - # - this will create a <'#! jobs/pixz.yaml':> in the yaml file - # - but the orange is <#! jobs/pixz.yaml> in the user job.yaml - # tested : no effect to job.sh - File.open(temp_yaml, "w") do |file| - YAML.dump(job_content, file) - end - - # generate unbroken job shell content - sh_export_top_env = sh_export_top_env(job_content) - job_sh_content = sh_export_top_env.as_a + job_sh_array - - # generate job.sh - job_sh = base_dir + "/#{job_content["id"]}/job.sh" - create_job_sh(job_sh_content.to_a, job_sh) - - job_dir = base_dir + "/#{job_content["id"]}" - - if job_sh_array.empty? - lkp_src = prepare_lkp_tests(job_content["lkp_initrd_user"], - job_content["os_arch"]) - - cmd = "#{lkp_src}/sbin/create-job-cpio.sh #{temp_yaml}" - idd = `#{cmd}` - else - cmd = "./create-job-cpio.sh #{job_dir}" - idd = `#{cmd}` - end - - # if the create job cpio failed, what to do? - puts idd if idd.match(/ERROR/) - # create result dir and copy job.sh, job.yaml and job.cgz to result dir - src_dir = File.dirname(temp_yaml) - dst_dir = File.join("/srv", job_content["result_root"].to_s) - FileUtils.mkdir_p(dst_dir) - # the job.yaml is not final version - files = ["#{src_dir}/job.sh", - "#{src_dir}/job.yaml", - "#{src_dir}/job.cgz"] - FileUtils.cp(files, dst_dir) - end - def self.unzip_cgz(source_path : String, target_path : String) FileUtils.mkdir_p(target_path) cmd = "cd #{target_path};gzip -dc #{source_path}|cpio -id" -- 2.23.0