[PATCH compass-ci] scheduler: prints logs in Sched class - Compass-ci

12 Dec 2020

Signed-off-by: Wu Zhende wuzhende666@163.com
---
 src/lib/sched.cr                      | 24 ++++++++--
 src/scheduler/close_job.cr            |  7 ++-
 src/scheduler/download_file.cr        | 14 ++++++
 src/scheduler/find_job_boot.cr        |  6 ++-
 src/scheduler/find_next_job_boot.cr   |  6 ++-
 src/scheduler/scheduler.cr            | 64 +++++----------------------
 src/scheduler/submit_job.cr           | 10 +++--
 src/scheduler/update_job_parameter.cr |  2 +-
 8 files changed, 68 insertions(+), 65 deletions(-)
 create mode 100644 src/scheduler/download_file.cr

diff --git a/src/lib/sched.cr b/src/lib/sched.cr
index 6225ac2..10a51ab 100644
--- a/src/lib/sched.cr
+++ b/src/lib/sched.cr
@@ -21,6 +21,7 @@ require "../scheduler/close_job"
 require "../scheduler/request_cluster_state"
 require "../scheduler/update_job_parameter"
 require "../scheduler/create_job_cpio"
+require "../scheduler/download_file"
class Sched
   property es
@@ -37,6 +38,15 @@ class Sched
     @log = env.log.as(JSONLogger)
   end
+  def debug_message(response)
+    @log.info(%({"from": "#{@env.request.remote_address}", "response": #{response.to_json}}))
+  end
+
+  def alive(version)
+    debug_message("Env= {\n#{`export`}}")
+    "LKP Alive! The time is #{Time.local}, version = #{version}"
+  end
+
   def normalize_mac(mac : String)
     mac.gsub(":", "-")
   end
@@ -97,11 +107,19 @@ class Sched
# json log
     hash["testbox"] = testbox
-    return hash.to_json
+    @log.info(hash.to_json)
   end
-  def report_ssh_port(testbox : String, ssh_port : String)
-    @redis.hash_set("sched/tbox2ssh_port", testbox, ssh_port)
+  def report_ssh_port
+    testbox = @env.params.query["tbox_name"]
+    ssh_port = @env.params.query["ssh_port"].to_s
+    job_id = @env.params.query["job_id"].to_s
+
+    if testbox && ssh_port
+      @redis.hash_set("sched/tbox2ssh_port", testbox, ssh_port)
+    end
+
+    @log.info(%({"job_id": "#{job_id}", "state": "set ssh port", "ssh_port": "#{ssh_port}", "tbox_name": "#{testbox}"}))
   end
private def query_consumable_keys(shortest_queue_name)
diff --git a/src/scheduler/close_job.cr b/src/scheduler/close_job.cr
index 2c33ffd..579c2c0 100644
--- a/src/scheduler/close_job.cr
+++ b/src/scheduler/close_job.cr
@@ -2,7 +2,10 @@
 # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
class Sched
-  def close_job(job_id : String)
+  def close_job
+    job_id = @env.params.query["job_id"]?
+    return unless job_id
+
     job = @redis.get_job(job_id)
delete_access_key_file(job) if job
@@ -22,7 +25,7 @@ class Sched
@redis.remove_finished_job(job_id)
-    return %({"job_id": "#{job_id}", "job_state": "complete"})
+    @log.info(%({"job_id": "#{job_id}", "job_state": "complete"}))
   end
def delete_access_key_file(job : Job)
diff --git a/src/scheduler/download_file.cr b/src/scheduler/download_file.cr
new file mode 100644
index 0000000..e1f0c9a
--- /dev/null
+++ b/src/scheduler/download_file.cr
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: MulanPSL-2.0+
+# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
+
+class Sched
+  def download_file
+    job_id = @env.params.url["job_id"]
+    job_package = @env.params.url["job_package"]
+    file_path = ::File.join [Kemal.config.public_folder, job_id, job_package]
+
+    @log.info(%({"job_id": "#{job_id}", "job_state": "download"}))
+
+    send_file @env, file_path
+  end
+end
diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr
index fb94e91..5a11523 100644
--- a/src/scheduler/find_job_boot.cr
+++ b/src/scheduler/find_job_boot.cr
@@ -16,7 +16,11 @@ class Sched
       host = value
     end
-    get_job_boot(host, boot_type)
+    response = get_job_boot(host, boot_type)
+    job_id = response[/tmpfs/(.*)/job.cgz/, 1]?
+    @log.info(%({"job_id": "#{job_id}", "job_state": "boot"})) if job_id
+
+    response
   end
# auto submit a job to collect the host information
diff --git a/src/scheduler/find_next_job_boot.cr b/src/scheduler/find_next_job_boot.cr
index 09abbd5..0bdfe5b 100644
--- a/src/scheduler/find_next_job_boot.cr
+++ b/src/scheduler/find_next_job_boot.cr
@@ -9,6 +9,10 @@ class Sched
       hostname = @redis.hash_get("sched/mac2host", normalize_mac(mac))
     end
-    get_job_boot(hostname, "ipxe")
+    response = get_job_boot(hostname, "ipxe")
+    job_id = response[/tmpfs/(.*)/job.cgz/, 1]?
+    @log.info(%({"job_id": "#{job_id}", "job_state": "boot"})) if job_id
+
+    response
   end
 end
diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr
index 4fde6e3..7ec149c 100644
--- a/src/scheduler/scheduler.cr
+++ b/src/scheduler/scheduler.cr
@@ -41,15 +41,9 @@ module Scheduler
     env.create_sched
   end
-  # for debug (maybe kemal debug|logger does better)
-  def self.debug_message(env, response)
-    env.log.info(%({"from": "#{env.request.remote_address}", "response": #{response.to_json}}))
-  end
-
   # echo alive
   get "/" do |env|
-    debug_message(env, "Env= {\n#{`export`}}")
-    "LKP Alive! The time is #{Time.local}, version = #{VERSION}"
+    env.sched.alive(VERSION)
   end
# for XXX_runner get job
@@ -58,46 +52,24 @@ module Scheduler
   # /boot.xxx/host/${hostname}
   # /boot.yyy/mac/${mac}
   get "/boot.:boot_type/:parameter/:value" do |env|
-    response = env.sched.find_job_boot
-
-    job_id = response[/tmpfs/(.*)/job.cgz/, 1]?
-    env.log.info(%({"job_id": "#{job_id}", "job_state": "boot"})) if job_id
-
-    response
+    env.sched.find_job_boot
   end
# /~lkp/cgi-bin/gpxelinux.cgi?hostname=:hostname&mac=:mac&last_kernel=:last_kernel
   get "/~lkp/cgi-bin/gpxelinux.cgi" do |env|
-    response = env.sched.find_next_job_boot
-
-    job_id = response[/tmpfs/(.*)/job.cgz/, 1]?
-    env.log.info(%({"job_id": "#{job_id}", "job_state": "boot"})) if job_id
-
-    response
+    env.sched.find_next_job_boot
   end
# enqueue
   #  - echo job_id to caller
   #  -- job_id = "0" ? means failed
   post "/submit_job" do |env|
-    job_messages = env.sched.submit_job
-
-    job_messages.each do |job_message|
-      env.log.info(job_message.to_json)
-    end
-
-    job_messages.to_json
+    env.sched.submit_job.to_json
   end
# file download server
   get "/job_initrd_tmpfs/:job_id/:job_package" do |env|
-    job_id = env.params.url["job_id"]
-    job_package = env.params.url["job_package"]
-    file_path = ::File.join [Kemal.config.public_folder, job_id, job_package]
-
-    env.log.info(%({"job_id": "#{job_id}", "job_state": "download"}))
-
-    send_file env, file_path
+    env.sched.download_file
   end
# client(runner) report its hostname and mac
@@ -155,7 +127,7 @@ module Scheduler
   #  ?job_file=/lkp/scheduled/job.yaml&job_state=post_run&job_id=10
   #  ?job_file=/lkp/scheduled/job.yaml&loadavg=0.28 0.82 0.49 1/105 3389&start_time=1587725398&end_time=1587725698&job_id=10
   get "/~lkp/cgi-bin/lkp-jobfile-append-var" do |env|
-    env.log.info(env.sched.update_job_parameter)
+    env.sched.update_job_parameter
"Done"
   end
@@ -178,42 +150,26 @@ module Scheduler
   #    response: get "server ip" from cluster state,
   #              return "server=<server ip>".
   get "/~lkp/cgi-bin/lkp-cluster-sync" do |env|
-    response = env.sched.request_cluster_state
-
-    debug_message(env, response)
-
-    response
+    env.sched.request_cluster_state
   end
# client(runner) report job post_run finished
   # /~lkp/cgi-bin/lkp-post-run?job_file=/lkp/scheduled/job.yaml&job_id=40
   #  curl "http://localhost:3000/~lkp/cgi-bin/lkp-post-run?job_file=/lkp/scheduled/job...."
   get "/~lkp/cgi-bin/lkp-post-run" do |env|
-    # get job_id from request
-    job_id = env.params.query["job_id"]?
-    if job_id
-      env.log.info(env.sched.close_job(job_id))
-    end
+    env.sched.close_job
"Done"
   end
get "/~lkp/cgi-bin/lkp-wtmp" do |env|
-    env.log.info(env.sched.update_tbox_wtmp)
+    env.sched.update_tbox_wtmp
"Done"
   end
get "/~lkp/cgi-bin/report_ssh_port" do |env|
-    testbox = env.params.query["tbox_name"]
-    ssh_port = env.params.query["ssh_port"].to_s
-    job_id = env.params.query["job_id"].to_s
-
-    if testbox && ssh_port
-      env.sched.report_ssh_port(testbox, ssh_port)
-    end
-
-    env.log.info(%({"job_id": "#{job_id}", "state": "set ssh port", "ssh_port": "#{ssh_port}", "tbox_name": "#{testbox}"}))
+    env.sched.report_ssh_port
"Done"
   end
diff --git a/src/scheduler/submit_job.cr b/src/scheduler/submit_job.cr
index 3909cb7..4303b1d 100644
--- a/src/scheduler/submit_job.cr
+++ b/src/scheduler/submit_job.cr
@@ -11,20 +11,24 @@ class Sched
cluster_file = job["cluster"]
     if cluster_file.empty? || cluster_file == "cs-localhost"
-      return submit_single_job(job)
+      response = submit_single_job(job)
     else
       cluster_config = get_cluster_config(cluster_file,
         job.lkp_initrd_user,
         job.os_arch).not_nil!
-      return submit_cluster_job(job, cluster_config)
+      response = submit_cluster_job(job, cluster_config)
     end
   rescue ex
     @log.warn(ex.inspect_with_backtrace)
-    return [{
+    response = [{
       "job_id"    => "0",
       "message"   => ex.to_s,
       "job_state" => "submit",
     }]
+  ensure
+    response.each do |job_message|
+      @log.info(job_message.to_json)
+    end
   end
# return:
diff --git a/src/scheduler/update_job_parameter.cr b/src/scheduler/update_job_parameter.cr
index cee8b9b..da528c1 100644
--- a/src/scheduler/update_job_parameter.cr
+++ b/src/scheduler/update_job_parameter.cr
@@ -29,6 +29,6 @@ class Sched
     # json log
     log = job_content.dup
     log["job_id"] = log.delete("id").not_nil!
-    return log.to_json
+    @log.info(log.to_json)
   end
 end
-- 
2.23.0

    