On Thu, Oct 22, 2020 at 07:08:53PM +0800, Luan Shengde wrote:

On Thu, Oct 22, 2020 at 06:01:47PM +0800, Tong Qunfeng wrote:

...

[why] when there has no job for a testbox, scheduler will consume 4-5 times testbox*1 tbox_group*1 tbox*1 tbox_group/idle*(1|2)

[how] query taskqueue the queue-name first. if the queue-name not exist, then there should be no job for this queue. submit idle job at background, let the consume not wait for submit idle job finished.

Signed-off-by: Tong Qunfeng taxcom@tom.com

src/lib/sched.cr | 52 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 5 deletions(-)

diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 7215df5..8fbe9aa 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -409,19 +409,26 @@ class Sched tbox_group = JobHelper.match_tbox_group(testbox) tbox = tbox_group.partition("--")[0]

• boxes = [testbox, tbox_group, tbox]

• queue_list = query_consumable_keys(tbox)
• boxes = ["sched/" + testbox,

•         "sched/" + tbox_group,
  

•         "sched/" + tbox,
  

•         "sched/" + tbox_group + "/idle"]
  

  boxes.each do |box|

•  next if queue_list.select(box).size == 0
  count.times do
  

•    job = prepare_job("sched/#{box}", testbox)
  

•    job = prepare_job(box, testbox)
    return job if job
  
    sleep(1) unless count == 1
  end
  

  end

• # when find no job at "sched/#{tbox_group}"
• # try to get from "sched/#{tbox_group}/idle"
• return get_idle_job(tbox_group, testbox)

• # when find no job, auto submit idle job at background

• spawn { auto_submit_idle_job(tbox_group) }

• return nil end

  private def prepare_job(queue_name, testbox)

@@ -571,4 +578,39 @@ class Sched

return %({"job_id": "#{job_id}", "job_state": "complete"})

end

• private def query_consumable_keys(shortest_queue_name)
• keys = [] of String
• search = "sched/" + shortest_queue_name + "*"
• response = @task_queue.query_keys(search)
• return keys unless response[0] == 200
• key_list = JSON.parse(response[1].to_json).as_a
• # add consumable keys
• key_list.each do |key|

•  queue_name = consumable_key?("#{key}")
  

consumable_key?("#{key}") => consumable_key?(key) ?

Type of key maybe not string. so I use "#{}" to convert it to string.

...

•  keys << queue_name if queue_name
  

• end
• return keys
• end
• private def consumable_key?(key_name)
• if key_name =~ /(.*)/(.*)$/

you may try: if key_name =~ %r{.*/.*$}

got. I need the match string, so need %r{(.*)/(.*)}. It is same.

Thanks Luan Shengde

...

•  case $2
  

•  when "in_process"
  

•    return nil
  

•  when "ready"
  

•    return $1
  

•  when "idle"
  

•    return key_name
  

•  else
  

•    return key_name
  

•  end
  

• end
• return nil
• end

end

2.23.0

On Thu, Oct 22, 2020 at 09:16:55PM +0800, Liu Shaofei wrote:

On Thu, Oct 22, 2020 at 06:01:47PM +0800, Tong Qunfeng wrote:

...

[why] when there has no job for a testbox, scheduler will consume 4-5 times testbox*1 tbox_group*1 tbox*1 tbox_group/idle*(1|2)

[how] query taskqueue the queue-name first. if the queue-name not exist, then there should be no job for this queue. submit idle job at background, let the consume not wait for submit idle job finished.

Signed-off-by: Tong Qunfeng taxcom@tom.com

src/lib/sched.cr | 52 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 5 deletions(-)

diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 7215df5..8fbe9aa 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -409,19 +409,26 @@ class Sched tbox_group = JobHelper.match_tbox_group(testbox) tbox = tbox_group.partition("--")[0]

• boxes = [testbox, tbox_group, tbox]

• queue_list = query_consumable_keys(tbox)
• boxes = ["sched/" + testbox,

•         "sched/" + tbox_group,
  

•         "sched/" + tbox,
  

•         "sched/" + tbox_group + "/idle"]
  

  boxes.each do |box|

•  next if queue_list.select(box).size == 0
  count.times do
  

•    job = prepare_job("sched/#{box}", testbox)
  

•    job = prepare_job(box, testbox)
    return job if job
  
    sleep(1) unless count == 1
  end
  

  end

• # when find no job at "sched/#{tbox_group}"
• # try to get from "sched/#{tbox_group}/idle"
• return get_idle_job(tbox_group, testbox)

• # when find no job, auto submit idle job at background

• spawn { auto_submit_idle_job(tbox_group) }

• return nil end

  private def prepare_job(queue_name, testbox)

@@ -571,4 +578,39 @@ class Sched

return %({"job_id": "#{job_id}", "job_state": "complete"})

end

• private def query_consumable_keys(shortest_queue_name)
• keys = [] of String
• search = "sched/" + shortest_queue_name + "*"
• response = @task_queue.query_keys(search)
• return keys unless response[0] == 200
• key_list = JSON.parse(response[1].to_json).as_a
• # add consumable keys
• key_list.each do |key|

•  queue_name = consumable_key?("#{key}")
  

•  keys << queue_name if queue_name
  

• end
• return keys
• end
• private def consumable_key?(key_name)
• if key_name =~ /(.*)/(.*)$/

   /(.*)\/(.*)$/
   the expression will match string like this:   "xx/"
   So $2 will be "", key_name = "xx/" ?

1. as default there should not be exists a key, that end with "/" 2. if there exists a key end with "/" ($2 = ""), it still works

   whether need change "/(.*)\/(.+)$/" or "/(.+)\/(.+)$/"?
   Thanks,
   Liushaofei

...

•  case $2
  

•  when "in_process"
  

•    return nil
  

•  when "ready"
  

•    return $1
  

•  when "idle"
  

•    return key_name
  

•  else
  

•    return key_name
  

•  end
  

• end
• return nil
• end

end

2.23.0