- Compass-ci - mailweb.openeuler.org

[PATCH v2 compass-ci] service/scheduler: initialize a Sched for each API
by Wu Zhende 30 Nov '20

30 Nov '20

So can use Sched's instance variable in Sched's any interface Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 7 +++--- src/scheduler/find_job_boot.cr | 6 ++--- src/scheduler/find_next_job_boot.cr | 6 ++--- src/scheduler/request_cluster_state.cr | 14 ++++++------ src/scheduler/scheduler.cr | 31 +++++++++++++++----------- src/scheduler/submit_job.cr | 4 ++-- src/scheduler/update_job_parameter.cr | 6 ++--- 7 files changed, 40 insertions(+), 34 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 2be7d41..26ddd92 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -25,12 +25,13 @@ class Sched property redis property block_helper - def initialize + def initialize(env : HTTP::Server::Context) @es = Elasticsearch::Client.new @redis = Redis::Client.new @task_queue = TaskQueueAPI.new @block_helper = BlockHelper.new @rgc = RemoteGitClient.new + @env = env end def normalize_mac(mac : String) @@ -67,7 +68,7 @@ class Sched extra_job_fields) if Dir.glob(full_path_patterns).size > 0 end - def update_tbox_wtmp(env : HTTP::Server::Context) + def update_tbox_wtmp testbox = "" hash = Hash(String, String).new @@ -75,7 +76,7 @@ class Sched hash["time"] = time %w(mac ip job_id tbox_name tbox_state).each do |parameter| - if (value = env.params.query[parameter]?) + if (value = @env.params.query[parameter]?) case parameter when "tbox_name" testbox = value diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index b5a23c5..9805464 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -2,9 +2,9 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. class Sched - def find_job_boot(env : HTTP::Server::Context) - value = env.params.url["value"] - boot_type = env.params.url["boot_type"] + def find_job_boot + value = @env.params.url["value"] + boot_type = @env.params.url["boot_type"] case boot_type when "ipxe" diff --git a/src/scheduler/find_next_job_boot.cr b/src/scheduler/find_next_job_boot.cr index 807a9ae..09abbd5 100644 --- a/src/scheduler/find_next_job_boot.cr +++ b/src/scheduler/find_next_job_boot.cr @@ -2,9 +2,9 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. class Sched - def find_next_job_boot(env) - hostname = env.params.query["hostname"]? - mac = env.params.query["mac"]? + def find_next_job_boot + hostname = @env.params.query["hostname"]? + mac = @env.params.query["mac"]? if !hostname && mac hostname = @redis.hash_get("sched/mac2host", normalize_mac(mac)) end diff --git a/src/scheduler/request_cluster_state.cr b/src/scheduler/request_cluster_state.cr index 72e3810..79a13cc 100644 --- a/src/scheduler/request_cluster_state.cr +++ b/src/scheduler/request_cluster_state.cr @@ -6,9 +6,9 @@ class Sched # all request states: # wait_ready | abort | failed | finished | wait_finish | # write_state | roles_ip - def request_cluster_state(env) - request_state = env.params.query["state"] - job_id = env.params.query["job_id"] + def request_cluster_state + request_state = @env.params.query["state"] + job_id = @env.params.query["job_id"] cluster_id = @redis.hash_get("sched/id2cluster", job_id).not_nil! cluster_state = "" @@ -41,10 +41,10 @@ class Sched return cluster_state when "write_state" - node_roles = env.params.query["node_roles"] - node_ip = env.params.query["ip"] - direct_ips = env.params.query["direct_ips"] - direct_macs = env.params.query["direct_macs"] + node_roles = @env.params.query["node_roles"] + node_ip = @env.params.query["ip"] + direct_ips = @env.params.query["direct_ips"] + direct_macs = @env.params.query["direct_macs"] job_info = {"roles" => node_roles, "ip" => node_ip, diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr index 5f97567..98b8e2e 100644 --- a/src/scheduler/scheduler.cr +++ b/src/scheduler/scheduler.cr @@ -30,12 +30,17 @@ require "../lib/sched" module Scheduler VERSION = "0.2.0" - sched = Sched.new + add_context_storage_type(Sched) before_all do |env| + env.set "sched", Sched.new(env) env.response.headers["Connection"] = "close" end + def self.get_sched_from_env(env) + env.get("sched").as(Sched) + end + # for debug (maybe kemal debug|logger does better) def self.debug_message(env, response) puts %({"from": "#{env.request.remote_address}", "response": #{response.to_json}}) @@ -53,7 +58,7 @@ module Scheduler # /boot.xxx/host/${hostname} # /boot.yyy/mac/${mac} get "/boot.:boot_type/:parameter/:value" do |env| - response = sched.find_job_boot(env) + response = get_sched_from_env(env).find_job_boot job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id @@ -63,7 +68,7 @@ module Scheduler # /~lkp/cgi-bin/gpxelinux.cgi?hostname=:hostname&mac=:mac&last_kernel=:last_kernel get "/~lkp/cgi-bin/gpxelinux.cgi" do |env| - response = sched.find_next_job_boot(env) + response = get_sched_from_env(env).find_next_job_boot job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id @@ -75,7 +80,7 @@ module Scheduler # - echo job_id to caller # -- job_id = "0" ? means failed post "/submit_job" do |env| - job_messages = sched.submit_job(env) + job_messages = get_sched_from_env(env).submit_job job_messages.each do |job_message| puts job_message.to_json @@ -103,7 +108,7 @@ module Scheduler # curl -X PUT "http://localhost:3000/set_host_mac?hostname=wfg&mac=00-01-02-03-04-05" put "/set_host_mac" do |env| if (client_hostname = env.params.query["hostname"]?) && (client_mac = env.params.query["mac"]?) - sched.set_host_mac(client_mac, client_hostname) + get_sched_from_env(env).set_host_mac(client_mac, client_hostname) "Done" else @@ -114,7 +119,7 @@ module Scheduler # curl -X PUT "http://localhost:3000/set_host2queues?queues=vm-2p8g.aarch64&host=vm-2p8g.a…" put "/set_host2queues" do |env| if (client_queues = env.params.query["queues"]?) && (client_host = env.params.query["host"]?) - sched.set_host2queues(client_host, client_queues) + get_sched_from_env(env).set_host2queues(client_host, client_queues) "Done" else @@ -125,7 +130,7 @@ module Scheduler # curl -X PUT "http://localhost:3000/del_host_mac?mac=00-01-02-03-04-05" put "/del_host_mac" do |env| if client_mac = env.params.query["mac"]? - sched.del_host_mac(client_mac) + get_sched_from_env(env).del_host_mac(client_mac) "Done" else @@ -136,7 +141,7 @@ module Scheduler # curl -X PUT "http://localhost:3000/del_host2queues?host=vm-2p8g.aarch64" put "/del_host2queues" do |env| if client_host = env.params.query["host"]? - sched.del_host2queues(client_host) + get_sched_from_env(env).del_host2queues(client_host) "Done" else @@ -150,7 +155,7 @@ module Scheduler # ?job_file=/lkp/scheduled/job.yaml&job_state=post_run&job_id=10 # ?job_file=/lkp/scheduled/job.yaml&loadavg=0.28 0.82 0.49 1/105 3389&start_time=1587725398&end_time=1587725698&job_id=10 get "/~lkp/cgi-bin/lkp-jobfile-append-var" do |env| - puts sched.update_job_parameter(env) + puts get_sched_from_env(env).update_job_parameter "Done" end @@ -173,7 +178,7 @@ module Scheduler # response: get "server ip" from cluster state, # return "server=<server ip>". get "/~lkp/cgi-bin/lkp-cluster-sync" do |env| - response = sched.request_cluster_state(env) + response = get_sched_from_env(env).request_cluster_state debug_message(env, response) @@ -187,14 +192,14 @@ module Scheduler # get job_id from request job_id = env.params.query["job_id"]? if job_id - puts sched.close_job(job_id) + puts get_sched_from_env(env).close_job(job_id) end "Done" end get "/~lkp/cgi-bin/lkp-wtmp" do |env| - puts sched.update_tbox_wtmp(env) + puts get_sched_from_env(env).update_tbox_wtmp "Done" end @@ -205,7 +210,7 @@ module Scheduler job_id = env.params.query["job_id"].to_s if testbox && ssh_port - sched.report_ssh_port(testbox, ssh_port) + get_sched_from_env(env).report_ssh_port(testbox, ssh_port) end puts %({"job_id": "#{job_id}", "state": "set ssh port", "ssh_port": "#{ssh_port}", "tbox_name": "#{testbox}"}) diff --git a/src/scheduler/submit_job.cr b/src/scheduler/submit_job.cr index 2836275..4e78102 100644 --- a/src/scheduler/submit_job.cr +++ b/src/scheduler/submit_job.cr @@ -2,8 +2,8 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. class Sched - def submit_job(env : HTTP::Server::Context) - body = env.request.body.not_nil!.gets_to_end + def submit_job + body = @env.request.body.not_nil!.gets_to_end job_content = JSON.parse(body) job = Job.new(job_content, job_content["id"]?) diff --git a/src/scheduler/update_job_parameter.cr b/src/scheduler/update_job_parameter.cr index 0692025..cee8b9b 100644 --- a/src/scheduler/update_job_parameter.cr +++ b/src/scheduler/update_job_parameter.cr @@ -2,8 +2,8 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. class Sched - def update_job_parameter(env : HTTP::Server::Context) - job_id = env.params.query["job_id"]? + def update_job_parameter + job_id = @env.params.query["job_id"]? if !job_id return false end @@ -13,7 +13,7 @@ class Sched job_content["id"] = job_id (%w(start_time end_time loadavg job_state)).each do |parameter| - value = env.params.query[parameter]? + value = @env.params.query[parameter]? if !value || value == "" next end -- 2.23.0

2 1

[PATCH v5 compass-ci] answerback-email: fix bugs and optimizi
by Luan Shengde 30 Nov '20

30 Nov '20

1. set default option -h for tool [why] the tool will show help message if no option specified the tool will get no email address without both '-e' and '-f' [how] add '-h' as default option if no option specified options will be reset to '-h' if both '-e' and '-f' are not specified 2. fix undefined method 'empty' [why] the original value is defined nil for my_email when execute answerback-email with no -e or -f, it will throw error as: raceback (most recent call last): 2: from ./answerback-email.rb:213:in `<main>' 1: from ./answerback-email.rb:184:in `send_account' ./answerback-email.rb:155:in `check_my_email': undefined method `empty?' for nil:NilClass (NoMethodError) [how] use '.nil?' instead of '.empty?' 3. option parse error [why] if add no required value for option, it will use the field that follow as the value, example: ./answerback-email.rb -e -n name Not a standard format email: -n. [how] add value check for options that need a value Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/assign-account/answerback-email.rb | 51 +++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index c6bfdd3..ec1f5b7 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -80,7 +80,7 @@ end options = OptionParser.new do |opts| opts.banner = 'Usage: answerback-mail.rb [-e|--email email] [-n|--name name] ' - opts.banner += "[-s|--ssh-pubkey pub_key_file] [-g|--gen-sshkey] [--login y|n] [--update]\n" + opts.banner += "[-s|--ssh-pubkey pub_key_file] [-g|--gen-sshkey] [-l|--login y|n] [-u|--update]\n" opts.banner += ' answerback-mail.rb [-f|--raw-email email_file] ' opts.banner += "[-g|--gen-sshkey] [--login y|n] [--update]\n" opts.banner += " -e|-f is required when applying account or updating account\n" @@ -94,19 +94,45 @@ options = OptionParser.new do |opts| opts.separator 'options:' opts.on('-e email_address', '--email email_address', 'appoint email address') do |email_address| + unless email_address =~ /[^@]+@[\d\w]+\.[\w\d]+/ + message = "Not a standard format email: #{email_address}.\n\n" + puts message + + return false + end + my_info['my_email'] = email_address end opts.on('-n name', '--name name', 'appoint name') do |name| + unless name =~ /^\w[\w\d ]+/ + message = "Name should only contains letters, digits and spaces\n\n" + puts message + + return false + end + stdin_info['my_name'] = name end opts.on('-s pub_key_file', '--ssh-pubkey pub_key_file', \ 'ssh pub_key file, enable password-less login') do |pub_key_file| + unless File.exist? pub_key_file + message = "File not found: #{pub_key_file}.\n\n" + puts message + + return false + end stdin_info['new_ssh_pubkey'] = File.read(pub_key_file).strip end opts.on('-f email_file', '--raw-email email_file', 'email file') do |email_file| + unless File.exist? email_file + message = "Email file not found: #{email_file}.\n\n" + puts message + + return false + end mail_content = Mail.read(email_file) init_info(mail_content, email_info, my_info) end @@ -121,12 +147,16 @@ options = OptionParser.new do |opts| end opts.on('-l value', '--login value', 'enable/disable login, value: y|n') do |value| - if value.downcase == 'y' + case value + when 'y', 'Y' conf_info['enable_login'] = true - elsif value.downcase == 'n' + when 'n', 'N' conf_info['enable_login'] = false else - raise 'invalid parameter, please use y|n' + message = "-l: bad value #{value}, please use y|n\n\n" + puts message + + return false end end @@ -136,6 +166,15 @@ options = OptionParser.new do |opts| end end +# if no option specified, set default option '-h' +# if both '-e' and '-f' are not specified, will clear the ARGV, +# and use '-h' instead +if ARGV.empty? + ARGV << '-h' +elsif (['-e', '-f'] - ARGV).eql? ['-e', '-f'] + ARGV.clear + ARGV << '-h' +end options.parse!(ARGV) def apply_account(my_info, conf_info) @@ -152,7 +191,7 @@ def check_my_email(my_info) message += "use -e to add an email address for applying account\n" message += 'or use -f to add an email file' - raise message if my_info['my_email'].empty? + raise message if my_info['my_email'].nil? end def build_my_info_from_input(my_info, email_info, my_info_es, stdin_info) @@ -210,4 +249,4 @@ def store_account_info(my_info) es.put_source_by_id(my_info['my_email'], my_info) end -send_account(my_info, conf_info, email_info, my_info_es, stdin_info) +# send_account(my_info, conf_info, email_info, my_info_es, stdin_info) -- 2.23.0

1 1

[PATCH compass-ci 4/5] user-client/maintain: write job result into report
by Liu Yinsi 30 Nov '20

30 Nov '20

close the file every time after open and write it, avoid duplicate printing of logs. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- user-client/maintain/walk-os-test/walk-os-iperf-test | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/user-client/maintain/walk-os-test/walk-os-iperf-test b/user-client/maintain/walk-os-test/walk-os-iperf-test index d9d6bd6..7ed07c5 100755 --- a/user-client/maintain/walk-os-test/walk-os-iperf-test +++ b/user-client/maintain/walk-os-test/walk-os-iperf-test @@ -34,6 +34,16 @@ OS_MOUNT = %w[cifs initramfs].freeze EMAIL_ADDRESS = (ENV['USER']).to_s.freeze +def write_report(report_all) + file = File.open(LOG, 'a') + log = Logger.new(file, 'weekly') + log.formatter = proc { |_severity, datetime, _progname, msg| + "[#{datetime}]: #{msg}\n" + } + log.info(report_all) + file.close +end + def monitor(query, actions, timeout) monitor = Monitor.new monitor.overrides = query -- 2.23.0

3 6

Re: [PATCH lkp-tests] job.rb: disable submit warn
by Ren Wen 30 Nov '20

30 Nov '20

> unless File.executable?(path) >+ next if File.symlink?(path) >+ Skip all symlinks here? How about skipping 'wait' only? Thanks, RenWen > log_warn "skip non-executable #{path}" unless path =~ /\.cr$/ > next > end >-- >2.23.0 >

1 0

[PATCH compass-ci] doc/job: add memory.md
by Zhang Yu 30 Nov '20

30 Nov '20

Add a document for introducing the memory field Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- doc/job/memory.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 doc/job/memory.md diff --git a/doc/job/memory.md b/doc/job/memory.md new file mode 100644 index 0000000..d45edb5 --- /dev/null +++ b/doc/job/memory.md @@ -0,0 +1,10 @@ +# memory + +Meaning: +- Memory exists on the machine, it is the memory function of the machine. +- The memory field comes from the file which in $LAB_SRC/hosts. +- When we run a job in compass we need to know the machine's memory first, + because the memory required for running the job must be smaller than the + actual memory of the machine, otherwise there will be an error. +- Here are some examples of the memory: + 8G, 16G, 128G ... -- 2.23.0

1 0

[PATCH v2 lkp-tests] distro/common: fix cgz file not generated
by Wang Yong 30 Nov '20

30 Nov '20

if packages have already installed in OS, cgz file should not be generated, so cgz file won't be uploaded Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- distro/common | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distro/common b/distro/common index cdb531403..166ff8550 100755 --- a/distro/common +++ b/distro/common @@ -89,5 +89,7 @@ upload_to_target_dir() local dir="$1" local file="$(ls -F ${dir} | grep -v "@")" + [ -z "${file}" ] && return 0 + upload_one_curl "${dir}/${file}" "${dir}" >/dev/null } -- 2.23.0

1 0

[PATCH lkp-tests] distro/common: fix cgz file not generated
by Wang Yong 30 Nov '20

30 Nov '20

if packages have already installed in OS, cci-depends cgz file should not be generated, so cgz file won't be uploaded Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- distro/common | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distro/common b/distro/common index cdb531403..166ff8550 100755 --- a/distro/common +++ b/distro/common @@ -89,5 +89,7 @@ upload_to_target_dir() local dir="$1" local file="$(ls -F ${dir} | grep -v "@")" + [ -z "${file}" ] && return 0 + upload_one_curl "${dir}/${file}" "${dir}" >/dev/null } -- 2.23.0

1 1

[PATCH compass-ci] service/scheduler: initialize a Sched for each API
by Wu Zhende 30 Nov '20

30 Nov '20

So can use Sched's class variable in Sched's any interfacce Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 7 +++--- src/scheduler/find_job_boot.cr | 6 ++--- src/scheduler/find_next_job_boot.cr | 6 ++--- src/scheduler/request_cluster_state.cr | 14 ++++++------ src/scheduler/scheduler.cr | 31 +++++++++++++++----------- src/scheduler/submit_job.cr | 4 ++-- src/scheduler/update_job_parameter.cr | 6 ++--- 7 files changed, 40 insertions(+), 34 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 2be7d41..26ddd92 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -25,12 +25,13 @@ class Sched property redis property block_helper - def initialize + def initialize(env : HTTP::Server::Context) @es = Elasticsearch::Client.new @redis = Redis::Client.new @task_queue = TaskQueueAPI.new @block_helper = BlockHelper.new @rgc = RemoteGitClient.new + @env = env end def normalize_mac(mac : String) @@ -67,7 +68,7 @@ class Sched extra_job_fields) if Dir.glob(full_path_patterns).size > 0 end - def update_tbox_wtmp(env : HTTP::Server::Context) + def update_tbox_wtmp testbox = "" hash = Hash(String, String).new @@ -75,7 +76,7 @@ class Sched hash["time"] = time %w(mac ip job_id tbox_name tbox_state).each do |parameter| - if (value = env.params.query[parameter]?) + if (value = @env.params.query[parameter]?) case parameter when "tbox_name" testbox = value diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index b5a23c5..9805464 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -2,9 +2,9 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. class Sched - def find_job_boot(env : HTTP::Server::Context) - value = env.params.url["value"] - boot_type = env.params.url["boot_type"] + def find_job_boot + value = @env.params.url["value"] + boot_type = @env.params.url["boot_type"] case boot_type when "ipxe" diff --git a/src/scheduler/find_next_job_boot.cr b/src/scheduler/find_next_job_boot.cr index 807a9ae..09abbd5 100644 --- a/src/scheduler/find_next_job_boot.cr +++ b/src/scheduler/find_next_job_boot.cr @@ -2,9 +2,9 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. class Sched - def find_next_job_boot(env) - hostname = env.params.query["hostname"]? - mac = env.params.query["mac"]? + def find_next_job_boot + hostname = @env.params.query["hostname"]? + mac = @env.params.query["mac"]? if !hostname && mac hostname = @redis.hash_get("sched/mac2host", normalize_mac(mac)) end diff --git a/src/scheduler/request_cluster_state.cr b/src/scheduler/request_cluster_state.cr index 72e3810..79a13cc 100644 --- a/src/scheduler/request_cluster_state.cr +++ b/src/scheduler/request_cluster_state.cr @@ -6,9 +6,9 @@ class Sched # all request states: # wait_ready | abort | failed | finished | wait_finish | # write_state | roles_ip - def request_cluster_state(env) - request_state = env.params.query["state"] - job_id = env.params.query["job_id"] + def request_cluster_state + request_state = @env.params.query["state"] + job_id = @env.params.query["job_id"] cluster_id = @redis.hash_get("sched/id2cluster", job_id).not_nil! cluster_state = "" @@ -41,10 +41,10 @@ class Sched return cluster_state when "write_state" - node_roles = env.params.query["node_roles"] - node_ip = env.params.query["ip"] - direct_ips = env.params.query["direct_ips"] - direct_macs = env.params.query["direct_macs"] + node_roles = @env.params.query["node_roles"] + node_ip = @env.params.query["ip"] + direct_ips = @env.params.query["direct_ips"] + direct_macs = @env.params.query["direct_macs"] job_info = {"roles" => node_roles, "ip" => node_ip, diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr index 5f97567..98b8e2e 100644 --- a/src/scheduler/scheduler.cr +++ b/src/scheduler/scheduler.cr @@ -30,12 +30,17 @@ require "../lib/sched" module Scheduler VERSION = "0.2.0" - sched = Sched.new + add_context_storage_type(Sched) before_all do |env| + env.set "sched", Sched.new(env) env.response.headers["Connection"] = "close" end + def self.get_sched_from_env(env) + env.get("sched").as(Sched) + end + # for debug (maybe kemal debug|logger does better) def self.debug_message(env, response) puts %({"from": "#{env.request.remote_address}", "response": #{response.to_json}}) @@ -53,7 +58,7 @@ module Scheduler # /boot.xxx/host/${hostname} # /boot.yyy/mac/${mac} get "/boot.:boot_type/:parameter/:value" do |env| - response = sched.find_job_boot(env) + response = get_sched_from_env(env).find_job_boot job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id @@ -63,7 +68,7 @@ module Scheduler # /~lkp/cgi-bin/gpxelinux.cgi?hostname=:hostname&mac=:mac&last_kernel=:last_kernel get "/~lkp/cgi-bin/gpxelinux.cgi" do |env| - response = sched.find_next_job_boot(env) + response = get_sched_from_env(env).find_next_job_boot job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id @@ -75,7 +80,7 @@ module Scheduler # - echo job_id to caller # -- job_id = "0" ? means failed post "/submit_job" do |env| - job_messages = sched.submit_job(env) + job_messages = get_sched_from_env(env).submit_job job_messages.each do |job_message| puts job_message.to_json @@ -103,7 +108,7 @@ module Scheduler # curl -X PUT "http://localhost:3000/set_host_mac?hostname=wfg&mac=00-01-02-03-04-05" put "/set_host_mac" do |env| if (client_hostname = env.params.query["hostname"]?) && (client_mac = env.params.query["mac"]?) - sched.set_host_mac(client_mac, client_hostname) + get_sched_from_env(env).set_host_mac(client_mac, client_hostname) "Done" else @@ -114,7 +119,7 @@ module Scheduler # curl -X PUT "http://localhost:3000/set_host2queues?queues=vm-2p8g.aarch64&host=vm-2p8g.a…" put "/set_host2queues" do |env| if (client_queues = env.params.query["queues"]?) && (client_host = env.params.query["host"]?) - sched.set_host2queues(client_host, client_queues) + get_sched_from_env(env).set_host2queues(client_host, client_queues) "Done" else @@ -125,7 +130,7 @@ module Scheduler # curl -X PUT "http://localhost:3000/del_host_mac?mac=00-01-02-03-04-05" put "/del_host_mac" do |env| if client_mac = env.params.query["mac"]? - sched.del_host_mac(client_mac) + get_sched_from_env(env).del_host_mac(client_mac) "Done" else @@ -136,7 +141,7 @@ module Scheduler # curl -X PUT "http://localhost:3000/del_host2queues?host=vm-2p8g.aarch64" put "/del_host2queues" do |env| if client_host = env.params.query["host"]? - sched.del_host2queues(client_host) + get_sched_from_env(env).del_host2queues(client_host) "Done" else @@ -150,7 +155,7 @@ module Scheduler # ?job_file=/lkp/scheduled/job.yaml&job_state=post_run&job_id=10 # ?job_file=/lkp/scheduled/job.yaml&loadavg=0.28 0.82 0.49 1/105 3389&start_time=1587725398&end_time=1587725698&job_id=10 get "/~lkp/cgi-bin/lkp-jobfile-append-var" do |env| - puts sched.update_job_parameter(env) + puts get_sched_from_env(env).update_job_parameter "Done" end @@ -173,7 +178,7 @@ module Scheduler # response: get "server ip" from cluster state, # return "server=<server ip>". get "/~lkp/cgi-bin/lkp-cluster-sync" do |env| - response = sched.request_cluster_state(env) + response = get_sched_from_env(env).request_cluster_state debug_message(env, response) @@ -187,14 +192,14 @@ module Scheduler # get job_id from request job_id = env.params.query["job_id"]? if job_id - puts sched.close_job(job_id) + puts get_sched_from_env(env).close_job(job_id) end "Done" end get "/~lkp/cgi-bin/lkp-wtmp" do |env| - puts sched.update_tbox_wtmp(env) + puts get_sched_from_env(env).update_tbox_wtmp "Done" end @@ -205,7 +210,7 @@ module Scheduler job_id = env.params.query["job_id"].to_s if testbox && ssh_port - sched.report_ssh_port(testbox, ssh_port) + get_sched_from_env(env).report_ssh_port(testbox, ssh_port) end puts %({"job_id": "#{job_id}", "state": "set ssh port", "ssh_port": "#{ssh_port}", "tbox_name": "#{testbox}"}) diff --git a/src/scheduler/submit_job.cr b/src/scheduler/submit_job.cr index 2836275..4e78102 100644 --- a/src/scheduler/submit_job.cr +++ b/src/scheduler/submit_job.cr @@ -2,8 +2,8 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. class Sched - def submit_job(env : HTTP::Server::Context) - body = env.request.body.not_nil!.gets_to_end + def submit_job + body = @env.request.body.not_nil!.gets_to_end job_content = JSON.parse(body) job = Job.new(job_content, job_content["id"]?) diff --git a/src/scheduler/update_job_parameter.cr b/src/scheduler/update_job_parameter.cr index 0692025..cee8b9b 100644 --- a/src/scheduler/update_job_parameter.cr +++ b/src/scheduler/update_job_parameter.cr @@ -2,8 +2,8 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. class Sched - def update_job_parameter(env : HTTP::Server::Context) - job_id = env.params.query["job_id"]? + def update_job_parameter + job_id = @env.params.query["job_id"]? if !job_id return false end @@ -13,7 +13,7 @@ class Sched job_content["id"] = job_id (%w(start_time end_time loadavg job_state)).each do |parameter| - value = env.params.query[parameter]? + value = @env.params.query[parameter]? if !value || value == "" next end -- 2.23.0

3 2

[PATCH v6 compass-ci 3/3] openeuler: configure files about build image with one command
by Wang Chenglong 30 Nov '20

30 Nov '20

[Contains] openeuler-${os_version}.repo: yum configuration files files-to-exclude: list of files that need to be excluded packages-to-install: list of packages that need to install README.md: descriptive documentation Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/osimage/openeuler/README.md | 15 +++++++ container/osimage/openeuler/files-to-exclude | 11 +++++ .../openeuler/openeuler-20.03-lts.repo | 43 +++++++++++++++++++ .../osimage/openeuler/openeuler-20.09.repo | 43 +++++++++++++++++++ .../osimage/openeuler/packages-to-install | 10 +++++ 5 files changed, 122 insertions(+) create mode 100644 container/osimage/openeuler/README.md create mode 100644 container/osimage/openeuler/files-to-exclude create mode 100644 container/osimage/openeuler/openeuler-20.03-lts.repo create mode 100644 container/osimage/openeuler/openeuler-20.09.repo create mode 100644 container/osimage/openeuler/packages-to-install diff --git a/container/osimage/openeuler/README.md b/container/osimage/openeuler/README.md new file mode 100644 index 0000000..cea68f9 --- /dev/null +++ b/container/osimage/openeuler/README.md @@ -0,0 +1,15 @@ +# Use the tool to create a new openeuler-${os_version} initramfs image. + +Usage: + cd ${CCI_SRC}/rootfs/initramfs/openeuler/aarch64/${os_version}/ + ./build + +Some configuration items: +./packages-to-install + If you want to pre-install the software, you can write the package names in ./packages-to-install. + +./files-to-exclude + If you want remove some unnecessary files, you can write the names in ./files-to-exclude + +$HOME/.config/compass-ci/rootfs-passwd + Set the password for the image into this file. diff --git a/container/osimage/openeuler/files-to-exclude b/container/osimage/openeuler/files-to-exclude new file mode 100644 index 0000000..3854947 --- /dev/null +++ b/container/osimage/openeuler/files-to-exclude @@ -0,0 +1,11 @@ +/.dockerenv +/lib/modules +/usr/lib/modules +/usr/share/doc +/usr/share/man +/usr/share/info +/usr/share/i18n +/usr/share/locale +/usr/share/terminfo +/tmp/files-to-exclude +/tmp/packages-to-install diff --git a/container/osimage/openeuler/openeuler-20.03-lts.repo b/container/osimage/openeuler/openeuler-20.03-lts.repo new file mode 100644 index 0000000..f80b6b6 --- /dev/null +++ b/container/osimage/openeuler/openeuler-20.03-lts.repo @@ -0,0 +1,43 @@ +# generic-repos is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +[OS] +name=OS +baseurl=http://repo.openeuler.org/openEuler-20.03-LTS/OS/$basearch/ +enabled=1 +gpgcheck=1 +gpgkey=http://repo.openeuler.org/openEuler-20.03-LTS/OS/$basearch/RPM-GPG-KEY-openEuler + +[everything] +name=everything +baseurl=http://repo.openeuler.org/openEuler-20.03-LTS/everything/$basearch/ +enabled=1 +gpgcheck=1 +gpgkey=http://repo.openeuler.org/openEuler-20.03-LTS/everything/$basearch/RPM-GPG-KEY-openEuler + +[EPOL] +name=EPOL +baseurl=http://repo.openeuler.org/openEuler-20.03-LTS/EPOL/$basearch/ +enabled=1 +gpgcheck=1 +gpgkey=http://repo.openeuler.org/openEuler-20.03-LTS/EPOL/$basearch/RPM-GPG-KEY-openEuler + +[debuginfo] +name=debuginfo +baseurl=http://repo.openeuler.org/openEuler-20.03-LTS/debuginfo/$basearch/ +enabled=1 +gpgcheck=1 +gpgkey=http://repo.openeuler.org/openEuler-20.03-LTS/debuginfo/$basearch/RPM-GPG-KEY-openEuler + +[source] +name=source +baseurl=http://repo.openeuler.org/openEuler-20.03-LTS/source/ +enabled=1 +gpgcheck=1 +gpgkey=http://repo.openeuler.org/openEuler-20.03-LTS/source/RPM-GPG-KEY-openEuler diff --git a/container/osimage/openeuler/openeuler-20.09.repo b/container/osimage/openeuler/openeuler-20.09.repo new file mode 100644 index 0000000..6685b08 --- /dev/null +++ b/container/osimage/openeuler/openeuler-20.09.repo @@ -0,0 +1,43 @@ +# generic-repos is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +[OS] +name=OS +baseurl=http://repo.openeuler.org/openEuler-20.09/OS/$basearch/ +enabled=1 +gpgcheck=1 +gpgkey=http://repo.openeuler.org/openEuler-20.09/OS/$basearch/RPM-GPG-KEY-openEuler + +[everything] +name=everything +baseurl=http://repo.openeuler.org/openEuler-20.09/everything/$basearch/ +enabled=1 +gpgcheck=1 +gpgkey=http://repo.openeuler.org/openEuler-20.09/everything/$basearch/RPM-GPG-KEY-openEuler + +[EPOL] +name=EPOL +baseurl=http://repo.openeuler.org/openEuler-20.09/EPOL/$basearch/ +enabled=1 +gpgcheck=1 +gpgkey=http://repo.openeuler.org/openEuler-20.09/EPOL/$basearch/RPM-GPG-KEY-openEuler + +[debuginfo] +name=debuginfo +baseurl=http://repo.openeuler.org/openEuler-20.09/debuginfo/$basearch/ +enabled=1 +gpgcheck=1 +gpgkey=http://repo.openeuler.org/openEuler-20.09/debuginfo/$basearch/RPM-GPG-KEY-openEuler + +[source] +name=source +baseurl=http://repo.openeuler.org/openEuler-20.09/source/ +enabled=1 +gpgcheck=1 +gpgkey=http://repo.openeuler.org/openEuler-20.09/source/RPM-GPG-KEY-openEuler diff --git a/container/osimage/openeuler/packages-to-install b/container/osimage/openeuler/packages-to-install new file mode 100644 index 0000000..f17eee1 --- /dev/null +++ b/container/osimage/openeuler/packages-to-install @@ -0,0 +1,10 @@ +cpio +systemd-udev.aarch64 +cifs-utils +nfs-utils +hostname +curl +openssh-server +openssl +iputils +wget -- 2.23.0

1 0

[PATCH v6 compass-ci 2/3] openeuler: build openeuler initramfs image with one command
by Wang Chenglong 30 Nov '20

30 Nov '20

[Why] Use the tool to create a new openeuler-$version initramfs image with one command, and we also can customize the pre-installed software by configuring ./packages-to-install. [How] We create a link button for the user to use. $CCI_SRC/rootfs/initramfs/openeuler/aarch64/${os_version}/build --- container/osimage/openeuler/run | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 container/osimage/openeuler/run diff --git a/container/osimage/openeuler/run b/container/osimage/openeuler/run new file mode 100755 index 0000000..5d7da72 --- /dev/null +++ b/container/osimage/openeuler/run @@ -0,0 +1,28 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. ${CCI_SRC}/container/osimage/openeuler/lib +. ${CCI_SRC}/container/defconfig.sh + +export_root_passwd +DIR=$(dirname $(realpath $0)) + +cmd=( + docker run + --name initramfs_${image} + -v $DIR/create-image:/root/bin/create-image + -v $DIR/packages-to-install:/tmp/packages-to-install + -v $DIR/files-to-exclude:/tmp/files-to-exclude + -v $DIR/${image}.repo:/etc/yum.repos.d/openEuler.repo + -e ROOT_PASSWD=$ROOT_PASSWD + -e IMAGE_NAME=$IMAGE_NAME + $image + /root/bin/create-image +) + +load_docker_image +"${cmd[@]}" +cp_image_to_host +docker_rm init_docker &> /dev/null +echo "build finished" -- 2.23.0

1 0