November 2020 - Compass-ci - mailweb.openeuler.org

[PATCH v2 compass-ci] sched: refactor sched class for the find job boot function
by Cao Xueliang 09 Nov '20

09 Nov '20

According to scheduler.cr API to refactor sched class. Extract find_job_boot function from sched.cr to find_job_boot.cr Extract find_next_job_boot function from sched.cr to find_next_job_boot.cr Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/lib/sched.cr | 237 +--------------------------- src/scheduler/find_job_boot.cr | 221 ++++++++++++++++++++++++++ src/scheduler/find_next_job_boot.cr | 14 ++ 3 files changed, 238 insertions(+), 234 deletions(-) create mode 100644 src/scheduler/find_job_boot.cr create mode 100644 src/scheduler/find_next_job_boot.cr diff --git a/src/lib/sched.cr b/src/lib/sched.cr index a4b12b4..6ecd95d 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -13,6 +13,9 @@ require "../scheduler/jobfile_operate" require "../scheduler/redis_client" require "../scheduler/elasticsearch_client" +require "../scheduler/find_job_boot" +require "../scheduler/find_next_job_boot" + class Sched property es property redis @@ -323,230 +326,6 @@ class Sched @es.set_job_content(job) end - private def ipxe_msg(msg) - "#!ipxe - echo ... - echo #{msg} - echo ... - reboot" - end - - private def grub_msg(msg) - "#!grub - echo ... - echo #{msg} - echo ... - reboot" - end - - private def get_boot_container(job : Job) - response = Hash(String, String).new - response["docker_image"] = "#{job.docker_image}" - response["lkp"] = "http://#{INITRD_HTTP_HOST}:#{INITRD_HTTP_PORT}" + - JobHelper.service_path("#{SRV_INITRD}/lkp/#{job.lkp_initrd_user}/lkp-#{job.arch}.cgz") - response["job"] = "http://#{SCHED_HOST}:#{SCHED_PORT}/job_initrd_tmpfs/#{job.id}/job.cgz" - - return response.to_json - end - - private def get_boot_grub(job : Job) - initrd_lkp_cgz = "lkp-#{job.os_arch}.cgz" - - response = "#!grub\n\n" - response += "linux (http,#{OS_HTTP_HOST}:#{OS_HTTP_PORT})" - response += "#{JobHelper.service_path("#{SRV_OS}/#{job.os_dir}/vmlinuz")} user=lkp" - response += " job=/lkp/scheduled/job.yaml RESULT_ROOT=/result/job" - response += " rootovl ip=dhcp ro root=#{job.kernel_append_root}\n" - - response += "initrd (http,#{OS_HTTP_HOST}:#{OS_HTTP_PORT})" - response += JobHelper.service_path("#{SRV_OS}/#{job.os_dir}/initrd.lkp") - response += " (http,#{INITRD_HTTP_HOST}:#{INITRD_HTTP_PORT})" - response += JobHelper.service_path("#{SRV_INITRD}/lkp/#{job.lkp_initrd_user}/#{initrd_lkp_cgz}") - response += " (http,#{SCHED_HOST}:#{SCHED_PORT})/job_initrd_tmpfs/" - response += "#{job.id}/job.cgz\n" - - response += "boot\n" - - return response - end - - def touch_access_key_file(job : Job) - FileUtils.touch(job.access_key_file) - end - - def boot_content(job : Job | Nil, boot_type : String) - touch_access_key_file(job) if job - - case boot_type - when "ipxe" - return job ? get_boot_ipxe(job) : ipxe_msg("No job now") - when "grub" - return job ? get_boot_grub(job) : grub_msg("No job now") - when "container" - return job ? get_boot_container(job) : Hash(String, String).new.to_json - else - raise "Not defined boot type #{boot_type}" - end - end - - def rand_queues(queues) - return queues if queues.empty? - - queues_size = queues.size - base = Random.rand(queues_size) - temp_queues = [] of String - - (0..queues_size - 1).each do |index| - temp_queues << queues[(index + base) % queues_size] - end - - return temp_queues - end - - def get_queues(host) - queues = [] of String - - queues_str = @redis.hash_get("sched/host2queues", host) - return queues unless queues_str - - queues_str.split(',', remove_empty: true) do |item| - queues << item.strip - end - - return rand_queues(queues) - end - - def get_job_from_queues(queues, testbox) - job = nil - - queues.each do |queue| - job = prepare_job("sched/#{queue}", testbox) - return job if job - end - - return job - end - - def get_job_boot(host, boot_type) - queues = get_queues(host) - job = get_job_from_queues(queues, host) - - if job - Jobfile::Operate.create_job_cpio(job.dump_to_json_any, Kemal.config.public_folder) - end - - return boot_content(job, boot_type) - end - - # auto submit a job to collect the host information - # grub hostname is link with ":", like "00:01:02:03:04:05" - # remind: if like with "-", last "-05" is treated as host number - # then hostname will be "sut-00-01-02-03-04" !!! - def submit_host_info_job(mac) - host = "sut-#{mac}" - set_host_mac(mac, host) - - Jobfile::Operate.auto_submit_job( - "#{ENV["LKP_SRC"]}/jobs/host-info.yaml", - "testbox: #{host}") - end - - def find_job_boot(env : HTTP::Server::Context) - value = env.params.url["value"] - boot_type = env.params.url["boot_type"] - - case boot_type - when "ipxe" - host = @redis.hash_get("sched/mac2host", normalize_mac(value)) - when "grub" - host = @redis.hash_get("sched/mac2host", normalize_mac(value)) - submit_host_info_job(value) unless host - when "container" - host = value - end - - get_job_boot(host, boot_type) - end - - def find_next_job_boot(env) - hostname = env.params.query["hostname"]? - mac = env.params.query["mac"]? - if !hostname && mac - hostname = @redis.hash_get("sched/mac2host", normalize_mac(mac)) - end - - get_job_boot(hostname, "ipxe") - end - - def get_testbox_boot_content(testbox, boot_type) - job = find_job(testbox) if testbox - Jobfile::Operate.create_job_cpio(job.dump_to_json_any, - Kemal.config.public_folder) if job - - return boot_content(job, boot_type) - end - - private def find_job(testbox : String, count = 1) - tbox_group = JobHelper.match_tbox_group(testbox) - tbox = tbox_group.partition("--")[0] - - queue_list = query_consumable_keys(tbox) - - boxes = ["sched/" + testbox, - "sched/" + tbox_group, - "sched/" + tbox, - "sched/" + tbox_group + "/idle"] - boxes.each do |box| - next if queue_list.select(box).size == 0 - count.times do - job = prepare_job(box, testbox) - return job if job - - sleep(1) unless count == 1 - end - end - - # when find no job, auto submit idle job at background - spawn { auto_submit_idle_job(tbox_group) } - - return nil - end - - private def prepare_job(queue_name, testbox) - response = @task_queue.consume_task(queue_name) - job_id = JSON.parse(response[1].to_json)["id"] if response[0] == 200 - job = nil - - if job_id - begin - job = @es.get_job(job_id.to_s) - rescue ex - puts "Invalid job (id=#{job_id}) in es. Info: #{ex}" - puts ex.inspect_with_backtrace - end - end - - if job - job.update({"testbox" => testbox}) - job.set_result_root - puts %({"job_id": "#{job_id}", "result_root": "/srv#{job.result_root}", "job_state": "set result root"}) - @redis.set_job(job) - end - return job - end - - private def get_idle_job(tbox_group, testbox) - job = prepare_job("sched/#{tbox_group}/idle", testbox) - - # if there has no idle job, auto submit and get 1 - if job.nil? - auto_submit_idle_job(tbox_group) - job = prepare_job("sched/#{tbox_group}/idle", testbox) - end - - return job - end - def auto_submit_idle_job(tbox_group) full_path_patterns = "#{ENV["CCI_REPOS"]}/lab-#{ENV["lab"]}/allot/idle/#{tbox_group}/*.yaml" extra_job_fields = [ @@ -561,16 +340,6 @@ class Sched extra_job_fields) if Dir.glob(full_path_patterns).size > 0 end - private def get_boot_ipxe(job : Job) - response = "#!ipxe\n\n" - response += job.initrds_uri - response += job.kernel_uri - response += job.kernel_params - response += "\nboot\n" - - return response - end - def update_job_parameter(env : HTTP::Server::Context) job_id = env.params.query["job_id"]? if !job_id diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr new file mode 100644 index 0000000..9f08288 --- /dev/null +++ b/src/scheduler/find_job_boot.cr @@ -0,0 +1,221 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +class Sched + def find_job_boot(env : HTTP::Server::Context) + value = env.params.url["value"] + boot_type = env.params.url["boot_type"] + + case boot_type + when "ipxe" + host = @redis.hash_get("sched/mac2host", normalize_mac(value)) + when "grub" + host = @redis.hash_get("sched/mac2host", normalize_mac(value)) + submit_host_info_job(value) unless host + when "container" + host = value + end + + get_job_boot(host, boot_type) + end + + # auto submit a job to collect the host information + # grub hostname is link with ":", like "00:01:02:03:04:05" + # remind: if like with "-", last "-05" is treated as host number + # then hostname will be "sut-00-01-02-03-04" !!! + def submit_host_info_job(mac) + host = "sut-#{mac}" + set_host_mac(mac, host) + + Jobfile::Operate.auto_submit_job( + "#{ENV["LKP_SRC"]}/jobs/host-info.yaml", + "testbox: #{host}") + end + + def rand_queues(queues) + return queues if queues.empty? + + queues_size = queues.size + base = Random.rand(queues_size) + temp_queues = [] of String + + (0..queues_size - 1).each do |index| + temp_queues << queues[(index + base) % queues_size] + end + + return temp_queues + end + + def get_queues(host) + queues = [] of String + + queues_str = @redis.hash_get("sched/host2queues", host) + return queues unless queues_str + + queues_str.split(',', remove_empty: true) do |item| + queues << item.strip + end + + return rand_queues(queues) + end + + def get_job_from_queues(queues, testbox) + job = nil + + queues.each do |queue| + job = prepare_job("sched/#{queue}", testbox) + return job if job + end + + return job + end + + def get_job_boot(host, boot_type) + queues = get_queues(host) + job = get_job_from_queues(queues, host) + + if job + Jobfile::Operate.create_job_cpio(job.dump_to_json_any, Kemal.config.public_folder) + end + + return boot_content(job, boot_type) + end + + private def ipxe_msg(msg) + "#!ipxe + echo ... + echo #{msg} + echo ... + reboot" + end + + private def grub_msg(msg) + "#!grub + echo ... + echo #{msg} + echo ... + reboot" + end + + private def get_boot_container(job : Job) + response = Hash(String, String).new + response["docker_image"] = "#{job.docker_image}" + response["lkp"] = "http://#{INITRD_HTTP_HOST}:#{INITRD_HTTP_PORT}" + + JobHelper.service_path("#{SRV_INITRD}/lkp/#{job.lkp_initrd_user}/lkp-#{job.arch}.cgz") + response["job"] = "http://#{SCHED_HOST}:#{SCHED_PORT}/job_initrd_tmpfs/#{job.id}/job.cgz" + + return response.to_json + end + + private def get_boot_grub(job : Job) + initrd_lkp_cgz = "lkp-#{job.os_arch}.cgz" + + response = "#!grub\n\n" + response += "linux (http,#{OS_HTTP_HOST}:#{OS_HTTP_PORT})" + response += "#{JobHelper.service_path("#{SRV_OS}/#{job.os_dir}/vmlinuz")} user=lkp" + response += " job=/lkp/scheduled/job.yaml RESULT_ROOT=/result/job" + response += " rootovl ip=dhcp ro root=#{job.kernel_append_root}\n" + + response += "initrd (http,#{OS_HTTP_HOST}:#{OS_HTTP_PORT})" + response += JobHelper.service_path("#{SRV_OS}/#{job.os_dir}/initrd.lkp") + response += " (http,#{INITRD_HTTP_HOST}:#{INITRD_HTTP_PORT})" + response += JobHelper.service_path("#{SRV_INITRD}/lkp/#{job.lkp_initrd_user}/#{initrd_lkp_cgz}") + response += " (http,#{SCHED_HOST}:#{SCHED_PORT})/job_initrd_tmpfs/" + response += "#{job.id}/job.cgz\n" + + response += "boot\n" + + return response + end + + def touch_access_key_file(job : Job) + FileUtils.touch(job.access_key_file) + end + + private def get_boot_ipxe(job : Job) + response = "#!ipxe\n\n" + response += job.initrds_uri + response += job.kernel_uri + response += job.kernel_params + response += "\nboot\n" + + return response + end + + def boot_content(job : Job | Nil, boot_type : String) + touch_access_key_file(job) if job + + case boot_type + when "ipxe" + return job ? get_boot_ipxe(job) : ipxe_msg("No job now") + when "grub" + return job ? get_boot_grub(job) : grub_msg("No job now") + when "container" + return job ? get_boot_container(job) : Hash(String, String).new.to_json + else + raise "Not defined boot type #{boot_type}" + end + end + + private def find_job(testbox : String, count = 1) + tbox_group = JobHelper.match_tbox_group(testbox) + tbox = tbox_group.partition("--")[0] + + queue_list = query_consumable_keys(tbox) + + boxes = ["sched/" + testbox, + "sched/" + tbox_group, + "sched/" + tbox, + "sched/" + tbox_group + "/idle"] + boxes.each do |box| + next if queue_list.select(box).size == 0 + count.times do + job = prepare_job(box, testbox) + return job if job + + sleep(1) unless count == 1 + end + end + + # when find no job, auto submit idle job at background + spawn { auto_submit_idle_job(tbox_group) } + + return nil + end + + private def prepare_job(queue_name, testbox) + response = @task_queue.consume_task(queue_name) + job_id = JSON.parse(response[1].to_json)["id"] if response[0] == 200 + job = nil + + if job_id + begin + job = @es.get_job(job_id.to_s) + rescue ex + puts "Invalid job (id=#{job_id}) in es. Info: #{ex}" + puts ex.inspect_with_backtrace + end + end + + if job + job.update({"testbox" => testbox}) + job.set_result_root + puts %({"job_id": "#{job_id}", "result_root": "/srv#{job.result_root}", "job_state": "set result root"}) + @redis.set_job(job) + end + return job + end + + private def get_idle_job(tbox_group, testbox) + job = prepare_job("sched/#{tbox_group}/idle", testbox) + + # if there has no idle job, auto submit and get 1 + if job.nil? + auto_submit_idle_job(tbox_group) + job = prepare_job("sched/#{tbox_group}/idle", testbox) + end + + return job + end + +end diff --git a/src/scheduler/find_next_job_boot.cr b/src/scheduler/find_next_job_boot.cr new file mode 100644 index 0000000..807a9ae --- /dev/null +++ b/src/scheduler/find_next_job_boot.cr @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +class Sched + def find_next_job_boot(env) + hostname = env.params.query["hostname"]? + mac = env.params.query["mac"]? + if !hostname && mac + hostname = @redis.hash_get("sched/mac2host", normalize_mac(mac)) + end + + get_job_boot(hostname, "ipxe") + end +end -- 2.23.0

3 4

[PATCH v2 lkp-tests] [multi-qemu] support queues parameter
by Xiao Shenwei 09 Nov '20

09 Nov '20

define queues field on multi-qemu: example: multi-qemu-0: nr_vm: 20 tbox_group: vm-2p8g queues: vm-2p8g.taishan200-2280-2s48p-256g--a1 Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- daemon/multi-qemu | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/daemon/multi-qemu b/daemon/multi-qemu index b407251f..8d9dd38d 100755 --- a/daemon/multi-qemu +++ b/daemon/multi-qemu @@ -1,6 +1,7 @@ #!/bin/sh # - tbox_group # - nr_vm +# - queues # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. @@ -8,10 +9,10 @@ multi_qemu() { export CCI_SRC=/c/compass-ci - local hostname=$tbox_group--$HOSTNAME + local hostname=$tbox_group.$HOSTNAME cd "$CACHE_DIR" - $CCI_SRC/providers/multi-qemu "$hostname" "$nr_vm" + $CCI_SRC/providers/multi-qemu -n "$hostname" -c "$nr_vm" -q "$queues" } multi_qemu -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] src/lib/web_backend.rb: "/compare_candidates" tbox_group regex error
by Zhang Yuhang 09 Nov '20

09 Nov '20

[error info] 1. /\d+$/ will be matched, such as "xxx123". 2. `index = "xxx123".index('--') || "xxx123".rindex('-')` - index equal nil. 3. `r = r[0, nil]` - raise a TypeError. Signed-off-by: Zhang Yuhang <zhangyuhang25(a)huawei.com> --- src/lib/web_backend.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index c3f6bef..55c5e4a 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -47,7 +47,7 @@ end def filter_tbox_group(es_result) result = Set.new es_result.each do |r| - if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-([a-zA-Z]+)|(\d+)$)/ + if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-(([a-zA-Z]+)|(\d+))$)/ index = r.index('--') || r.rindex('-') r = r[0, index] end -- 2.23.0

1 0

[PATCH v2 lab-z9] cluster: add cluster config file
by Zhang Yu 09 Nov '20

09 Nov '20

About config filename: "cs-s1-a102-c1": - cs : cluster - s1 : one server node - a102 : server node name - c1 : one client node Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- cluster/cs-s1-a102-c1 | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 cluster/cs-s1-a102-c1 diff --git a/cluster/cs-s1-a102-c1 b/cluster/cs-s1-a102-c1 new file mode 100644 index 0000000..af57123 --- /dev/null +++ b/cluster/cs-s1-a102-c1 @@ -0,0 +1,10 @@ +switch: Switch-P10 +ip0: 1 +nodes: + taishan200-2280-2s48p-512g--a102: + roles: [ server ] + macs: [ "44:67:47:c9:ea:37" ] + + taishan200-2280-2s48p-256g--a101: + roles: [ client ] + macs: [ "44:67:47:d7:6c:a3" ] -- 2.23.0

1 0

[PATCH v3 compass-ci] sched: simplify cluster state updating
by Ren Wen 09 Nov '20

09 Nov '20

There are three steps when updating cluster state: 1) get cluster state from redis. 2) update cluster state. 3) rewrite to redis. Before: write to redis one job info once. After: write to redis more job info once. It will save time when writing more than one job info once. Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/lib/sched.cr | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index a4b12b4..bff6090 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -58,11 +58,11 @@ class Sched return cluster_state end - # get -> modify -> set - def update_cluster_state(cluster_id, job_id, property, value) + # Update job info according to cluster id. + def update_cluster_state(cluster_id, job_id, job_info : Hash(String, String)) cluster_state = get_cluster_state(cluster_id) if cluster_state[job_id]? - cluster_state[job_id].merge!({property => value}) + cluster_state[job_id].merge!(job_info) @redis.hash_set("sched/cluster_state", cluster_id, cluster_state.to_json) end end @@ -86,9 +86,9 @@ class Sched case request_state when "abort", "finished", "failed" # update node state only - update_cluster_state(cluster_id, job_id, "state", states[request_state]) + update_cluster_state(cluster_id, job_id, {"state" => states[request_state]}) when "wait_ready" - update_cluster_state(cluster_id, job_id, "state", states[request_state]) + update_cluster_state(cluster_id, job_id, {"state" => states[request_state]}) @block_helper.block_until_finished(cluster_id) { cluster_state = sync_cluster_state(cluster_id, job_id, states[request_state]) cluster_state == "ready" || cluster_state == "abort" @@ -96,7 +96,7 @@ class Sched return cluster_state when "wait_finish" - update_cluster_state(cluster_id, job_id, "state", states[request_state]) + update_cluster_state(cluster_id, job_id, {"state" => states[request_state]}) while 1 sleep(10) cluster_state = sync_cluster_state(cluster_id, job_id, states[request_state]) @@ -110,10 +110,11 @@ class Sched direct_ips = env.params.query["direct_ips"] direct_macs = env.params.query["direct_macs"] - update_cluster_state(cluster_id, job_id, "roles", node_roles) - update_cluster_state(cluster_id, job_id, "ip", node_ip) - update_cluster_state(cluster_id, job_id, "direct_ips", direct_ips) - update_cluster_state(cluster_id, job_id, "direct_macs", direct_macs) + job_info = {"roles" => node_roles, + "ip" => node_ip, + "direct_ips" => direct_ips, + "direct_macs" => direct_macs} + update_cluster_state(cluster_id, job_id, job_info) when "roles_ip" role = "server" role_state = get_role_state(cluster_id, role) -- 2.23.0

1 0

[PATCH v7 lkp-tests 1/2] jobs/iozone-bs.yaml: combine iozone's multiple -i parameter to single
by Lu Kaiyi 07 Nov '20

07 Nov '20

[why] avoid explosion of parameter for iozone-bs.yaml Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- jobs/iozone-bs.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jobs/iozone-bs.yaml b/jobs/iozone-bs.yaml index e2cd9f48..53f1ac46 100644 --- a/jobs/iozone-bs.yaml +++ b/jobs/iozone-bs.yaml @@ -2,9 +2,7 @@ suite: iozone category: benchmark file_size: 4g -write_rewrite: true -read_reread: true -random_read_write: true +test: write, read, rand_rw block_size: - 64k -- 2.23.0

2 1

[PATCH v7 lkp-tests 2/2] tests/iozone: change the way of parsing parameter
by Lu Kaiyi 07 Nov '20

07 Nov '20

[why] iozone-bs.yaml has combined multiple parameter to single, so, iozone tool also need change the way of parsing parameter. [how] change the way of parsing parameter. small example for parsing parameter: test=" write, read, rand_rw" OLD_IFS="$IFS" IFS="," array=($test) IFS="$OLD_IFS" for ele in ${array[@]} do case $ele in "write") echo $ele ;; "read") echo $ele ;; "rand_rw") echo $ele ;; esac done output: ~/workspace/shell_learning% ./test.sh write read rand_rw if use "${array[@]}" in for loop, there will be no output. Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- tests/iozone | 66 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/tests/iozone b/tests/iozone index 88a92a18..b6a1e206 100755 --- a/tests/iozone +++ b/tests/iozone @@ -1,42 +1,52 @@ #!/bin/sh # - block_size # - file_size -# - write_rewrite -# - read_reread -# - random_read_write -# - read_backwards -# - rewrite_record -# - stride_read -# - fwrite_refwrite -# - fread_refread -# - random_mix -# - pwrite_repwrite -# - pread_repread -# - pwritev_repwritev -# - preadv_repreadv +# - test ## IOzone is a filesystem benchmark tool. The benchmark generates ## and measures a variety of file operations. . $LKP_SRC/lib/reproduce-log.sh - args="iozone" if [ -n "$block_size" ]; then args+=" -r $block_size" - [ -n "$file_size" ] && args+=" -s $file_size" - [ -n "$write_rewrite" ] && args+=" -i 0" - [ -n "$read_reread" ] && args+=" -i 1" - [ -n "$random_read_write" ] && args+=" -i 2" - [ -n "$read_backwards" ] && args+=" -i 3" - [ -n "$rewrite_record" ] && args+=" -i 4" - [ -n "$stride_read" ] && args+=" -i 5" - [ -n "$fwrite_refwrite" ] && args+=" -i 6" - [ -n "$fread_refread" ] && args+=" -i 7" - [ -n "$random_mix" ] && args+=" -i 8" - [ -n "$pwrite_repwrite" ] && args+=" -i 9" - [ -n "$pread_repread" ] && args+=" -i 10" - [ -n "$pwritev_repwritev" ] && args+=" -i 11" - [ -n "$preadv_repreadv" ] && args+=" -i 12" + [ -n "$file_size" ] && args+=" -s $file_size" + OLD_IFS="$IFS" + # reset IFS to split $test by "," and then restore default + IFS="," + array=($test) + IFS="$OLD_IFS" + for ele in ${array[@]} + do + case $ele in + "write") args+=" -i 0" + ;; + "read") args+=" -i 1" + ;; + "rand_rw") args+=" -i 2" + ;; + "backwards") args+=" -i 3" + ;; + "record") args+=" -i 4" + ;; + "stride") args+=" -i 5" + ;; + "fwrite") args+=" -i 6" + ;; + "fread") args+=" -i 7" + ;; + "rand_mix") args+=" -i 8" + ;; + "pwrite") args+=" -i 9" + ;; + "pread") args+=" -i 10" + ;; + "pwritev") args+=" -i 11" + ;; + "preadv") args+=" -i 12" + ;; + esac + done else args+=" -a" fi -- 2.23.0

2 1

[PATCH v2 compass-ci 07/10] container/mail-robot: Dockerfile
by Luan Shengde 07 Nov '20

07 Nov '20

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 container/mail-robot/Dockerfile diff --git a/container/mail-robot/Dockerfile b/container/mail-robot/Dockerfile new file mode 100644 index 0000000..4b660c1 --- /dev/null +++ b/container/mail-robot/Dockerfile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM debian + +MAINTAINER shdluan(a)163.com + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && \ + apt-get install -y git uuid-runtime curl ruby-listen ruby-json ruby-mail && \ + gem install fileutils elasticsearch activesupport -- 2.23.0

2 4

[PATCH v2 compass-ci 10/10] doc/mail-robot.md
by Luan Shengde 07 Nov '20

07 Nov '20

documentation for mail-robot service Signed-off-by: Luan Shengde <shdluan(a)163.com> --- doc/mail-robot.md | 76 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 doc/mail-robot.md diff --git a/doc/mail-robot.md b/doc/mail-robot.md new file mode 100644 index 0000000..1e65d39 --- /dev/null +++ b/doc/mail-robot.md @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ + +# email monitor robot + +## purpose + +apply account/uuid for user + uuid: used for user to submit jobs + account: used for user to check the test data + +## allowed email format: + +subject: apply account +mail content + my oss commit: commit_url +attahcment + ssh pub_key + - id_rsa.pub + +## steps overview + +1. monitor + - the monitor will listen to the mailbox for new email files + handle new email file: + check to send account + email subject: apply account + +2. assign account + - check email + check commit url + check whether there is a standard commit url + check whether the repo's url in upstream-repos + check whether the commit url valid + if the hub name is gitee.com + # authentication is needed qwhen use curl to check the url + clone the repo and check the commit + if the hub name is not gitee.com + check the commit with curl + + check pub_key + check whether there is a ssh pub_key in the email file + + it will return failure email if any check end with failure + + - generate uuid + - apply account + build user_info + - my_email + - my_name + - my_uuid + - lab + - pub_key + execute command to apply account on jumper and return the account info + curl -XGET "${JUMPER_HOST}:${JUMPER_PORT}/assign_account" -d '${user_info}' + write my_email, my_name, my_uuid, lab to the defaule config file: + + - store apply infos + apply_infos: + - my_email + - my_name + - my_commit_url + - my_login_name + - my_uuid + invoke es to store the apply infos + + - send uuid + build send uuid email message with the apply_infos + send mail + +3. continue the monitor + continue to monitor the mailbox for new email file + cycle run step 1 and 2 if matched email files + +4. service log + use the following command to get the logs + docker logs -f --tail=100 fluentd |grep mail-robot -- 2.23.0

2 2

[PATCH v7 compass-ci 3/3] assign-account: optimize answerback-email.rb
by Luan Shengde 07 Nov '20

07 Nov '20

1. disable use golbal variable apply_info [why] rubocop: Style/GlobalVars: Do not introduce global variables. [how] use local variables instead of global variables 2. fix OptionParser argument [why] when write this: opts.on('-e|--email email_address', 'appoint email address') will lead to could not identify the options ./answerback-email.rb --email xxx(a)163.com ./answerback-email.rb:72:in `<main>': invalid option: --email (OptionParser::InvalidOption) [how] write seprately like: opts.on('-e email_address', '--email email_address', 'appoint email address') 3. add my info when execute apply account command [why] when applying account, the assign-account service need to write the my info to user's default config file ~/.config/compass-ci/defaults/account.yaml my_info: - my_email - my_name - my_uuid [how] transfer the user info along with the pub_key when sending apply account request Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/assign-account/answerback-email.rb | 94 +++++++++++--------- 1 file changed, 54 insertions(+), 40 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index bb8e809..686a327 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -12,58 +12,63 @@ require 'mail' require 'set' require 'optparse' require_relative '../defconfig' +require_relative '../../lib/es_client' names = Set.new %w[ JUMPER_HOST JUMPER_PORT - SEND_MAIL_HOST_INTERNET - SEND_MAIL_PORT_INTERNET + SEND_MAIL_HOST + SEND_MAIL_PORT ] defaults = relevant_defaults(names) JUMPER_HOST = defaults['JUMPER_HOST'] || 'api.compass-ci.openeuler.org' JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 -SEND_MAIL_HOST = defaults['SEND_MAIL_HOST_INTERNET'] || 'localhost' -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT_INTERNET'] || 11312 +SEND_MAIL_HOST = defaults['SEND_MAIL_HOST'] || 'localhost' +SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 49000 -$apply_info = { +my_info = { 'my_email' => nil, + 'my_name' => nil, + 'my_uuid' => %x(uuidgen).chomp, 'my_ssh_pubkey' => nil } -def init_info(email_file) +def init_info(email_file, my_info) mail_content = Mail.read(email_file) - - $apply_info['my_email'] = mail_content.from[0] - $apply_info['my_ssh_pubkey'] = if mail_content.part[1].filename == 'id_rsa.pub' - mail_content.part[1].body.decoded.gsub(/\r|\n/, '') - end - - $apply_info + my_info['my_email'] = mail_content.from[0] + my_info['my_name'] = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') + my_info['my_ssh_pubkey'] = if mail_content.part[1].filename == 'id_rsa.pub' + mail_content.part[1].body.decoded + end end options = OptionParser.new do |opts| - opts.banner = "Usage: answerback-mail.rb [--email email] [--ssh-pubkey pub_key_file] [--raw-email email_file]\n" + opts.banner = 'Usage: answerback-mail.rb [-e|--email email] ' + opts.banner += "[-s|--ssh-pubkey pub_key_file] [-f|--raw-email email_file]\n" opts.banner += " -e or -f is required\n" opts.banner += ' -s is optional when use -e' opts.separator '' opts.separator 'options:' - opts.on('-e|--email email_address', 'appoint email address') do |email_address| - $apply_info['my_email'] = email_address + opts.on('-e email_address', '--email email_address', 'appoint email address') do |email_address| + my_info['my_email'] = email_address + # when apply account with email address, will get no user name + my_info['my_name'] = '' end - opts.on('-s|--ssh-pubkey pub_key_file', 'ssh pub_key file, enable password-less login') do |pub_key_file| - $apply_info['my_ssh_pubkey'] = File.read(pub_key_file) + opts.on('-s pub_key_file', '--ssh-pubkey pub_key_file', \ + 'ssh pub_key file, enable password-less login') do |pub_key_file| + my_info['my_ssh_pubkey'] = File.read(pub_key_file) end - opts.on('-f|--raw-email email_file', 'email file') do |email_file| - init_info(email_file) + opts.on('-f email_file', '--raw-email email_file', 'email file') do |email_file| + init_info(email_file, my_info) end - opts.on_tail('-h|--help', 'show this message') do + opts.on_tail('-h', '--help', 'show this message') do puts opts exit end @@ -71,21 +76,24 @@ end options.parse!(ARGV) -def build_message(email, acct_infos) +def build_message(email, account_info) message = <<~EMAIL_MESSAGE To: #{email} - Subject: jumper account is ready + Subject: [compass-ci] jumper account is ready Dear user: Thank you for joining us. You can use the following command to login the jumper server: - login command: - ssh -p #{acct_infos['jumper_port']} #{acct_infos['account']}@#{acct_infos['jumper_ip']} + Login command: + ssh -p #{account_info['jumper_port']} #{account_info['my_login_name']}@#{account_info['jumper_host']} + + Account password: + #{account_info['my_password']} - account password: - #{acct_infos['passwd']} + Suggest: + If you use the password to login, change it in time. regards compass-ci @@ -94,26 +102,32 @@ def build_message(email, acct_infos) return message end -def account_info(pub_key) - account_info_str = if pub_key.nil? - %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account') - else - %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") - end +def apply_account(my_info) + account_info_str = %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d '#{my_info.to_json}') JSON.parse account_info_str end -def send_account +def send_account(my_info) message = "No email address specified\n" - message += "use -e email_address add a email address\n" + message += "use -e to add a email address\n" message += 'or use -f to add a email file' - raise message if $apply_info['my_email'].nil? + raise message if my_info['my_email'].nil? - acct_info = account_info($apply_info['my_ssh_pubkey']) - - message = build_message($apply_info['my_email'], acct_info) + account_info = apply_account(my_info) + # for manually assign account, there will be no my_commit_url + # but the key my_commit_url is required for es + my_info['my_commit_url'] = '' + my_info['my_login_name'] = account_info['my_login_name'] + my_info.delete 'my_ssh_pubkey' + store_account_info(my_info) + message = build_message(my_info['my_email'], account_info) %x(curl -XPOST '#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end -send_account +def store_account_info(my_info) + es = ESClient.new(index: 'accounts') + es.put_source_by_id(my_info['my_email'], my_info) +end + +send_account(my_info) -- 2.23.0

1 0