- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci] log_system/fluentd: change the architecture
by Wu Zhende 21 Oct '20

21 Oct '20

[Why] Now we have three types of fluentd:sub-fluentd/fluentd/master-fluentd. sub-fluentd and fluentd have similar functions, can be merged. So change to two types of fluentd:sub-fluentd and fluentd. sub-fluentd only collections logs then forward to fluentd. All data processing is performed on fluentd. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/fluentd/docker-fluentd.conf | 86 +++++++------------- container/fluentd/start | 2 - container/master-fluentd/Dockerfile | 6 -- container/master-fluentd/build | 6 -- container/master-fluentd/build-depends | 1 - container/master-fluentd/docker-fluentd.conf | 48 ----------- container/master-fluentd/start | 26 ------ container/sub-fluentd/docker-fluentd.conf | 54 ++++++++++++ container/sub-fluentd/start | 12 +-- 9 files changed, 89 insertions(+), 152 deletions(-) delete mode 100644 container/master-fluentd/Dockerfile delete mode 100755 container/master-fluentd/build delete mode 100755 container/master-fluentd/build-depends delete mode 100644 container/master-fluentd/docker-fluentd.conf delete mode 100755 container/master-fluentd/start diff --git a/container/fluentd/docker-fluentd.conf b/container/fluentd/docker-fluentd.conf index 23c7585..4660988 100644 --- a/container/fluentd/docker-fluentd.conf +++ b/container/fluentd/docker-fluentd.conf @@ -2,17 +2,9 @@ workers 10 </system> -<worker 0> <source> - @type tail - path /srv/cci/serial/logs/* - pos_file /srv/cci/serial/fluentd-pos/serial.log.pos - tag serial.* - path_key serial_path - refresh_interval 1s - <parse> - @type none - </parse> + @type forward + bind 0.0.0.0 </source> <filter **> @@ -24,34 +16,16 @@ </filter> <match serial.**> - @type copy - <store> - @type rabbitmq - host 172.17.0.1 - exchange logging-test - exchange_type fanout - exchange_durable false - heartbeat 10 - <format> - @type json - </format> - </store> + @type rabbitmq + host 172.17.0.1 + exchange logging-test + exchange_type fanout + exchange_durable false + heartbeat 10 + <format> + @type json + </format> </match> -</worker> - -<worker 1-9> -<source> - @type forward - bind 0.0.0.0 -</source> - -<filter **> - @type record_transformer - enable_ruby - <record> - time ${time.strftime('%Y-%m-%d %H:%M:%S%z')} - </record> -</filter> <filter **> @type parser @@ -65,31 +39,29 @@ @type copy <store> - @type stdout - <format> - @type stdout - output_type single_value - message_key log - add_newline true - </format> + @type elasticsearch + host 172.17.0.1 + port 9202 + suppress_type_name true + flush_interval 1s + index_name ${tag} + ssl_verify false + log_es_400_reason true + with_transporter_log true </store> <store> - @type forward - flush_interval 0 - send_timeout 60 - heartbeat_type udp - heartbeat_interval 1 - recover_wait 10 - hard_timeout 60 - <server> - master-fluentd - host 172.17.0.1 - port 24225 - </server> + @type rabbitmq + host 172.17.0.1 + exchange logging-test + exchange_type fanout + exchange_durable false + heartbeat 10 + <format> + @type json + </format> </store> </match> -</worker> <label @FLUENT_LOG> <match fluent.*> diff --git a/container/fluentd/start b/container/fluentd/start index 8ad7c1e..b7f9c73 100755 --- a/container/fluentd/start +++ b/container/fluentd/start @@ -12,8 +12,6 @@ cmd = %w[ --restart=always --name fluentd -v /etc/localtime:/etc/localtime:ro - -v /srv/cci/serial/logs:/srv/cci/serial/logs:ro - -v /srv/cci/serial/fluentd-pos:/srv/cci/serial/fluentd-pos -d -u 1090:1090 -p 24224:24224/tcp diff --git a/container/master-fluentd/Dockerfile b/container/master-fluentd/Dockerfile deleted file mode 100644 index 192ece3..0000000 --- a/container/master-fluentd/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: MulanPSL-2.0+ -# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. - -FROM fluentd-base:alpine - -COPY --chown=1090:1090 docker-fluentd.conf /fluentd/etc/docker-fluentd.conf diff --git a/container/master-fluentd/build b/container/master-fluentd/build deleted file mode 100755 index 7d0291d..0000000 --- a/container/master-fluentd/build +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env ruby -# SPDX-License-Identifier: MulanPSL-2.0+ -# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -# frozen_string_literal: true - -system 'docker build -t master-fluentd:alpine .' diff --git a/container/master-fluentd/build-depends b/container/master-fluentd/build-depends deleted file mode 100755 index 29ee9d3..0000000 --- a/container/master-fluentd/build-depends +++ /dev/null @@ -1 +0,0 @@ -fluentd-base diff --git a/container/master-fluentd/docker-fluentd.conf b/container/master-fluentd/docker-fluentd.conf deleted file mode 100644 index ef97d5c..0000000 --- a/container/master-fluentd/docker-fluentd.conf +++ /dev/null @@ -1,48 +0,0 @@ -<system> - workers 10 -</system> - -<source> - @type forward - bind 0.0.0.0 -</source> - -<match **> - @type copy - - <store> - @type elasticsearch - host 172.17.0.1 - port 9202 - suppress_type_name true - flush_interval 1s - index_name ${tag} - ssl_verify false - log_es_400_reason true - with_transporter_log true - </store> - - <store> - @type rabbitmq - host 172.17.0.1 - exchange logging-test - exchange_type fanout - exchange_durable false - heartbeat 10 - <format> - @type json - </format> - </store> -</match> - -<label @FLUENT_LOG> - <match fluent.*> - @type stdout - <format> - @type stdout - output_type single_value - message_key message - add_newline true - </format> - </match> -</label> diff --git a/container/master-fluentd/start b/container/master-fluentd/start deleted file mode 100755 index 9e02980..0000000 --- a/container/master-fluentd/start +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env ruby -# SPDX-License-Identifier: MulanPSL-2.0+ -# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -# frozen_string_literal: true - -require_relative '../defconfig.rb' - -docker_rm "master-fluentd" - -cmd = %w[ - docker run - --restart=always - --name master-fluentd - -v /etc/localtime:/etc/localtime:ro - -d - -u 1090:1090 - -p 24225:24224/tcp - -p 24225:24224/udp - -e FLUENTD_CONF=docker-fluentd.conf - --log-driver json-file - --log-opt max-size=1g - master-fluentd:alpine -] -cmd += ['sh', '-c', 'umask 002 & fluentd -c /fluentd/etc/docker-fluentd.conf'] - -system(*cmd) diff --git a/container/sub-fluentd/docker-fluentd.conf b/container/sub-fluentd/docker-fluentd.conf index 24ad2e6..0092982 100644 --- a/container/sub-fluentd/docker-fluentd.conf +++ b/container/sub-fluentd/docker-fluentd.conf @@ -1,3 +1,8 @@ +<system> + workers 10 +</system> + +<worker 0> <source> @type tail path /srv/cci/serial/logs/* @@ -26,3 +31,52 @@ overflow_action block </buffer> </match> +</worker> + +<worker 1-9> +<source> + @type forward + bind 0.0.0.0 +</source> + +<match **> + @type copy + + <store> + @type stdout + <format> + @type stdout + output_type single_value + message_key log + add_newline true + </format> + </store> + + <store> + @type forward + flush_interval 0 + send_timeout 60 + heartbeat_type udp + heartbeat_interval 1 + recover_wait 10 + hard_timeout 60 + <server> + master-fluentd + host "#{ENV['FLUENTD_SERVER_HOST']}" + port "#{ENV['FLUENTD_SERVER_PORT']}" + </server> + </store> +</match> +</worker> + +<label @FLUENT_LOG> + <match fluent.*> + @type stdout + <format> + @type stdout + output_type single_value + message_key message + add_newline true + </format> + </match> +</label> diff --git a/container/sub-fluentd/start b/container/sub-fluentd/start index fa64dba..53a74c4 100755 --- a/container/sub-fluentd/start +++ b/container/sub-fluentd/start @@ -12,12 +12,13 @@ names = Set.new %w[ ] defaults = relevant_defaults(names) -FLUENTD_SERVER_PORT = defaults['FLUENTD_SERVER_PORT'] || '24224' +defaults['FLUENTD_SERVER_PORT'] ||= '24224' +env = docker_env(defaults) raise 'must config FLUENTD_SERVER_HOST' unless defaults['FLUENTD_SERVER_HOST'] docker_rm 'sub-fluentd' -cmd = %W[ +cmd = %w[ docker run --restart=always --name sub-fluentd @@ -25,12 +26,11 @@ cmd = %W[ -v /srv/cci/serial/logs:/srv/cci/serial/logs:ro -v /srv/cci/serial/fluentd-pos:/srv/cci/serial/fluentd-pos -d +] + env + %w[ -u 1090:1090 - -p 24224:24224/tcp - -p 24224:24224/udp + -p 24225:24224/tcp + -p 24225:24224/udp -e FLUENTD_CONF=docker-fluentd.conf - -e FLUENTD_SERVER_HOST=#{defaults['FLUENTD_SERVER_HOST']} - -e FLUENTD_SERVER_PORT=#{FLUENTD_SERVER_PORT} --log-driver json-file --log-opt max-size=1g sub-fluentd:alpine -- 2.23.0

1 0

[PATCH compass-ci 1/2] fix: sched: set 2 fields to job: direct_macs, direct_ips
by Ren Wen 21 Oct '20

21 Oct '20

[Why] nodes need communicate with each other when running cluster jobs. after adding 2 fileds, 'direct_ips' and 'direct_macs', can bind ips to macs. 'macs' get from cluster file ('$LKP_SRC/cluster/*'), 'ips' set by scheduler default: 192.168.222.<1..250> Now use 192.168.222.<1..250> to test, In future, use DHCP sever interface to get direct_ips. Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/lib/sched.cr | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 7215df5..d669c50 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -147,6 +147,25 @@ class Sched return YAML.parse(File.read(cluster_file_path)).as_h end + # a private-ip pools, each return 10 ips + private def get_private_ips + # ip = net_id + host_id + # host_id : 1..250 + # will divide into 10 groups used by maximum 10 cluster jobs + host_ids = [] of Int32 + 25.times do |i| + host_ids << i * 10 + 1 + end # [1, 11, ..., 241] + net_id = "192.168.222" + index = @redis.get_seqno % 25 + host_id = host_ids[index] + ips = [] of String + 10.times do |i| + ips << "#{net_id}.#{host_id + i}" + end + ips + end + def get_commit_date(job) if (job["upstream_repo"] != "") && (job["upstream_commit"] != "") data = JSON.parse(%({"git_repo": "#{job["upstream_repo"]}.git", @@ -198,6 +217,9 @@ class Sched # collect all job ids job_ids = [] of String + # an array consists of 10 ips + ips = get_private_ips + # steps for each host cluster_config.each do |host, config| tbox_group = host.to_s @@ -219,7 +241,13 @@ class Sched job["testbox"] = tbox_group job.update_tbox_group(tbox_group) job["node_roles"] = config["roles"].as_a.join(" ") - job["node_macs"] = config["macs"].as_a.join(" ") + direct_macs = config["macs"].as_a + direct_ips = [] of String + direct_macs.size.times do + direct_ips << ips.pop {raise "more than 10 macs"} + end + job["direct_macs"] = direct_macs.join(" ") + job["direct_ips"] = direct_ips.join(" ") response = add_job(job, job_id) message = (response["error"]? ? response["error"]["root_cause"] : "") -- 2.23.0

2 2

[PATCH compass-ci 2/2] feat: redis: add a function to get a seq number
by Ren Wen 21 Oct '20

21 Oct '20

[Why] Need to allocate ips to nodes when running a cluster job. A seq number 'n' used to divide ip regions, to avoid ip conflict when more than 1 cluster jobs running at same time. Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/scheduler/redis_client.cr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/scheduler/redis_client.cr b/src/scheduler/redis_client.cr index b816d67..7675056 100644 --- a/src/scheduler/redis_client.cr +++ b/src/scheduler/redis_client.cr @@ -28,6 +28,10 @@ class Redis::Client @client.hdel(key, field.to_s) end + def get_seqno + @client.incr("sched/seqno") + end + def get_job(job_id : String) job_hash = @client.hget("sched/id2job", job_id) if !job_hash -- 2.23.0

1 1

[PATCH compass-ci] feat: taskqueue: support query keys
by Tong Qunfeng 21 Oct '20

21 Oct '20

[why] support multi-queue-name for [tbox_group, host, ...] scheduler need know the sub-queue for it. like: "sched/$tbox_group-1" "sched/$tbox_group/1024" "sched/$tbox_group.taishan200-2280..." [how] - taskqueue support "/keys" API. - scheduler use @task_queue.query_keys("sched/$tbox_group*") to get the keys list. Signed-off-by: Tong Qunfeng <taxcom(a)tom.com> --- src/lib/taskqueue_api.cr | 8 ++++++++ src/taskqueue/queue.cr | 12 +++++++++++- src/taskqueue/redis_client.cr | 22 ++++++++++++++++++++++ src/taskqueue/taskqueue.cr | 15 +++++++++++++++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/lib/taskqueue_api.cr b/src/lib/taskqueue_api.cr index 99c3933..5a83a15 100644 --- a/src/lib/taskqueue_api.cr +++ b/src/lib/taskqueue_api.cr @@ -18,6 +18,14 @@ class TaskQueueAPI arrange_response(response) end + def query_keys(service_key_with_wild_char : String) + params = HTTP::Params.encode({"queue" => service_key_with_wild_char}) + client = HTTP::Client.new(@host, port: @port) + response = client.get("/keys?" + params) + client.close + arrange_response(response) + end + def consume_task(service_queue_path : String) params = HTTP::Params.encode({"queue" => service_queue_path}) response_put_api("consume", params) diff --git a/src/taskqueue/queue.cr b/src/taskqueue/queue.cr index 674606e..f98ee06 100644 --- a/src/taskqueue/queue.cr +++ b/src/taskqueue/queue.cr @@ -146,6 +146,16 @@ class TaskQueue end end + def queue_respond_keys(env) + queue_name_pattern, ext_set = queue_check_params(env, ["queue"]) + return ext_set if ext_set + + response = get_keys(queue_name_pattern[0]) + env.response.status_code = 201 if response.empty? + + return response + end + # loop try: when there has no task, return until get one or timeout # default timeout is 0.015ms, we delay for 10ms at each time private def operate_with_timeout(timeout) @@ -195,7 +205,7 @@ class TaskQueue # prefix is start with "queues", need delete it s_name = prefix.sub("#{QUEUE_NAME_BASE}/", "") move_first_task_in_redis_with_score("#{s_name}/#{uuid}", - "#{s_name}/ready") + "#{s_name}/ready") end end end diff --git a/src/taskqueue/redis_client.cr b/src/taskqueue/redis_client.cr index 27d46cf..7f8ea59 100644 --- a/src/taskqueue/redis_client.cr +++ b/src/taskqueue/redis_client.cr @@ -261,6 +261,28 @@ class TaskQueue return keys end + private def get_keys(queue_name) + query_prefix = QUEUE_NAME_BASE + "/" + query_prefix_len = query_prefix.size + search = query_prefix + queue_name + response = [] of String + + cursor = "0" + loop do + cursor, keys = @redis.scan(cursor, search, 256) + case keys + when Array(Redis::RedisValue) + keys.each do |key| + response << "#{key}"[query_prefix_len..-1] + end + end + + break if cursor == "0" + end + + return response + end + private def move_first_task_in_redis_with_score(from : String, to : String) # result was ["crystal.87230", "1600782938.9017849"] result = @redis.zrange("#{QUEUE_NAME_BASE}/#{from}", 0, 0, with_scores: true) diff --git a/src/taskqueue/taskqueue.cr b/src/taskqueue/taskqueue.cr index 98c5623..13f48db 100644 --- a/src/taskqueue/taskqueue.cr +++ b/src/taskqueue/taskqueue.cr @@ -104,6 +104,21 @@ class TaskQueue nil end + # ------------------- + # request: curl http://localhost:3060/keys? + # queue=sched* + # wild match: *, ?, [-] + # + # response: 200 ["scheda", "schedb", ...] + # 201 ## when no find + # 400 "Missing parameter <queue>" + # 413 "Query results too large keys" + get "/keys" do |env| + response = queue_respond_keys(env) + # debug_message(env, response, Time.utc) # maybe too large + response.to_json unless env.response.status_code == 201 + end + @port = (ENV.has_key?("TASKQUEUE_PORT") ? ENV["TASKQUEUE_PORT"].to_i32 : TASKQUEUE_PORT) Kemal.run(@port) end -- 2.23.0

1 0

[PATCH lkp-tests] lib/job2sh.rb: fix monitors run twice
by Wei Jihui 21 Oct '20

21 Oct '20

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/job2sh.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/job2sh.rb b/lib/job2sh.rb index 75e28ef6..9c95ff4b 100755 --- a/lib/job2sh.rb +++ b/lib/job2sh.rb @@ -142,7 +142,7 @@ class Job2sh < Job end def parse_one(ancestors, key, val, pass) - if ['pp', 'define_files'].include?(key) + if ['pp', 'define_files', 'monitors'].include?(key) return false end tabs = indent(ancestors) -- 2.23.0

1 0

[PATCH compass-ci 3/3] user-client/maintain: mail rootfs test report
by Liu Yinsi 21 Oct '20

21 Oct '20

use send-mail service to mail test report, data line must be at the top, or mail failed. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- .../maintain/walk-os-test/walk-os-iperf-test | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/user-client/maintain/walk-os-test/walk-os-iperf-test b/user-client/maintain/walk-os-test/walk-os-iperf-test index ccbff68..91a3541 100755 --- a/user-client/maintain/walk-os-test/walk-os-iperf-test +++ b/user-client/maintain/walk-os-test/walk-os-iperf-test @@ -45,6 +45,20 @@ TESTBOX = %w[vm-2p8g vm-pxe-2p8g].freeze TEST_REPORT = 'walk-test.report' TEST_YAML = 'iperf-walk-os.yaml' +def mail_report(data) + %x(curl -X POST 'localhost:11312/send_mail_yaml' -d "#{data}") +end + +def get_mail_data + mail_body = File.read(TEST_REPORT).chomp + data = " +subject: os rootfs test report +to: user@emailaddress +body: '#{mail_body}'" + + mail_report(data) +end + def logging(*log_params) log_format = '%-18s %-37s %-23s %-23s %-12s %-13s %-50s' File.open(TEST_REPORT, 'a') do |f| @@ -114,3 +128,5 @@ end logging('Job_ID', 'OS', 'Begin', 'End', 'Cost/sec', 'Job_State', 'Result_Url') MOUNT_TYPE.each { |os_mount| get_testbox(os_mount) } + +get_mail_data -- 2.23.0

2 4

[PATCH v6 compass-ci] feat: build centos initramfs image with one command
by Wang Chenglong 21 Oct '20

21 Oct '20

Use the tool to create a new centos-$version initramfs image with one command. and we also can customize the pre-installed software by configuring ./bin/packages-to-install. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/osimage/centos/README.md | 15 ++++++++ container/osimage/centos/create-image | 37 ++++++++++++++++++++ container/osimage/centos/files-to-exclude | 11 ++++++ container/osimage/centos/lib | 37 ++++++++++++++++++++ container/osimage/centos/packages-to-install | 9 +++++ container/osimage/centos/run | 27 ++++++++++++++ 6 files changed, 136 insertions(+) create mode 100644 container/osimage/centos/README.md create mode 100755 container/osimage/centos/create-image create mode 100644 container/osimage/centos/files-to-exclude create mode 100755 container/osimage/centos/lib create mode 100644 container/osimage/centos/packages-to-install create mode 100755 container/osimage/centos/run diff --git a/container/osimage/centos/README.md b/container/osimage/centos/README.md new file mode 100644 index 0000000..88166c8 --- /dev/null +++ b/container/osimage/centos/README.md @@ -0,0 +1,15 @@ +# Use the tool to create a new centos-${os_version} initramfs image. + +Usage: + cd ${CCI_SRC}/rootfs/initramfs/centos/aarch64/${os_version}/ + ./build + +Some configuration items: +./packages-to-install + If you want to pre-install the software, you can write the package names in ./packages-to-install. + +./files-to-exclude + If you want remove some unnecessary files, you can write the names in ./files-to-exclude + +$HOME/.config/compass-ci/rootfs-passwd + Set the password for the image into this file. diff --git a/container/osimage/centos/create-image b/container/osimage/centos/create-image new file mode 100755 index 0000000..a6a2766 --- /dev/null +++ b/container/osimage/centos/create-image @@ -0,0 +1,37 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +install_packages() +{ + yum repolist + yum -y install --skip-broken $(</tmp/packages-to-install) + yum clean all +} + +setup_rootfs() +{ + ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime +} + + +setup_login() { + [ -n "$ROOT_PASSWD" ] || return + echo "Changing root password" + passwd_md5=$(openssl passwd -1 "$ROOT_PASSWD") + sed -i -r "s/^root:[^:]*:(.*)/root:${passwd_md5//\//\\/}:\1/" "$ROOTFS_DIR/etc/shadow" + + sed -i 's/[# ]PermitRootLogin.*/PermitRootLogin yes/' "$ROOTFS_DIR/etc/ssh/sshd_config" +} + +pack_cgz() +{ + echo "Packing package. Please wait." + find / ! -path "/tmp/${IMAGE_NAME}" | grep -vf /tmp/files-to-exclude | cpio -o -Hnewc | gzip -9 > /tmp/${IMAGE_NAME} + chmod 644 /tmp/${IMAGE_NAME} +} + +install_packages +setup_rootfs +setup_login +pack_cgz diff --git a/container/osimage/centos/files-to-exclude b/container/osimage/centos/files-to-exclude new file mode 100644 index 0000000..3854947 --- /dev/null +++ b/container/osimage/centos/files-to-exclude @@ -0,0 +1,11 @@ +/.dockerenv +/lib/modules +/usr/lib/modules +/usr/share/doc +/usr/share/man +/usr/share/info +/usr/share/i18n +/usr/share/locale +/usr/share/terminfo +/tmp/files-to-exclude +/tmp/packages-to-install diff --git a/container/osimage/centos/lib b/container/osimage/centos/lib new file mode 100755 index 0000000..5bf1463 --- /dev/null +++ b/container/osimage/centos/lib @@ -0,0 +1,37 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +DIR="$(pwd)" +os_name="$(echo $DIR |awk -F "/" '{print $(NF - 2)}')" +os_version="$(echo $DIR |awk -F "/" '{print $NF}')" +image="${os_name}:${os_version}" +IMAGE_NAME="${os_name}-${os_version}-$(date +"%Y%m%d").cgz" + +export_root_passwd() { + local root_pwd_file="$HOME/.config/compass-ci/rootfs-passwd" + [ -f "$root_pwd_file" ] || { + echo "[INFO] Please set the password file." + echo "$HOME/.config/compass-ci/rootfs-passwd" + exit 1 + } + + export ROOT_PASSWD=$(cat "$root_pwd_file") +} + +pull_docker_image() +{ + docker pull $image + if [ $? = 0 ]; then + echo "finish downloading image" + else + echo "[ERROR] Pull docker image failed." + exit 1 + fi +} + +docker_cp_image() +{ + docker cp -a init_docker:/tmp/${IMAGE_NAME} $HOME/ + echo "result: $(ls $HOME/${IMAGE_NAME})" +} diff --git a/container/osimage/centos/packages-to-install b/container/osimage/centos/packages-to-install new file mode 100644 index 0000000..a09bc28 --- /dev/null +++ b/container/osimage/centos/packages-to-install @@ -0,0 +1,9 @@ +openssh-server +openssl +cifs-utils +hostname +curl +iproute +iputils +which +time diff --git a/container/osimage/centos/run b/container/osimage/centos/run new file mode 100755 index 0000000..51220c6 --- /dev/null +++ b/container/osimage/centos/run @@ -0,0 +1,27 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. ${CCI_SRC}/container/osimage/centos/lib +. ${CCI_SRC}/container/defconfig.sh + +export_root_passwd + +DIR=$(dirname $(realpath $0)) +cmd=( + docker run + --name init_docker + -v $DIR/create-image:/root/bin/create-image + -v $DIR/packages-to-install:/tmp/packages-to-install + -v $DIR/files-to-exclude:/tmp/files-to-exclude + -e ROOT_PASSWD=$ROOT_PASSWD + -e IMAGE_NAME=$IMAGE_NAME + $image + /root/bin/create-image +) + +pull_docker_image +${cmd[@]} +docker_cp_image +docker_rm init_docker &> /dev/null +echo "build finished" -- 2.23.0

2 1

[PATCH lkp-tests] lib: add commment in add_pp and add_monitors
by Hu Xuejiao 21 Oct '20

21 Oct '20

Signed-off-by: Hu XueJiao <huxuejiao1(a)huawei.com> --- lib/job.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/job.rb b/lib/job.rb index e2416eb8..f24435a0 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -565,6 +565,7 @@ class Job end def add_monitors + @job[comment_to_symbol('auto generated by add_monitors()')] = nil @job['monitors'] = {} for_each_in(@job, monitor_params) do |_pk, _h, m_n, _m_args| @job['monitors'][m_n] = nil @@ -572,6 +573,7 @@ class Job end def add_pp() + @job[comment_to_symbol('auto generated by add_pp()')] = nil @job["pp"] = Hash.new() init_program_options() for_each_in(@job, @referenced_programs.keys) do |_pk, h, p_n, p_args| -- 2.23.0

1 0

[PATCH v2 lkp-tests] tests/deploy-cci: determine whether GIT_SERVER exists
by Cui Yili 21 Oct '20

21 Oct '20

Determine whether GIT_SERVER exists, if not, there is no need to convert the git address. Signed-off-by: cuiyili <2268260388(a)qq.com> --- tests/deploy-cci | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/deploy-cci b/tests/deploy-cci index edfc5540..e296568f 100755 --- a/tests/deploy-cci +++ b/tests/deploy-cci @@ -3,10 +3,12 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # redirect remote-repository address to "/srv/git/gitee.com" -cat >> /etc/gitconfig <<EOF -[url "git://$GIT_SERVER/gitee.com"] - insteadOf=https://gitee.com -EOF +if [ -n "$GIT_SERVER" ]; then + cat >> /etc/gitconfig <<-EOF + [url "git://$GIT_SERVER/gitee.com"] + insteadOf=https://gitee.com + EOF +fi export initrd_http_host=$INITRD_HTTP_HOST export initrd_http_port=$INITRD_HTTP_PORT -- 2.23.0

1 0

[PATCH lkp-tests] tests/deploy-cci: determine whether GIT_SERVER exists
by Cui Yili 21 Oct '20

21 Oct '20

Determine whether GIT_SERVER exists, if not, there is no need to convert the git address. Signed-off-by: cuiyili <2268260388(a)qq.com> --- tests/deploy-cci | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/deploy-cci b/tests/deploy-cci index edfc5540..a7bce678 100755 --- a/tests/deploy-cci +++ b/tests/deploy-cci @@ -3,10 +3,12 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # redirect remote-repository address to "/srv/git/gitee.com" -cat >> /etc/gitconfig <<EOF -[url "git://$GIT_SERVER/gitee.com"] - insteadOf=https://gitee.com -EOF +if [ -n "$GIT_SERVER" ];then + cat >> /etc/gitconfig <<-EOF + [url "git://$GIT_SERVER/gitee.com"] + insteadOf=https://gitee.com + EOF +fi export initrd_http_host=$INITRD_HTTP_HOST export initrd_http_port=$INITRD_HTTP_PORT -- 2.23.0

2 1

[PATCH v3 compass-ci] /src/lib/web_backend.rb: fix "/compare_candidates" tbox_group regex bug
by Zhang Yuhang 21 Oct '20

21 Oct '20

Info: "/compare_canditates" error, due to fault regular expression Detail: Miss a couple brackets, so will match /\d+$/. Error Flow: 1. like "xxx123" will be matched 2. index = "xxx123".index('--') || "xxx123".rindex('-') 3. index equal "nil" 4. r = r[0, nil] will raise an error "TypeError (no implicit conversion from nil to integer)" Solution: Fill a couple brackets, so will match "vm-abc-0abc-abc" or "vm-abc-0abc-012". Signed-off-by: Zhang Yuhang <zhangyuhang25(a)huawei.com> --- src/lib/web_backend.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 48cfadb..fda01ca 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -47,7 +47,7 @@ end def filter_tbox_group(es_result) result = Set.new es_result.each do |r| - if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-([a-zA-Z]+)|(\d+)$)/ + if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-(([a-zA-Z]+)|(\d+))$)/ index = r.index('--') || r.rindex('-') r = r[0, index] end -- 2.23.0

3 5

[PATCH compass-ci 2/3] user-client/maintain: output rootfs test logging
by Liu Yinsi 21 Oct '20

21 Oct '20

1. monitor job state to get job runtime, start time: job_state => 'boot' end_time : job_state => 'extract-finished' 2. es-find to get job_state and result_root. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- .../maintain/walk-os-test/walk-os-iperf-test | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/user-client/maintain/walk-os-test/walk-os-iperf-test b/user-client/maintain/walk-os-test/walk-os-iperf-test index 9910723..ccbff68 100755 --- a/user-client/maintain/walk-os-test/walk-os-iperf-test +++ b/user-client/maintain/walk-os-test/walk-os-iperf-test @@ -17,6 +17,21 @@ # End Cost/sec Job_State Result_Url # 2020-10-19/18:44:48 127 finished #{SRV_HTTP_HOST}:#{SRV_HTTP_PORT}/#{result_root} +require 'set' +require 'open3' +require "#{ENV['LKP_SRC']}/lib/monitor" +require "#{ENV['CCI_SRC']}/container/defconfig.rb" + +names = Set.new %w[ + SRV_HTTP_HOST + SRV_HTTP_PORT +] + +defaults = relevant_defaults(names) + +SRV_HTTP_HOST = defaults['SRV_HTTP_HOST'] || Open3.capture3('curl ip.sb')[0].chomp +SRV_HTTP_PORT = defaults['SRV_HTTP_PORT'] || '11300' + TEST_OS = [ 'openeuler aarch64 20.03', 'centos aarch64 7.6.1810', @@ -30,6 +45,44 @@ TESTBOX = %w[vm-2p8g vm-pxe-2p8g].freeze TEST_REPORT = 'walk-test.report' TEST_YAML = 'iperf-walk-os.yaml' +def logging(*log_params) + log_format = '%-18s %-37s %-23s %-23s %-12s %-13s %-50s' + File.open(TEST_REPORT, 'a') do |f| + f.puts format(log_format, *log_params) + end +end + +def monitor(query, actions, timeout) + monitor = Monitor.new + monitor.overrides = query + monitor.action = actions + monitor.run(timeout) +end + +def monitor_job(job_id, job_state) + state_code = monitor({ 'job_id' => job_id, 'job_state' => job_state }, { 'stop' => true }, '600s') + Time.now.to_s.split[0..1].join '/' if state_code.zero? +end + +def es_find(job_id, search_field) + %x(es-find id=#{job_id}| grep #{search_field}| awk -F '"' '{print $4}').chomp +end + +def get_log_params(os_array, message) + job_id = message.split('=')[1].chomp + os_params = os_array.join '/' + s_time = monitor_job(job_id, 'boot') + e_time = monitor_job(job_id, 'extract_finished') + return if s_time.nil? || e_time.nil? + + c_time = (s_time - e_time).floor + job_state = es_find(job_id, 'job_state') + result_root = es_find(job_id, 'result_root') + result_url = SRV_HTTP_HOST.concat ':', SRV_HTTP_PORT, result_root + + logging(job_id, os_params, s_time, e_time, c_time, job_state, result_url) +end + def submit_job(os_array, testbox) os = os_array[0] os_arch = os_array[1] @@ -42,6 +95,8 @@ def submit_job(os_array, testbox) # Process.fork do # %x(./my-qemu.sh >/dev/null 2>&1) # end + + get_log_params(os_array, message) end def get_os_array(os_mount, testbox) @@ -56,4 +111,6 @@ def get_testbox(os_mount) TESTBOX.each { |testbox| get_os_array(os_mount, testbox) } end +logging('Job_ID', 'OS', 'Begin', 'End', 'Cost/sec', 'Job_State', 'Result_Url') + MOUNT_TYPE.each { |os_mount| get_testbox(os_mount) } -- 2.23.0

3 3

[PATCH v2 compass-ci] sched:add monitors depends to job
by Cao Xueliang 21 Oct '20

21 Oct '20

[why] It may add monitors field in job yaml, sched will parse the monitors field and add depends to the initrd string. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/lib/job.cr | 52 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 2cb8a2c..3c0b64a 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -281,7 +281,7 @@ class Job end private def set_kernel_uri - self["kernel_uri"] = "kernel #{OS_HTTP_PREFIX}" + + self["kernel_uri"] = "kernel #{OS_HTTP_PREFIX}" + "#{JobHelper.service_path("#{SRV_OS}/#{os_dir}/vmlinuz")}" end @@ -393,29 +393,45 @@ class Job private def set_depends_initrd initrd_deps_arr = Array(String).new initrd_pkg_arr = Array(String).new + + get_depends_initrd(get_program_params(), initrd_deps_arr, initrd_pkg_arr) + + self["initrd_deps"] = initrd_deps_arr.uniq.join(" ") + self["initrd_pkg"] = initrd_pkg_arr.uniq.join(" ") + end + + private def get_program_params + program_params = Hash(String, JSON::Any).new + if @hash["monitors"]? + program_params.merge!(@hash["monitors"].as_h) + end + + if @hash["pp"]? + program_params.merge!(@hash["pp"].as_h) + end + + return program_params + end + + private def get_depends_initrd(program_params, initrd_deps_arr, initrd_pkg_arr) initrd_http_prefix = "http://#{INITRD_HTTP_HOST}:#{INITRD_HTTP_PORT}" mount_type = os_mount == "cifs" ? "nfs" : os_mount.dup - if @hash["pp"]? - program_params = @hash["pp"].as_h - program_params.keys.each do |program| - if program =~ /^(.*)-\d+$/ - program = $1 - end - deps_dest_file = "#{SRV_INITRD}/deps/#{mount_type}/#{os_dir}/#{program}.cgz" - pkg_dest_file = "#{SRV_INITRD}/pkg/#{mount_type}/#{os_dir}/#{program}.cgz" - if File.exists?("#{user_lkp_src}/distro/depends/#{program}") - initrd_deps_arr << "#{initrd_http_prefix}" + JobHelper.service_path(deps_dest_file) - end + program_params.keys.each do |program| + if program =~ /^(.*)-\d+$/ + program = $1 + end - if File.exists?("#{user_lkp_src}/pkg/#{program}") - initrd_pkg_arr << "#{initrd_http_prefix}" + JobHelper.service_path(pkg_dest_file) - end + deps_dest_file = "#{SRV_INITRD}/deps/#{mount_type}/#{os_dir}/#{program}.cgz" + pkg_dest_file = "#{SRV_INITRD}/pkg/#{mount_type}/#{os_dir}/#{program}.cgz" + + if File.exists?(deps_dest_file) + initrd_deps_arr << "#{initrd_http_prefix}" + JobHelper.service_path(deps_dest_file) + end + if File.exists?(pkg_dest_file) + initrd_pkg_arr << "#{initrd_http_prefix}" + JobHelper.service_path(pkg_dest_file) end end - - self["initrd_deps"] = initrd_deps_arr.uniq.join(" ") - self["initrd_pkg"] = initrd_pkg_arr.uniq.join(" ") end def update_tbox_group(tbox_group) -- 2.23.0

1 0

[PATCH v4 compass-ci 1/6] container/send-internet-mail: mail-post.rb
by Luan Shengde 21 Oct '20

21 Oct '20

get request for send mail analysis email data invoke send-internet-mail to send the mail Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- lib/mail-post.rb | 66 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100755 lib/mail-post.rb diff --git a/lib/mail-post.rb b/lib/mail-post.rb new file mode 100755 index 0000000..889fb23 --- /dev/null +++ b/lib/mail-post.rb @@ -0,0 +1,66 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'sinatra' +require 'json' +require 'yaml' +require 'open3' +# require_relative "#{ENV['SMTP_FILE']}" + +set :bind, '0.0.0.0' +set :port, ENV['SEND_MAIL_PORT'] + +post '/send_mail_yaml' do + data = YAML.safe_load request.body.read + raise TypeError, data, 'request data type error' unless data.class.eql? Hash + + mail_info = { + 'references' => data['references'], + 'from' => data['from'], + 'subject' => data['subject'], + 'to' => data['to'], + 'body' => data['body'] + } + check_send_mail(mail_info) +end + +post '/send_mail_text' do + data = Mail.read_from_string(request.body.read) + + mail_info = { + 'references' => data.references, + 'from' => data.from, + 'subject' => data.subject, + 'to' => data.to, + 'body' => data.body.decoded + } + check_send_mail(mail_info) +end + +def check_send_mail(mail_info) + if ENV['SMTP_REGION'] == 'internet-smtp' + mail_info['from'] = ENV['ROBOT_EMAIL_ADDRESS'] + else + mail_info['from'] = ENV['ROBOT_EMAIL_ADDRESS'] if mail_info['from'].nil? + end + + raise 'no subject.' if mail_info['subject'].nil? + raise 'no email address.' if mail_info['to'].nil? + raise 'no email content.' if mail_info['body'].nil? + + send_mail(mail_info) +end + +def send_mail(mail_info) + setup_smtp + mail = Mail.new do + references mail_info['references'] + from mail_info['from'] + subject mail_info['subject'] + to mail_info['to'] + body mail_info['body'] + end + mail.deliver! +end -- 2.23.0

2 7

[PATCH v4 compass-ci 3/6] container/send-internet-mail: run.rb
by Luan Shengde 21 Oct '20

21 Oct '20

invoke mail-post/internet-smtp to run the send mail service Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/send-internet-mail/run.rb | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 container/send-internet-mail/run.rb diff --git a/container/send-internet-mail/run.rb b/container/send-internet-mail/run.rb new file mode 100644 index 0000000..efda5b9 --- /dev/null +++ b/container/send-internet-mail/run.rb @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby + +require 'sinatra' + +require "#{ENV['CCI_SRC']}/lib/internet-smtp" +require "#{ENV['CCI_SRC']}/lib/mail-post" -- 2.23.0

2 1

[PATCH v5 compass-ci] container/assign-account: answerback-email.rb
by Luan Shengde 21 Oct '20

21 Oct '20

add parameters for answerback-email -e,--email email_address: add email address -s,--ssh-pubkey pub_key_file: add a ssh login pub_key -f,--raw-email email_file: add email address[, pub_key] via email file Usage: ruby answerback-email.rb -e email_address [-s pub_key_file] ruby answerback-email.rb -f email_file why: easier for administrator to manully assign jumper account for user with the parameters Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/answerback-email.rb | 179 ++++++++----------- 1 file changed, 77 insertions(+), 102 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index b5283ba..a3108cb 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -3,74 +3,90 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true -=begin - -repo_list: - all repos url list from upstream-repos.git - -API: -call graph: -read_mail_content -send_account_request - email_addr - get email address - email_message_id - get email message_id - check_email_available - check email available - pub_key_value - get pub key - account - send apply account request and return account info - build_message - build email message - send_mail - call send_mail to send mail whit build message - -the returned data for account_info like: -{ - "account" => "guest", - "passwd" => "Use pub_key to login", - "jumper_ip" => "10.10.10.10", - "jumper_port" => "10000" -} -=end +# xx --email xxx --login --ssh-pubkey xxx --raw-email email-file +# samba mount +# ssh logshn (huawei, ) (install pubkey / send password) require 'json' require 'mail' require 'set' -require_relative '../defconfig.rb' +require 'optparse' +require_relative '../defconfig' names = Set.new %w[ - JUMPER_IP + JUMPER_HOST JUMPER_PORT - CRYSTAL_INTRANET - SEND_MAIL_PORT + SEND_MAIL_HOST + SEND_INTERNET_MAIL_PORT ] defaults = relevant_defaults(names) -JUMPER_IP = defaults['JUMPER_IP'] -JUMPER_PORT = defaults['JUMPER_PORT'] -CRYSTAL_INTRANET = defaults['CRYSTAL_INTRANET'] -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] +JUMPER_HOST = defaults['JUMPER_HOST'] || '183.134.196.212' +JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 +SEND_MAIL_HOST = defaults['SEND_MAIL_HOST'] || localhost +SEND_MAIL_PORT = defaults['SEND_INTERNET_MAIL_PORT'] || 11312 + +apply_info = { + 'my_email' => nil, + 'my_pub_key' => nil +} + +def init_info(email_file, apply_info) + mail_content = Mail.read(email_file) + + my_email = mail_content.from[0] + my_pub_key = mail_content.part[1].body.decoded.gsub(/\r|\n/, '') if mail_content.part[1].filename == 'id_rsa.pub' + + apply_info['my_email'] = my_email + apply_info['my_pub_key'] = my_pub_key + + apply_info +end + +options = OptionParser.new do |opts| + opts.banner = "Usage: answerback-mail.rb [--email email] [--ssh-pubkey pub_key_file] [--raw-email email_file]\n" + opts.banner += " -e or -f is required\n" + opts.banner += ' -s is optional when use -e' + + opts.separator '' + opts.separator 'options:' + + opts.on('-e email_address', '--email email_address', 'appoint email address') do |email_address| + apply_info['my_email'] = email_address + end + + opts.on('-s pub_key_file', '--ssh-pubkey pub_key_file', 'ssh pub_key file, enable keyless login') do |pub_key_file| + apply_info['my_pub_key'] = File.read(pub_key_file) + end + + opts.on('-f email_file', '--raw-email email_file', 'email file') do |email_file| + init_info(email_file, apply_info) + end + + opts.on_tail('-h', '--help', 'show this message') do + puts opts + exit + end +end -def build_message(email, message_id, infos) +options.parse!(ARGV) + +def build_message(email, acct_infos) message = <<~EMAIL_MESSAGE To: #{email} - Message-ID: #{message_id} Subject: jumper account is ready - Dear #{email} + Dear user: Thank you for joining us. You can use the following command to login the jumper server: login command: - ssh -p #{infos['jumper_port']} #{infos['account']}@#{infos['jumper_ip']} + ssh -p #{acct_infos['jumper_port']} #{acct_infos['account']}@#{acct_infos['jumper_ip']} - account passwd: - account_password: #{infos['passwd']} + account password: + #{acct_infos['passwd']} regards compass-ci @@ -79,67 +95,26 @@ def build_message(email, message_id, infos) return message end -def email_addr(mail_content) - msg = 'not an applying account email' - - raise msg unless mail_content.subject =~ /apply ssh account/i - - email = mail_content.from.join(',') - - return email -end - -# def check_email_available(mail_content, email) -# oos_list = File.read('/c/upstream-repos/repo_list').split(/\n/) -# url = mail_content.body.decoded.split(/\n/).find { |line| line =~ /https?:\/\// } -# base_url = url.split('/')[0,5].join('/') -# message = 'The url is not in upstream repo_list' -# -# raise message unless oos_list.include? base_url -# -# url_fdback = %x(curl #{url}) -# email_index = url_fdback.index email -# -# message = 'No commit info found from the url for the email' -# raise message unless email_index -# end - -def email_message_id(mail_content) - message_id = mail_content.message_id - return message_id -end - -def pub_key_value(mail_content) - pub_key = mail_content.body.decoded.split(/\n/).find { |line| line =~ /ssh-rsa/ } - return pub_key -end - def account_info(pub_key) - account_info_str = %x(curl -XGET '#{JUMPER_IP}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") - account_info = JSON.parse account_info_str - - return account_info + account_info_str = if pub_key.nil? + %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account') + else + %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") + end + JSON.parse account_info_str end -def send_account(mail_content) - email = email_addr(mail_content) - message_id = email_message_id(mail_content) - # check_email_available(mail_content, email) +def send_account(apply_info) + message = "No email address specified\n" + message += "use -e email_address add a email address\n" + message += 'or use -f to add a email file' + raise message if apply_info['my_email'].nil? - pub_key = pub_key_value(mail_content) - acct_info = account_info(pub_key) - - message = build_message(email, message_id, acct_info) - - %x(curl -XPOST '#{CRYSTAL_INTRANET}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") -end + acct_info = account_info(apply_info['my_pub_key']) -def read_mail_content(mail_file) - mail_content = Mail.read(mail_file) + message = build_message(apply_info['my_email'], acct_info) - return mail_content + %x(curl -XPOST '#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end -mail_file = ARGV[0] -mail_content = read_mail_content(mail_file) -send_account(mail_content) +send_account(apply_info) -- 2.23.0

3 9

[PATCH lkp-tests] lib/monitor: add stop_query as rule to stop monitor
by Wu Zhende 21 Oct '20

21 Oct '20

[Why] In some cases,the query for exiting the monitor may not be the same as the query for monitoring. To meet this requirement, an additional zone stop_query is added for processing. Example: query = {"job_id" => 1}, means I want to monitoring the job 1. stop_query = {"job_state" => "extract_finished"}, means I want to stop monitoring when I get a log, "job_state" equals to "extract_finished". Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/monitor.rb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/monitor.rb b/lib/monitor.rb index baf172d3..fcf7e220 100755 --- a/lib/monitor.rb +++ b/lib/monitor.rb @@ -11,7 +11,7 @@ require 'eventmachine' require 'json' class Monitor - attr_accessor :monitor_url, :query, :overrides, :action, :job, :result + attr_accessor :monitor_url, :query, :overrides, :action, :job, :result, :stop_query def initialize(monitor_url = '', query = {}, action = {}) @monitor_url = monitor_url @@ -26,6 +26,7 @@ class Monitor @defaults = {} load_default @result = [] + @stop_query = {} end def load_default @@ -100,6 +101,13 @@ class Monitor EM.stop end + def stop(data, web_socket) + @stop_query.each do |key, value| + return false unless data[key] == value + end + stop_em(web_socket) + end + def run(timeout = nil) merge_overrides field_check @@ -128,7 +136,7 @@ class Monitor output(data) connect(data, ws) - stop_em(ws) if @action['stop'] + stop(data, ws) if @action['stop'] end ws.on :close do |event| -- 2.23.0

1 0

[PATCH lkp-tests 2/2] spec/submit: add monitors to spec results
by Wei Jihui 21 Oct '20

21 Oct '20

according to spec results, add monitors args. spec cmd: rspec spec/submit_spec.rb Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- spec/submit/ftq/ftq-1000-2-100000ss-add.yaml | 3 +- spec/submit/ftq/ftq-10000-2-100000ss-add.yaml | 3 +- spec/submit/ftq/ftq-20-2-6000ss-cache.yaml | 3 +- ...-68000000-1-65535-set_get-never-never.yaml | 29 ++++++++++++++++++- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/spec/submit/ftq/ftq-1000-2-100000ss-add.yaml b/spec/submit/ftq/ftq-1000-2-100000ss-add.yaml index 3849bb30..3f26d1ae 100644 --- a/spec/submit/ftq/ftq-1000-2-100000ss-add.yaml +++ b/spec/submit/ftq/ftq-1000-2-100000ss-add.yaml @@ -7,9 +7,9 @@ samples: 100000ss ftq: test: add freq: 1000 +lab: spec_lab testbox: vm-2p8g--spec_submit tbox_group: vm-2p8g--spec_submit -lab: spec_lab provider: qemu template: kvm nr_node: 1 @@ -26,3 +26,4 @@ pp: freq: 1000 test: add cpufreq_governor: +monitors: {} diff --git a/spec/submit/ftq/ftq-10000-2-100000ss-add.yaml b/spec/submit/ftq/ftq-10000-2-100000ss-add.yaml index 2d9785e4..87945a00 100644 --- a/spec/submit/ftq/ftq-10000-2-100000ss-add.yaml +++ b/spec/submit/ftq/ftq-10000-2-100000ss-add.yaml @@ -7,9 +7,9 @@ samples: 100000ss ftq: test: add freq: 10000 +lab: spec_lab testbox: vm-2p8g--spec_submit tbox_group: vm-2p8g--spec_submit -lab: spec_lab provider: qemu template: kvm nr_node: 1 @@ -26,3 +26,4 @@ pp: freq: 10000 test: add cpufreq_governor: +monitors: {} diff --git a/spec/submit/ftq/ftq-20-2-6000ss-cache.yaml b/spec/submit/ftq/ftq-20-2-6000ss-cache.yaml index 6c915e93..247cdc37 100644 --- a/spec/submit/ftq/ftq-20-2-6000ss-cache.yaml +++ b/spec/submit/ftq/ftq-20-2-6000ss-cache.yaml @@ -7,9 +7,9 @@ samples: 6000ss ftq: test: cache freq: 20 +lab: spec_lab testbox: vm-2p8g--spec_submit tbox_group: vm-2p8g--spec_submit -lab: spec_lab provider: qemu template: kvm nr_node: 1 @@ -26,3 +26,4 @@ pp: freq: 20 test: cache cpufreq_governor: +monitors: {} diff --git a/spec/submit/redis/redis-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml b/spec/submit/redis/redis-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml index 43132b8f..466a2054 100644 --- a/spec/submit/redis/redis-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml +++ b/spec/submit/redis/redis-1-cs-localhost-even-1024-68000000-5-3-4-68000000-1-65535-set_get-never-never.yaml @@ -25,10 +25,10 @@ if role client: requests: 68000000 n_pipeline: 3 key_len: 68000000 +lab: spec_lab testbox: vm-2p8g--spec_submit tbox_group: vm-2p8g--spec_submit node_roles: server client -lab: spec_lab provider: qemu template: kvm nr_node: 1 @@ -86,3 +86,30 @@ pp: nr_processes: 4 cpu_node_bind: even cpufreq_governor: +monitors: + kmsg: + boot-time: + uptime: + iostat: + heartbeat: + vmstat: + numa-numastat: + numa-vmstat: + numa-meminfo: + proc-vmstat: + proc-stat: + meminfo: + slabinfo: + interrupts: + lock_stat: + latency_stats: + softirqs: + bdi_dev_mapping: + diskstats: + nfsstat: + cpuidle: + cpufreq-stats: + sched_debug: + perf-stat: + mpstat: + perf-profile: -- 2.23.0

1 0

[PATCH lkp-tests 1/2] job: add monitors to job
by Wei Jihui 21 Oct '20

21 Oct '20

find all monitors and add to job scheduler will use it to get monitors program's cgz. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/job.rb | 7 +++++++ sbin/submit | 1 + 2 files changed, 8 insertions(+) diff --git a/lib/job.rb b/lib/job.rb index caa76c4b..e2416eb8 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -564,6 +564,13 @@ class Job return option_value end + def add_monitors + @job['monitors'] = {} + for_each_in(@job, monitor_params) do |_pk, _h, m_n, _m_args| + @job['monitors'][m_n] = nil + end + end + def add_pp() @job["pp"] = Hash.new() init_program_options() diff --git a/sbin/submit b/sbin/submit index a749ea20..8b46db32 100755 --- a/sbin/submit +++ b/sbin/submit @@ -108,6 +108,7 @@ ARGV.each do |jobfile| raise 'Please configure SCHED_PORT' unless job['SCHED_PORT'] job.add_pp + job.add_monitors job.add_define_files if opt_auto_define_files # get job shell function -- 2.23.0

1 0

[PATCH v6 compass-ci] container/assign-account: answerback-email.rb
by Luan Shengde 21 Oct '20

21 Oct '20

add parameters for answerback-email easier for administrator to manully assign jumper account for user with the parameters -e,--email email_address: add email address -s,--ssh-pubkey pub_key_file: add a ssh login pub_key -f,--raw-email email_file: add email address[, pub_key] via email file Usage: ruby answerback-email.rb -e email_address [-s pub_key_file] ruby answerback-email.rb -f email_file Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/answerback-email.rb | 178 ++++++++----------- 1 file changed, 76 insertions(+), 102 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index b5283ba..bb8e809 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -3,74 +3,89 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true -=begin - -repo_list: - all repos url list from upstream-repos.git - -API: -call graph: -read_mail_content -send_account_request - email_addr - get email address - email_message_id - get email message_id - check_email_available - check email available - pub_key_value - get pub key - account - send apply account request and return account info - build_message - build email message - send_mail - call send_mail to send mail whit build message - -the returned data for account_info like: -{ - "account" => "guest", - "passwd" => "Use pub_key to login", - "jumper_ip" => "10.10.10.10", - "jumper_port" => "10000" -} -=end +# xx --email xxx --login --ssh-pubkey xxx --raw-email email-file +# samba mount +# ssh logshn (huawei, ) (install pubkey / send password) require 'json' require 'mail' require 'set' -require_relative '../defconfig.rb' +require 'optparse' +require_relative '../defconfig' names = Set.new %w[ - JUMPER_IP + JUMPER_HOST JUMPER_PORT - CRYSTAL_INTRANET - SEND_MAIL_PORT + SEND_MAIL_HOST_INTERNET + SEND_MAIL_PORT_INTERNET ] defaults = relevant_defaults(names) -JUMPER_IP = defaults['JUMPER_IP'] -JUMPER_PORT = defaults['JUMPER_PORT'] -CRYSTAL_INTRANET = defaults['CRYSTAL_INTRANET'] -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] +JUMPER_HOST = defaults['JUMPER_HOST'] || 'api.compass-ci.openeuler.org' +JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 +SEND_MAIL_HOST = defaults['SEND_MAIL_HOST_INTERNET'] || 'localhost' +SEND_MAIL_PORT = defaults['SEND_MAIL_PORT_INTERNET'] || 11312 + +$apply_info = { + 'my_email' => nil, + 'my_ssh_pubkey' => nil +} + +def init_info(email_file) + mail_content = Mail.read(email_file) + + $apply_info['my_email'] = mail_content.from[0] + $apply_info['my_ssh_pubkey'] = if mail_content.part[1].filename == 'id_rsa.pub' + mail_content.part[1].body.decoded.gsub(/\r|\n/, '') + end + + $apply_info +end + +options = OptionParser.new do |opts| + opts.banner = "Usage: answerback-mail.rb [--email email] [--ssh-pubkey pub_key_file] [--raw-email email_file]\n" + opts.banner += " -e or -f is required\n" + opts.banner += ' -s is optional when use -e' + + opts.separator '' + opts.separator 'options:' + + opts.on('-e|--email email_address', 'appoint email address') do |email_address| + $apply_info['my_email'] = email_address + end + + opts.on('-s|--ssh-pubkey pub_key_file', 'ssh pub_key file, enable password-less login') do |pub_key_file| + $apply_info['my_ssh_pubkey'] = File.read(pub_key_file) + end + + opts.on('-f|--raw-email email_file', 'email file') do |email_file| + init_info(email_file) + end + + opts.on_tail('-h|--help', 'show this message') do + puts opts + exit + end +end + +options.parse!(ARGV) -def build_message(email, message_id, infos) +def build_message(email, acct_infos) message = <<~EMAIL_MESSAGE To: #{email} - Message-ID: #{message_id} Subject: jumper account is ready - Dear #{email} + Dear user: Thank you for joining us. You can use the following command to login the jumper server: login command: - ssh -p #{infos['jumper_port']} #{infos['account']}@#{infos['jumper_ip']} + ssh -p #{acct_infos['jumper_port']} #{acct_infos['account']}@#{acct_infos['jumper_ip']} - account passwd: - account_password: #{infos['passwd']} + account password: + #{acct_infos['passwd']} regards compass-ci @@ -79,67 +94,26 @@ def build_message(email, message_id, infos) return message end -def email_addr(mail_content) - msg = 'not an applying account email' - - raise msg unless mail_content.subject =~ /apply ssh account/i - - email = mail_content.from.join(',') - - return email -end - -# def check_email_available(mail_content, email) -# oos_list = File.read('/c/upstream-repos/repo_list').split(/\n/) -# url = mail_content.body.decoded.split(/\n/).find { |line| line =~ /https?:\/\// } -# base_url = url.split('/')[0,5].join('/') -# message = 'The url is not in upstream repo_list' -# -# raise message unless oos_list.include? base_url -# -# url_fdback = %x(curl #{url}) -# email_index = url_fdback.index email -# -# message = 'No commit info found from the url for the email' -# raise message unless email_index -# end - -def email_message_id(mail_content) - message_id = mail_content.message_id - return message_id -end - -def pub_key_value(mail_content) - pub_key = mail_content.body.decoded.split(/\n/).find { |line| line =~ /ssh-rsa/ } - return pub_key -end - def account_info(pub_key) - account_info_str = %x(curl -XGET '#{JUMPER_IP}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") - account_info = JSON.parse account_info_str - - return account_info + account_info_str = if pub_key.nil? + %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account') + else + %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") + end + JSON.parse account_info_str end -def send_account(mail_content) - email = email_addr(mail_content) - message_id = email_message_id(mail_content) - # check_email_available(mail_content, email) +def send_account + message = "No email address specified\n" + message += "use -e email_address add a email address\n" + message += 'or use -f to add a email file' + raise message if $apply_info['my_email'].nil? - pub_key = pub_key_value(mail_content) - acct_info = account_info(pub_key) - - message = build_message(email, message_id, acct_info) - - %x(curl -XPOST '#{CRYSTAL_INTRANET}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") -end + acct_info = account_info($apply_info['my_ssh_pubkey']) -def read_mail_content(mail_file) - mail_content = Mail.read(mail_file) + message = build_message($apply_info['my_email'], acct_info) - return mail_content + %x(curl -XPOST '#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end -mail_file = ARGV[0] -mail_content = read_mail_content(mail_file) -send_account(mail_content) +send_account -- 2.23.0

1 0

[PATCH compass-ci] sched:add monitors depends to job
by Cao Xueliang 21 Oct '20

21 Oct '20

[why] It may add monitors field in job yaml, sched will parse the monitors field and add depends to the initrd string. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/lib/job.cr | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 12d3980..00b5e68 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -396,12 +396,42 @@ class Job initrd_pkg_arr = Array(String).new initrd_http_prefix = "http://#{INITRD_HTTP_HOST}:#{INITRD_HTTP_PORT}" mount_type = os_mount == "cifs" ? "nfs" : os_mount.dup + + get_monitors_initrd(mount_type, initrd_http_prefix, initrd_deps_arr, initrd_pkg_arr) + get_pp_initrd(mount_type, initrd_http_prefix, initrd_deps_arr, initrd_pkg_arr) + + self["initrd_deps"] = initrd_deps_arr.uniq.join(" ") + self["initrd_pkg"] = initrd_pkg_arr.uniq.join(" ") + end + + private def get_monitors_initrd(mount_type, initrd_http_prefix, initrd_deps_arr, initrd_pkg_arr) + if @hash["monitors"]? + program_params = @hash["monitors"].as_h + program_params.keys.each do |program| + deps_dest_file = "#{SRV_INITRD}/deps/#{mount_type}/#{os_dir}/#{program}.cgz" + pkg_dest_file = "#{SRV_INITRD}/pkg/#{mount_type}/#{os_dir}/#{program}.cgz" + + if File.exists?("#{user_lkp_src}/monitors/#{program}") + if File.exists?(deps_dest_file) + initrd_deps_arr << "#{initrd_http_prefix}" + JobHelper.service_path(deps_dest_file) + end + + if File.exists?(pkg_dest_file) + initrd_pkg_arr << "#{initrd_http_prefix}" + JobHelper.service_path(pkg_dest_file) + end + end + end + end + end + + private def get_pp_initrd(mount_type, initrd_http_prefix, initrd_deps_arr, initrd_pkg_arr) if @hash["pp"]? program_params = @hash["pp"].as_h program_params.keys.each do |program| if program =~ /^(.*)-\d+$/ program = $1 end + deps_dest_file = "#{SRV_INITRD}/deps/#{mount_type}/#{os_dir}/#{program}.cgz" pkg_dest_file = "#{SRV_INITRD}/pkg/#{mount_type}/#{os_dir}/#{program}.cgz" @@ -414,9 +444,6 @@ class Job end end end - - self["initrd_deps"] = initrd_deps_arr.uniq.join(" ") - self["initrd_pkg"] = initrd_pkg_arr.uniq.join(" ") end def update_tbox_group(tbox_group) -- 2.23.0

2 3

[PATCH lkp-tests 2/2] feat(iso2rootfs): add step: submit rootfs test job after rootfs is made
by Yu Chuan 21 Oct '20

21 Oct '20

[Why] The produced dailybuild rootfs needs further testing by submit a job. [How] 1. Configure lkp env by {LKP_SRC}/sbin/install-dependencies.sh 2. Generate global compass-ci yaml. - The path of yaml: - /etc/compass-ci/defaults/sparrow.yaml - The yaml have two necessary fields: - SCHED_HOST - SCHED_PORT - The value of these two fields are consistent with the value in job.yaml of iso2rootfs. 3. Modify job.yaml of rootfs test job. - Add the required fields of job.yaml of rootfs test job to job.yaml of iso2rootfs; - os - os_arch - os_version - os_mount - testbox - Replace/insert them before submit the rootfs test job. 4. Submit the job.yaml of rootfs test job. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 6 ++++ tests/iso2rootfs | 72 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index f413c9cee1f0..644c64be06b4 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -11,3 +11,9 @@ iso2rootfs: rootfs_path: initrd_http_host: initrd_http_port: + test_yaml: + test_os: + test_os_arch: + test_os_version: + test_os_mount: + test_testbox: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index bf1f995e78df..5f56f5c21db9 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -53,6 +53,12 @@ check_yaml_vars() "rootfs_path" "initrd_http_host" "initrd_http_port" + "test_yaml" + "test_os" + "test_os_arch" + "test_os_version" + "test_os_mount" + "test_testbox" ) local yaml_t @@ -404,6 +410,69 @@ test_rootfs() EOF } +############ submit test job yaml ############ +generate_global_yaml() +{ + local config_yaml="/etc/compass-ci/defaults/sparrow.yaml" + + mkdir -p "$(dirname "${config_yaml}")" + echo "SCHED_HOST: ${SCHED_HOST}" >> "${config_yaml}" + echo "SCHED_PORT: ${SCHED_PORT}" >> "${config_yaml}" +} + +assign_var() +{ + local var_name=$1 + local var_value=$2 + + if grep -qw "${var_name}:" "${test_yaml}" + then + sed -i "s/^$ *${var_name}:$.*/\1 ${var_value}/g" "${test_yaml}" + else + echo "${var_name}: ${var_value}" >> "${test_yaml}" + fi +} + +modify_test_job_yaml() +{ + log_info "starting prepare ${test_yaml}" + + [ -f "${test_yaml}" ] || die "test yaml$${test_yaml}$ exist in {LKP_SRC}/jobs !!!" + + local wait_add_vars=( + "os" + "os_arch" + "os_version" + "os_mount" + "testbox" + ) + for vt in "${wait_add_vars[@]}" + do + assign_var "$vt" "$(eval echo "\$test_${vt}")" + done +} + +submit_test_job_yaml() +{ + log_info "starting submit ${test_yaml} ..." + + [ -d "${LKP_SRC}" ] || die "no LKP_SRC !!!" + + cd "${LKP_SRC}" && { + chmod +x ./sbin/install-dependencies.sh + ./sbin/install-dependencies.sh + + generate_global_yaml + } + + cd "${LKP_SRC}/jobs" && { + modify_test_job_yaml + "${LKP_SRC}/sbin/submit" "${test_yaml}" + } + + log_info "submit ${test_yaml} finished" +} + ############ post works ############ post_works() { @@ -414,7 +483,7 @@ post_works() local soft_link="${iso_version}-dailybuild" cd "${ROOTFS_LOCAL_PATH}" && rm -f "${soft_link}" && - ln -s "$(basename ${ROOTFS_DES_DIR})" "${soft_link}" + ln -s "$(basename "${ROOTFS_DES_DIR}")" "${soft_link}" cd / && umount "${ROOTFS_LOCAL_PATH}" @@ -433,6 +502,7 @@ main() run_qcow2rootfs test_rootfs + submit_test_job_yaml post_works } -- 2.23.0

4 5

[PATCH compass-ci 1/2] src/lib: find depends_initrd for multitask
by Li Ping 21 Oct '20

21 Oct '20

[why] when the job.yaml like multi-qemu.yaml: multi-qemu-0: multi-qemu-1: the code need multi-qemu, not multi-qemu-x, and scheduler can not find the depends_initrd multi-qemu-x.cgz. Signed-off-by: Li Ping <15396232681(a)163.com> --- src/lib/job.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 12d3980..f0a9007 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -399,7 +399,7 @@ class Job if @hash["pp"]? program_params = @hash["pp"].as_h program_params.keys.each do |program| - if program =~ /^(.*)-\d+$/ + if program =~ /(.*)-\d+$/ program = $1 end deps_dest_file = "#{SRV_INITRD}/deps/#{mount_type}/#{os_dir}/#{program}.cgz" -- 2.23.0

1 1

[PATCH compass-ci 2/2] job.cr: change the default values of os and os_version
by Li Ping 21 Oct '20

21 Oct '20

--- src/lib/job.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index f0a9007..54d0862 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -39,10 +39,10 @@ class Job getter hash : Hash(String, JSON::Any) INIT_FIELD = { - os: "debian", + os: "openeuler", lab: LAB, os_arch: "aarch64", - os_version: "sid", + os_version: "20.03", lkp_initrd_user: "latest", docker_image: "centos:7", } -- 2.23.0

1 0

[PATCH lkp-tests 1/2] fix(install-dependencies.sh): fix some problems when configure lkp env
by Yu Chuan 21 Oct '20

21 Oct '20

[Why] 1. param: '-y' + ./sbin/install-dependencies.sh ...Omit... Total download size: 3.7 M Installed size: 18 M Is this ok [y/N]: 2. pkgs: gcc, gcc-c++, make ERROR: Error installing rest-client: ERROR: Failed to build gem native extension. current directory: /usr/local/share/gems/gems/unf_ext-0.0.7.7/ext/unf_ext /usr/bin/ruby -r ./siteconf20201020-5195-h45kpd.rb extconf.rb checking for -lstdc++... *** extconf.rb failed *** Could not create Makefile due to some reason, probably lack of necessary libraries and/or headers. Check the mkmf.log file for more details. You may need configuration options. ...Omit... /usr/share/ruby/mkmf.rb:456:in `try_do': The compiler failed to generate an executable file. (RuntimeError) You have to install development tools first. ...Omit... 3. pkg: ruby-devel ERROR: Error installing rest-client: ERROR: Failed to build gem native extension. current directory: /usr/local/share/gems/gems/unf_ext-0.0.7.7/ext/unf_ext /usr/bin/ruby -r ./siteconf20201020-5371-1gqrhar.rb extconf.rb mkmf.rb can't find header files for ruby at /usr/share/include/ruby.h extconf failed, exit code 1 Gem files will remain installed in /usr/local/share/gems/gems/unf_ext-0.0.7.7 for inspection. Results logged to /usr/local/lib64/gems/ruby/unf_ext-0.0.7.7/gem_make.out 4. pkg: git + {LKP_SRC}/sbin/submit test-tools.yaml /usr/local/share/gems/gems/git-1.7.0/lib/git/lib.rb:989:in `command': git '-c' 'color.ui=false' version 2>&1:sh: git: command not found (Git::GitExecuteError) Signed-off-by: Yu Chuan <13186087857(a)163.com> --- sbin/install-dependencies.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbin/install-dependencies.sh b/sbin/install-dependencies.sh index 48457d4c61cc..5673f7a773a3 100644 --- a/sbin/install-dependencies.sh +++ b/sbin/install-dependencies.sh @@ -14,9 +14,9 @@ ubuntu|debian) ;; openEuler|fedora|rhel|centos) if type dnf > /dev/null 2>&1; then - dnf install --skip-broken ruby rubygems + dnf install -y --skip-broken ruby rubygems gcc gcc-c++ make ruby-devel git else - yum install --skip-broken ruby rubygems + yum install -y --skip-broken ruby rubygems gcc gcc-c++ make ruby-devel git fi gem install -f git activesupport rest-client ;; -- 2.23.0

3 5

[PATCH v2 compass-ci 2/2] fix: scheduler: refactor match_tbox_group
by Tong Qunfeng 21 Oct '20

21 Oct '20

[why] simplify string match operate. Signed-off-by: Tong Qunfeng <taxcom(a)tom.com> --- src/lib/job.cr | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 12d3980..8dad205 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -22,9 +22,8 @@ end module JobHelper def self.match_tbox_group(testbox : String) tbox_group = testbox - find = testbox.match(/(.*)(\-\d{1,}$)/) - if find != nil - tbox_group = find.not_nil![1] + if testbox =~ /(.*)-\d+$/ + tbox_group = $1 end return tbox_group end -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] fix: scheduler: cut down log
by Tong Qunfeng 21 Oct '20

21 Oct '20

[why] when consume with "No job now", scheduler output 14 lines log. when submit 1 job and consume it, scheduler output 78 lines log. it is a bit of heavy for log collect and some logs are redundancy. [how] 1.merge 4 lines to 1 line (form to json) for debug_message. 2.remove output "Done", it's same with kemal-frame meesage "200 GET/PUT..." 3.only 0 | 1 extra message for 1 API: if there has an active message, do not call debug_message as default, there has 1 kemal-frame message for each API 2020-10-20 09:40:28 UTC 200 GET /boot.ipxe/mac/0a-4c-34-18-0a-17 7.18ms <time in> <response status> <----request API----> <response time span> [now] this patch cut 92 lines log to 26 line, interactive with 15 API. examples: --- <4 to 1> and <remove single "Done"> >> 172.168.131.113:29720 << Done 2020-10... 200 PUT /set_host_mac?hostname=vm... 289.2µs --> 2020-10... 200 PUT /set_host_mac?hostname=vm... 977.56µs --- <0 | 1 extra message> >> 172.18.222.81:44210 << Done {"job_state":"running","job_id":"crystal.105070"} 2020-10... 200 GET /~lkp/cgi-bin/lkp-jobfile-append-var?... 2.36ms --> {"job_state":"running","job_id":"crystal.105071"} 2020-10... UTC 200 GET /~lkp/cgi-bin/lkp-jobfile-append-var?... 2.8ms Signed-off-by: Tong Qunfeng <taxcom(a)tom.com> --- src/lib/sched.cr | 9 +++------ src/scheduler/scheduler.cr | 30 ++++++++++-------------------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index e6f353a..7215df5 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -317,7 +317,6 @@ class Sched JobHelper.service_path("#{SRV_INITRD}/lkp/#{job.lkp_initrd_user}/lkp-#{job.arch}.cgz") response["job"] = "http://#{SCHED_HOST}:#{SCHED_PORT}/job_initrd_tmpfs/#{job.id}/job.cgz" - puts %({"job_id": "#{job.id}", "job_state": "boot"}) return response.to_json end @@ -339,7 +338,6 @@ class Sched response += "boot\n" - puts %({"job_id": "#{job.id}", "job_state": "boot"}) return response end @@ -481,7 +479,6 @@ class Sched response += job.kernel_params response += "\nboot\n" - puts %({"job_id": "#{job.id}", "job_state": "boot"}) return response end @@ -512,7 +509,7 @@ class Sched # json log log = job_content.dup log["job_id"] = log.delete("id").not_nil! - puts log.to_json + return log.to_json end def update_tbox_wtmp(env : HTTP::Server::Context) @@ -541,7 +538,7 @@ class Sched # json log hash["testbox"] = testbox - puts hash.to_json + return hash.to_json end def report_ssh_port(testbox : String, ssh_port : String) @@ -572,6 +569,6 @@ class Sched @redis.remove_finished_job(job_id) - puts %({"job_id": "#{job_id}", "job_state": "complete"}) + return %({"job_id": "#{job_id}", "job_state": "complete"}) end end diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr index e38d15c..5befcf9 100644 --- a/src/scheduler/scheduler.cr +++ b/src/scheduler/scheduler.cr @@ -34,9 +34,7 @@ module Scheduler # for debug (maybe kemal debug|logger does better) def self.debug_message(env, response) - puts "\n\n" - puts ">> #{env.request.remote_address}" - puts "<< #{response}" + puts %({"from": "#{env.request.remote_address}", "response": #{response.to_json}}) end # echo alive @@ -53,7 +51,8 @@ module Scheduler get "/boot.:boot_type/:parameter/:value" do |env| response = sched.find_job_boot(env) - debug_message(env, response) + job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? + puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id response end @@ -62,7 +61,8 @@ module Scheduler get "/~lkp/cgi-bin/gpxelinux.cgi" do |env| response = sched.find_next_job_boot(env) - debug_message(env, response) + job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? + puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id response end @@ -87,7 +87,6 @@ module Scheduler file_path = ::File.join [Kemal.config.public_folder, job_id, job_package] puts %({"job_id": "#{job_id}", "job_state": "download"}) - debug_message(env, file_path) send_file env, file_path end @@ -102,8 +101,6 @@ module Scheduler if (client_hostname = env.params.query["hostname"]?) && (client_mac = env.params.query["mac"]?) sched.set_host_mac(client_mac, client_hostname) - debug_message(env, "Done") - "Done" else "No yet!" @@ -115,8 +112,6 @@ module Scheduler if client_mac = env.params.query["mac"]? sched.del_host_mac(client_mac) - debug_message(env, "Done") - "Done" else "No yet!" @@ -129,10 +124,8 @@ module Scheduler # ?job_file=/lkp/scheduled/job.yaml&job_state=post_run&job_id=10 # ?job_file=/lkp/scheduled/job.yaml&loadavg=0.28 0.82 0.49 1/105 3389&start_time=1587725398&end_time=1587725698&job_id=10 get "/~lkp/cgi-bin/lkp-jobfile-append-var" do |env| - # get job_id from request - debug_message(env, "Done") + puts sched.update_job_parameter(env) - sched.update_job_parameter(env) "Done" end @@ -168,17 +161,15 @@ module Scheduler # get job_id from request job_id = env.params.query["job_id"]? if job_id - debug_message(env, "Done") - - sched.close_job(job_id) + puts sched.close_job(job_id) end + "Done" end get "/~lkp/cgi-bin/lkp-wtmp" do |env| - debug_message(env, "Done") + puts sched.update_tbox_wtmp(env) - sched.update_tbox_wtmp(env) "Done" end @@ -188,12 +179,11 @@ module Scheduler job_id = env.params.query["job_id"].to_s if testbox && ssh_port - debug_message(env, "Done") - sched.report_ssh_port(testbox, ssh_port) end puts %({"job_id": "#{job_id}", "state": "set ssh port", "ssh_port": "#{ssh_port}", "tbox_name": "#{testbox}"}) + "Done" end end -- 2.23.0

1 0

[PATCH v4 compass-ci] /src/lib/web_backend.rb: fix "/compare_candidates" tbox_group regex bug
by Zhang Yuhang 21 Oct '20

21 Oct '20

Error Flow: 1. /\d+$/ will be matched, such as "xxx123". 2. `index = "xxx123".index('--') || "xxx123".rindex('-')` - index equal nil. 3. `r = r[0, nil]` - raise a TypeError. Solution: Fill a couple brackets, so will just match "vm-abc-0abc-abc" or "vm-abc-0abc-012". Signed-off-by: Zhang Yuhang <zhangyuhang25(a)huawei.com> --- src/lib/web_backend.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 48cfadb..fda01ca 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -47,7 +47,7 @@ end def filter_tbox_group(es_result) result = Set.new es_result.each do |r| - if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-([a-zA-Z]+)|(\d+)$)/ + if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-(([a-zA-Z]+)|(\d+))$)/ index = r.index('--') || r.rindex('-') r = r[0, index] end -- 2.23.0

2 3

[PATCH lkp-tests] fix: daemon: don't run 'netserver' foreground
by Ren Wen 21 Oct '20

21 Oct '20

[Why] "-D : Do not daemonize" Option '-D' will run 'netserver' foreground. Need to get exit status of executed 'netserver' when running cluster test, so remove '-D' [Ref] [Running 'netserver' as a standalone Daemon] (http://www.cs.kent.edu/~farrell/dist/ref/Netperf.html#0.2.2Z141Z1.SUJSTF.7R…) Signed-off-by: Ren Wen <15991987063(a)163.com> --- daemon/netserver | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon/netserver b/daemon/netserver index 0302dd35..d6eff6da 100755 --- a/daemon/netserver +++ b/daemon/netserver @@ -9,5 +9,5 @@ opt_ip= [ "$ip" = 'ipv6' ] && opt_ip='-6' # start netserver -log_echo netserver $opt_ip -D -exec netserver $opt_ip -D +log_echo netserver $opt_ip +exec netserver $opt_ip -- 2.23.0

2 2

[PATCH lab-z9] hosts: add hostfile for a95
by Bai Jing 21 Oct '20

21 Oct '20

Signed-off-by: Bai Jing <799286817(a)qq.com> --- hosts/taishan200-2280-2s48p-256g--a95 | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 hosts/taishan200-2280-2s48p-256g--a95 diff --git a/hosts/taishan200-2280-2s48p-256g--a95 b/hosts/taishan200-2280-2s48p-256g--a95 new file mode 100644 index 0000000..3fd8124 --- /dev/null +++ b/hosts/taishan200-2280-2s48p-256g--a95 @@ -0,0 +1,19 @@ +nr_node: 4 +nr_cpu: 96 +memory: 512G +nr_hdd_partitions: 4 +hdd_partitions: + - /dev/disk/by-id/scsi-35000039988792e31 + - /dev/disk/by-id/scsi-35000039988793705 + - /dev/disk/by-id/scsi-35000039988792aa5 + - /dev/disk/by-id/scsi-350000399887932fd +mac_addr: + - cc:05:77:fd:fb:61 + - cc:05:77:fd:fb:62 + - cc:05:77:fd:fb:63 + - cc:05:77:fd:fb:64 + - cc:05:77:fd:fb:65 + - cc:05:77:fd:fb:66 + - cc:05:77:fd:fb:67 + - cc:05:77:fd:fb:68 +model_name: Kunpeng-920 -- 2.23.0

1 0

[PATCH lkp-tests] lib: fix btrfs-progs is not installed when os_mount=initramfs
by Hu Xuejiao 21 Oct '20

21 Oct '20

When job.yaml container "fs: **", and /srv/initrd/deps/**/**/fs.cgz(btrfs-progs), it will execute mkfs.btrfs -f $rootfs_partition. but job.yaml don't container "fs: **" to appear warning: mkfs.btrfs command not found Signed-off-by: Hu XueJiao <huxuejiao1(a)huawei.com> --- lib/bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index a7776d68..16789c5c 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -456,6 +456,7 @@ mount_rootfs() ROOTFS_DIR=/opt/rootfs mkdir -p $ROOTFS_DIR mount $rootfs_partition $ROOTFS_DIR || { + command -v mkfs.btrfs >/dev/null || return mkfs.btrfs -f $rootfs_partition mount $rootfs_partition $ROOTFS_DIR } -- 2.23.0

2 1

[PATCH compass-ci] auto_submit: add upstream_tag if the commit is tagged
by Li Yuanchao 21 Oct '20

21 Oct '20

sometimes the testcase needs a specific version software, add upstream_tag can help find the specific version. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sbin/auto_submit b/sbin/auto_submit index 1f55701..88ec560 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -28,12 +28,25 @@ class AutoSubmit return pkgbuild_repo end + def if_tagged(tag_list, commit_id) + tag_list.each_line do |line| + if line.include?(commit_id) + tag = line.split + return tag[1].delete_prefix('refs/tags/') + end + end + return nil + end + def submit(newrefs_info, submit_argv) + tag_list = %x(git -C /srv/git/#{newrefs_info['git_repo']}.git show-ref --tags) newrefs_info['new_refs']['heads'].each do |branch, commit_id| real_argvs = Array.new(submit_argv) commit_date = `git -C /srv/git/#{newrefs_info['git_repo']}.git log --format=%ct -1 #{commit_id}` real_argvs.push("upstream_branch=#{branch.delete_prefix('refs/heads/')}") real_argvs.push("upstream_commit=#{commit_id}") + tag = if_tagged(tag_list, commit_id) + real_argvs.push("upstream_tag=#{tag}") if tag real_argvs.push("commit_date=#{commit_date}") system(real_argvs.join(' ')) -- 2.23.0

7 17

[PATCH v5 lkp-tests] tests: add get_ipmi_ip function to host-info
by Zhang Yu 21 Oct '20

21 Oct '20

Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- tests/host-info | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/host-info b/tests/host-info index a57855dc..99b14083 100755 --- a/tests/host-info +++ b/tests/host-info @@ -80,6 +80,25 @@ get_partitions() fi } +get_ipmi_ip() +{ + # root@localhost ~# ipmitool lan print + # Set in Progress : Set Complete + # IP Address Source : Static Address + # IP Address : 9.3.4.1 + # Subnet Mask : 255.255.0.0 + # MAC Address : e0:00:84:2b:51:7b + # SNMP Community String : TrapAdmin12#$ + # IP Header : TTL=0x40 Flags=0x40 Precedence=0x00 TOS=0x10 + # Default Gateway IP : 9.3.0.1 + # 802.1q VLAN ID : Disabled + # RMCP+ Cipher Suites : 0,1,2,3,17 + # Cipher Suite Priv Max : XuuaXXXXXXXXXXX + imp_ip=$(ipmitool lan print |awk 'NR==3{print $4}') + [ $? -eq 0 ] && echo "ipmi_ip:" $imp_ip + +} + main() { get_node @@ -87,6 +106,7 @@ main() get_partitions get_mac get_model_name + get_ipmi_ip } main -- 2.23.0

3 2

[PATCH compass-ci] fix: scheduler: cut down log
by Tong Qunfeng 21 Oct '20

21 Oct '20

[why] when consume with "No job now", scheduler output 14 lines log. when submit 1 job and consume it, scheduler output 78 lines log. it is a bit of heavy for log collect and some logs are redundancy. [how] 1.merge 4 lines to 1 line (form to json) for debug_message. 2.remove output "Done", it's same with kemal-frame meesage "200 GET/PUT..." 3.only 0 | 1 extra message for 1 API: if there has an active message, do not call debug_message as default, there has 1 kemal-frame message for each API 2020-10-20 09:40:28 UTC 200 GET /boot.ipxe/mac/0a-4c-34-18-0a-17 7.18ms <time in> <response status> <----request API----> <response time span> [now] this patch cut 92 lines log to 26 line, interactive with 15 API. examples: --- <4 to 1> and <remove single "Done"> >> 172.168.131.113:29720 << Done 2020-10... 200 PUT /set_host_mac?hostname=vm... 289.2µs --> 2020-10... 200 PUT /set_host_mac?hostname=vm... 977.56µs --- <0 | 1 extra message> >> 172.18.222.81:44210 << Done {"job_state":"running","job_id":"crystal.105070"} 2020-10... 200 GET /~lkp/cgi-bin/lkp-jobfile-append-var?... 2.36ms --> {"job_state":"running","job_id":"crystal.105071"} 2020-10... UTC 200 GET /~lkp/cgi-bin/lkp-jobfile-append-var?... 2.8ms Signed-off-by: Tong Qunfeng <taxcom(a)tom.com> --- src/lib/sched.cr | 9 +++------ src/scheduler/scheduler.cr | 30 ++++++++++-------------------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index e6f353a..7215df5 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -317,7 +317,6 @@ class Sched JobHelper.service_path("#{SRV_INITRD}/lkp/#{job.lkp_initrd_user}/lkp-#{job.arch}.cgz") response["job"] = "http://#{SCHED_HOST}:#{SCHED_PORT}/job_initrd_tmpfs/#{job.id}/job.cgz" - puts %({"job_id": "#{job.id}", "job_state": "boot"}) return response.to_json end @@ -339,7 +338,6 @@ class Sched response += "boot\n" - puts %({"job_id": "#{job.id}", "job_state": "boot"}) return response end @@ -481,7 +479,6 @@ class Sched response += job.kernel_params response += "\nboot\n" - puts %({"job_id": "#{job.id}", "job_state": "boot"}) return response end @@ -512,7 +509,7 @@ class Sched # json log log = job_content.dup log["job_id"] = log.delete("id").not_nil! - puts log.to_json + return log.to_json end def update_tbox_wtmp(env : HTTP::Server::Context) @@ -541,7 +538,7 @@ class Sched # json log hash["testbox"] = testbox - puts hash.to_json + return hash.to_json end def report_ssh_port(testbox : String, ssh_port : String) @@ -572,6 +569,6 @@ class Sched @redis.remove_finished_job(job_id) - puts %({"job_id": "#{job_id}", "job_state": "complete"}) + return %({"job_id": "#{job_id}", "job_state": "complete"}) end end diff --git a/src/scheduler/scheduler.cr b/src/scheduler/scheduler.cr index e38d15c..5befcf9 100644 --- a/src/scheduler/scheduler.cr +++ b/src/scheduler/scheduler.cr @@ -34,9 +34,7 @@ module Scheduler # for debug (maybe kemal debug|logger does better) def self.debug_message(env, response) - puts "\n\n" - puts ">> #{env.request.remote_address}" - puts "<< #{response}" + puts %({"from": "#{env.request.remote_address}", "response": #{response.to_json}}) end # echo alive @@ -53,7 +51,8 @@ module Scheduler get "/boot.:boot_type/:parameter/:value" do |env| response = sched.find_job_boot(env) - debug_message(env, response) + job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? + puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id response end @@ -62,7 +61,8 @@ module Scheduler get "/~lkp/cgi-bin/gpxelinux.cgi" do |env| response = sched.find_next_job_boot(env) - debug_message(env, response) + job_id = response[/tmpfs\/(.*)\/job\.cgz/, 1]? + puts %({"job_id": "#{job_id}", "job_state": "boot"}) if job_id response end @@ -87,7 +87,6 @@ module Scheduler file_path = ::File.join [Kemal.config.public_folder, job_id, job_package] puts %({"job_id": "#{job_id}", "job_state": "download"}) - debug_message(env, file_path) send_file env, file_path end @@ -102,8 +101,6 @@ module Scheduler if (client_hostname = env.params.query["hostname"]?) && (client_mac = env.params.query["mac"]?) sched.set_host_mac(client_mac, client_hostname) - debug_message(env, "Done") - "Done" else "No yet!" @@ -115,8 +112,6 @@ module Scheduler if client_mac = env.params.query["mac"]? sched.del_host_mac(client_mac) - debug_message(env, "Done") - "Done" else "No yet!" @@ -129,10 +124,8 @@ module Scheduler # ?job_file=/lkp/scheduled/job.yaml&job_state=post_run&job_id=10 # ?job_file=/lkp/scheduled/job.yaml&loadavg=0.28 0.82 0.49 1/105 3389&start_time=1587725398&end_time=1587725698&job_id=10 get "/~lkp/cgi-bin/lkp-jobfile-append-var" do |env| - # get job_id from request - debug_message(env, "Done") + puts sched.update_job_parameter(env) - sched.update_job_parameter(env) "Done" end @@ -168,17 +161,15 @@ module Scheduler # get job_id from request job_id = env.params.query["job_id"]? if job_id - debug_message(env, "Done") - - sched.close_job(job_id) + puts sched.close_job(job_id) end + "Done" end get "/~lkp/cgi-bin/lkp-wtmp" do |env| - debug_message(env, "Done") + puts sched.update_tbox_wtmp(env) - sched.update_tbox_wtmp(env) "Done" end @@ -188,12 +179,11 @@ module Scheduler job_id = env.params.query["job_id"].to_s if testbox && ssh_port - debug_message(env, "Done") - sched.report_ssh_port(testbox, ssh_port) end puts %({"job_id": "#{job_id}", "state": "set ssh port", "ssh_port": "#{ssh_port}", "tbox_name": "#{testbox}"}) + "Done" end end -- 2.23.0

1 0

[PATCH lkp-tests] job: add monitors to job
by Wei Jihui 21 Oct '20

21 Oct '20

find all monitors and add to job scheduler will use it to get monitors program's cgz. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/job.rb | 7 +++++++ sbin/submit | 1 + 2 files changed, 8 insertions(+) diff --git a/lib/job.rb b/lib/job.rb index caa76c4b..e2416eb8 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -564,6 +564,13 @@ class Job return option_value end + def add_monitors + @job['monitors'] = {} + for_each_in(@job, monitor_params) do |_pk, _h, m_n, _m_args| + @job['monitors'][m_n] = nil + end + end + def add_pp() @job["pp"] = Hash.new() init_program_options() diff --git a/sbin/submit b/sbin/submit index a749ea20..8b46db32 100755 --- a/sbin/submit +++ b/sbin/submit @@ -108,6 +108,7 @@ ARGV.each do |jobfile| raise 'Please configure SCHED_PORT' unless job['SCHED_PORT'] job.add_pp + job.add_monitors job.add_define_files if opt_auto_define_files # get job shell function -- 2.23.0

2 1

[PATCH v4 compass-ci 1/2] container/assign-account: answerback-email.rb
by Luan Shengde 21 Oct '20

21 Oct '20

add parameters for answerback-email -e,--email email_addr: add email address -s,--ssh-pubkey pub_key: add a ssh login pub_key -f,--raw-email mail_file: add email address[, pub_key] via email file why: easier for administrator to manully assign jumper account for user with: - user's email - user's email and a pub_key - user's email_file Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/answerback-email.rb | 173 ++++++++----------- 1 file changed, 74 insertions(+), 99 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index b5283ba..5a5234c 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -3,46 +3,18 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true -=begin - -repo_list: - all repos url list from upstream-repos.git - -API: -call graph: -read_mail_content -send_account_request - email_addr - get email address - email_message_id - get email message_id - check_email_available - check email available - pub_key_value - get pub key - account - send apply account request and return account info - build_message - build email message - send_mail - call send_mail to send mail whit build message - -the returned data for account_info like: -{ - "account" => "guest", - "passwd" => "Use pub_key to login", - "jumper_ip" => "10.10.10.10", - "jumper_port" => "10000" -} -=end +# xx --email xxx --login --ssh-pubkey xxx --raw-email email-file +# samba mount +# ssh logshn (huawei, ) (install pubkey / send password) require 'json' require 'mail' require 'set' -require_relative '../defconfig.rb' +require 'optparse' +require_relative '../defconfig' names = Set.new %w[ - JUMPER_IP + JUMPER_HOST JUMPER_PORT CRYSTAL_INTRANET SEND_MAIL_PORT @@ -50,27 +22,71 @@ names = Set.new %w[ defaults = relevant_defaults(names) -JUMPER_IP = defaults['JUMPER_IP'] -JUMPER_PORT = defaults['JUMPER_PORT'] -CRYSTAL_INTRANET = defaults['CRYSTAL_INTRANET'] -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] +JUMPER_HOST = defaults['JUMPER_HOST'] || '183.134.196.212' +JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 +CRYSTAL_INTRANET = defaults['CRYSTAL_INTRANET'] || localhost +SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 11312 + +apply_info = { + 'email_addr' => nil, + 'pub_key' => nil +} + +def init_info(email_file, apply_info) + mail_content = Mail.read(email_file) + + email_addr = mail_content.from[0] + pub_key = mail_content.part[1].body.decoded.gsub(/\r|\n/, '') if mail_content.part[1].filename == 'id_rsa.pub' + + apply_info['email_addr'] = email_addr + apply_info['pub_key'] = pub_key + + apply_info +end + +options = OptionParser.new do |opts| + opts.banner = "Usage: answerback-mail.rb [--email email] [--ssh-pubkey pub_key] [--raw-email email_file]\n" + opts.banner += " -e or -f is required\n" + opts.banner += ' -s is optional when use -e' + + opts.separator '' + opts.separator 'options:' + + opts.on('-e email_addr', '--email email_addr', 'appoint email address') do |email_addr| + apply_info['email_addr'] = email_addr + end + + opts.on('-s pub_key', '--ssh-pubkey pub_key', 'ssh pub_key to enable keyless login') do |pub_key| + apply_info['pub_key'] = pub_key + end + + opts.on('-f email_file', '--raw-email email_file', 'email file') do |email_file| + init_info(email_file, apply_info) + end + + opts.on_tail('-h', '--help', 'show this message') do + puts opts + exit + end +end -def build_message(email, message_id, infos) +options.parse!(ARGV) + +def build_message(email, acct_infos) message = <<~EMAIL_MESSAGE To: #{email} - Message-ID: #{message_id} Subject: jumper account is ready - Dear #{email} + Dear user: Thank you for joining us. You can use the following command to login the jumper server: login command: - ssh -p #{infos['jumper_port']} #{infos['account']}@#{infos['jumper_ip']} + ssh -p #{acct_infos['jumper_port']} #{acct_infos['account']}@#{acct_infos['jumper_ip']} - account passwd: - account_password: #{infos['passwd']} + account password: + #{acct_infos['passwd']} regards compass-ci @@ -79,67 +95,26 @@ def build_message(email, message_id, infos) return message end -def email_addr(mail_content) - msg = 'not an applying account email' - - raise msg unless mail_content.subject =~ /apply ssh account/i - - email = mail_content.from.join(',') - - return email -end - -# def check_email_available(mail_content, email) -# oos_list = File.read('/c/upstream-repos/repo_list').split(/\n/) -# url = mail_content.body.decoded.split(/\n/).find { |line| line =~ /https?:\/\// } -# base_url = url.split('/')[0,5].join('/') -# message = 'The url is not in upstream repo_list' -# -# raise message unless oos_list.include? base_url -# -# url_fdback = %x(curl #{url}) -# email_index = url_fdback.index email -# -# message = 'No commit info found from the url for the email' -# raise message unless email_index -# end - -def email_message_id(mail_content) - message_id = mail_content.message_id - return message_id -end - -def pub_key_value(mail_content) - pub_key = mail_content.body.decoded.split(/\n/).find { |line| line =~ /ssh-rsa/ } - return pub_key -end - def account_info(pub_key) - account_info_str = %x(curl -XGET '#{JUMPER_IP}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") - account_info = JSON.parse account_info_str - - return account_info + account_info_str = if pub_key.nil? + %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account') + else + %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") + end + JSON.parse account_info_str end -def send_account(mail_content) - email = email_addr(mail_content) - message_id = email_message_id(mail_content) - # check_email_available(mail_content, email) +def send_account(apply_info) + message = "No email address specified\n" + message += "use -e email_addr add a email address\n" + message += 'or use -f to add a mail file' + raise message if apply_info['email_addr'].nil? - pub_key = pub_key_value(mail_content) - acct_info = account_info(pub_key) + acct_info = account_info(apply_info['pub_key']) - message = build_message(email, message_id, acct_info) + message = build_message(apply_info['email_addr'], acct_info) %x(curl -XPOST '#{CRYSTAL_INTRANET}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end -def read_mail_content(mail_file) - mail_content = Mail.read(mail_file) - - return mail_content -end - -mail_file = ARGV[0] -mail_content = read_mail_content(mail_file) -send_account(mail_content) +send_account(apply_info) -- 2.23.0

2 3

[PATCH v3 compass-ci] /src/lib/web_backend.rb: fix "/compare_candidates" tbox_group regex bug
by Zhang Yuhang 21 Oct '20

21 Oct '20

Info: "/compare_canditates" error, due to fault regular expression. Detail: Miss a couple brackets, so will match /\d+$/. Error Flow: 1. like "xxx123" will be matched 2. index = "xxx123".index('--') || "xxx123".rindex('-') 3. index equal "nil" 4. r = r[0, nil] will raise an error "TypeError (no implicit conversion from nil to integer)" Solution: Fill a couple brackets, so will match "vm-abc-0abc-abc" or "vm-abc-0abc-012". Signed-off-by: Zhang Yuhang <zhangyuhang25(a)huawei.com> --- src/lib/web_backend.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 48cfadb..fda01ca 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -47,7 +47,7 @@ end def filter_tbox_group(es_result) result = Set.new es_result.each do |r| - if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-([a-zA-Z]+)|(\d+)$)/ + if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-(([a-zA-Z]+)|(\d+))$)/ index = r.index('--') || r.rindex('-') r = r[0, index] end -- 2.23.0

1 1

[PATCH v4 compass-ci 2/2] container/assign-account: answerback-email.rb
by Luan Shengde 21 Oct '20

21 Oct '20

add parameters for answerback-email -e,--email email_addr: add email address -s,--ssh-pubkey pub_key: add a ssh login pub_key -f,--raw-email mail_file: add email address[, pub_key] via email file why: easier for administrator to manully assign jumper account for user with the parameters Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/answerback-email.rb | 42 ++++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index 5a5234c..a3108cb 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -16,48 +16,48 @@ require_relative '../defconfig' names = Set.new %w[ JUMPER_HOST JUMPER_PORT - CRYSTAL_INTRANET - SEND_MAIL_PORT + SEND_MAIL_HOST + SEND_INTERNET_MAIL_PORT ] defaults = relevant_defaults(names) JUMPER_HOST = defaults['JUMPER_HOST'] || '183.134.196.212' JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 -CRYSTAL_INTRANET = defaults['CRYSTAL_INTRANET'] || localhost -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 11312 +SEND_MAIL_HOST = defaults['SEND_MAIL_HOST'] || localhost +SEND_MAIL_PORT = defaults['SEND_INTERNET_MAIL_PORT'] || 11312 apply_info = { - 'email_addr' => nil, - 'pub_key' => nil + 'my_email' => nil, + 'my_pub_key' => nil } def init_info(email_file, apply_info) mail_content = Mail.read(email_file) - email_addr = mail_content.from[0] - pub_key = mail_content.part[1].body.decoded.gsub(/\r|\n/, '') if mail_content.part[1].filename == 'id_rsa.pub' + my_email = mail_content.from[0] + my_pub_key = mail_content.part[1].body.decoded.gsub(/\r|\n/, '') if mail_content.part[1].filename == 'id_rsa.pub' - apply_info['email_addr'] = email_addr - apply_info['pub_key'] = pub_key + apply_info['my_email'] = my_email + apply_info['my_pub_key'] = my_pub_key apply_info end options = OptionParser.new do |opts| - opts.banner = "Usage: answerback-mail.rb [--email email] [--ssh-pubkey pub_key] [--raw-email email_file]\n" + opts.banner = "Usage: answerback-mail.rb [--email email] [--ssh-pubkey pub_key_file] [--raw-email email_file]\n" opts.banner += " -e or -f is required\n" opts.banner += ' -s is optional when use -e' opts.separator '' opts.separator 'options:' - opts.on('-e email_addr', '--email email_addr', 'appoint email address') do |email_addr| - apply_info['email_addr'] = email_addr + opts.on('-e email_address', '--email email_address', 'appoint email address') do |email_address| + apply_info['my_email'] = email_address end - opts.on('-s pub_key', '--ssh-pubkey pub_key', 'ssh pub_key to enable keyless login') do |pub_key| - apply_info['pub_key'] = pub_key + opts.on('-s pub_key_file', '--ssh-pubkey pub_key_file', 'ssh pub_key file, enable keyless login') do |pub_key_file| + apply_info['my_pub_key'] = File.read(pub_key_file) end opts.on('-f email_file', '--raw-email email_file', 'email file') do |email_file| @@ -106,15 +106,15 @@ end def send_account(apply_info) message = "No email address specified\n" - message += "use -e email_addr add a email address\n" - message += 'or use -f to add a mail file' - raise message if apply_info['email_addr'].nil? + message += "use -e email_address add a email address\n" + message += 'or use -f to add a email file' + raise message if apply_info['my_email'].nil? - acct_info = account_info(apply_info['pub_key']) + acct_info = account_info(apply_info['my_pub_key']) - message = build_message(apply_info['email_addr'], acct_info) + message = build_message(apply_info['my_email'], acct_info) - %x(curl -XPOST '#{CRYSTAL_INTRANET}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") + %x(curl -XPOST '#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end send_account(apply_info) -- 2.23.0

1 1

[PATCH v4 compass-ci 2/6] container/send-internet-mail: internet-smtp.rb
by Luan Shengde 21 Oct '20

21 Oct '20

setup smtp for email server Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- lib/internet-smtp.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 lib/internet-smtp.rb diff --git a/lib/internet-smtp.rb b/lib/internet-smtp.rb new file mode 100644 index 0000000..3f181ba --- /dev/null +++ b/lib/internet-smtp.rb @@ -0,0 +1,20 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'mail' + +def setup_smtp + smtp = { + address: 'smtp.qq.com', + port: 25, + domain: 'qq.com', + user_name: ENV['ROBOT_EMAIL_ADDRESS'], + password: ENV['ROBOT_EMAIL_PASSWORD'], + openssl_verify_mode: 'none', + enable_starttls_auto: true + } + + Mail.defaults { delivery_method :smtp, smtp } +end -- 2.23.0

2 2

[PATCH v2 compass-ci] /src/lib/web_backend.rb: fix "/compare_candidates" tbox_group regex bug
by Zhang Yuhang 21 Oct '20

21 Oct '20

info: "/compare_canditates" error, due to fault regular expression before: /^vm-.*-\d\w*-([a-zA-Z]+)|(\d+)$/ afeter: /^vm-.*-\d\w*-(([a-zA-Z]+)|(\d+))$/ before miss a couple brackets, so will match /\d+$/. It will cause an exception. Signed-off-by: Zhang Yuhang <zhangyuhang25(a)huawei.com> --- src/lib/web_backend.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 48cfadb..fda01ca 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -47,7 +47,7 @@ end def filter_tbox_group(es_result) result = Set.new es_result.each do |r| - if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-([a-zA-Z]+)|(\d+)$)/ + if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-(([a-zA-Z]+)|(\d+))$)/ index = r.index('--') || r.rindex('-') r = r[0, index] end -- 2.23.0

2 2

【Meeting Notice】compass-ci fluentd 日志处理系统 Time: 2020-10-22 18:30-20:30
by Meeting Book 21 Oct '20

21 Oct '20

1 0

【Meeting Notice】compass-ci fluentd 日志处理系统 Time: 2020-10-22 10:00-11:00
by Meeting Book 21 Oct '20

21 Oct '20

1 0

[PATCH v2 compass-ci] fix: sched: alter conditions when submitting cluster and normal jobs
by Ren Wen 21 Oct '20

21 Oct '20

[Why] I change the rules to judge the job is whether a cluster job. according to codes in '$LKP_SRC/lib/job.sh': ``` should_wait_cluster() ... [ "$cluster" = "cs-localhost" ] && return 1 ... ``` [Result] When the field 'cluster' values doesn't starts with 'cs-localhost', do the cluster job; normal job else. Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/lib/sched.cr | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index a4ddcec..a18bdee 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -139,22 +139,12 @@ class Sched return node_state end - # EXAMPLE: - # cluster_file: "cs-lkp-hsw-ep5" - # return: Hash(YAML::Any, YAML::Any) | Nil, 0 | <hosts_size> - # {"lkp-hsw-ep5" => {"roles" => ["server"], "macs" => ["ec:f4:bb:cb:7b:92"]}, - # "lkp-hsw-ep2" => {"roles" => ["client"], "macs" => ["ec:f4:bb:cb:54:92"]}}, 2 + # get cluster config using own lkp_src cluster file, + # a hash type will be returned def get_cluster_config(cluster_file, lkp_initrd_user, os_arch) lkp_src = Jobfile::Operate.prepare_lkp_tests(lkp_initrd_user, os_arch) cluster_file_path = Path.new(lkp_src, "cluster", cluster_file) - - if File.file?(cluster_file_path) - cluster_config = YAML.parse(File.read(cluster_file_path)).as_h - hosts_size = cluster_config.values.size - return cluster_config, hosts_size - end - - return nil, 0 + return YAML.parse(File.read(cluster_file_path)).as_h end def get_commit_date(job) @@ -177,16 +167,17 @@ class Sched job = Job.new(job_content, job_content["id"]?) job["commit_date"] = get_commit_date(job) + # it is not a cluster job if cluster field is empty or + # field's prefix is 'cs-localhost' cluster_file = job["cluster"] - if cluster_file != "" - cluster_config, hosts_size = get_cluster_config( - cluster_file, job.lkp_initrd_user, job.os_arch) - - return submit_cluster_job( - job, cluster_config.not_nil!) if hosts_size >= 2 + if cluster_file.empty? || cluster_file.starts_with?("cs-localhost") + return submit_single_job(job) + else + cluster_config = get_cluster_config(cluster_file, + job.lkp_initrd_user, + job.os_arch) + return submit_cluster_job(job, cluster_config) end - - return submit_single_job(job) rescue ex puts ex.inspect_with_backtrace return [{ -- 2.23.0

2 2

[PATCH v3 compass-ci] container/assign-account: answerback-email.rb
by Luan Shengde 21 Oct '20

21 Oct '20

add parameters for answerback-email -e,--email email_addr: add email address -s,--ssh-pubkey pub_key: add a ssh login pub_key -f,--raw-email mail_file: add email address[, pub_key] via email file why: easier for administrator to manully assign jumper account for user with: - user's email - user's email and a pub_key - user's email_file Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/assign-account/answerback-email.rb | 173 ++++++++----------- 1 file changed, 74 insertions(+), 99 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index b5283ba..5a5234c 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -3,46 +3,18 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true -=begin - -repo_list: - all repos url list from upstream-repos.git - -API: -call graph: -read_mail_content -send_account_request - email_addr - get email address - email_message_id - get email message_id - check_email_available - check email available - pub_key_value - get pub key - account - send apply account request and return account info - build_message - build email message - send_mail - call send_mail to send mail whit build message - -the returned data for account_info like: -{ - "account" => "guest", - "passwd" => "Use pub_key to login", - "jumper_ip" => "10.10.10.10", - "jumper_port" => "10000" -} -=end +# xx --email xxx --login --ssh-pubkey xxx --raw-email email-file +# samba mount +# ssh logshn (huawei, ) (install pubkey / send password) require 'json' require 'mail' require 'set' -require_relative '../defconfig.rb' +require 'optparse' +require_relative '../defconfig' names = Set.new %w[ - JUMPER_IP + JUMPER_HOST JUMPER_PORT CRYSTAL_INTRANET SEND_MAIL_PORT @@ -50,27 +22,71 @@ names = Set.new %w[ defaults = relevant_defaults(names) -JUMPER_IP = defaults['JUMPER_IP'] -JUMPER_PORT = defaults['JUMPER_PORT'] -CRYSTAL_INTRANET = defaults['CRYSTAL_INTRANET'] -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] +JUMPER_HOST = defaults['JUMPER_HOST'] || '183.134.196.212' +JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 +CRYSTAL_INTRANET = defaults['CRYSTAL_INTRANET'] || localhost +SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 11312 + +apply_info = { + 'email_addr' => nil, + 'pub_key' => nil +} + +def init_info(email_file, apply_info) + mail_content = Mail.read(email_file) + + email_addr = mail_content.from[0] + pub_key = mail_content.part[1].body.decoded.gsub(/\r|\n/, '') if mail_content.part[1].filename == 'id_rsa.pub' + + apply_info['email_addr'] = email_addr + apply_info['pub_key'] = pub_key + + apply_info +end + +options = OptionParser.new do |opts| + opts.banner = "Usage: answerback-mail.rb [--email email] [--ssh-pubkey pub_key] [--raw-email email_file]\n" + opts.banner += " -e or -f is required\n" + opts.banner += ' -s is optional when use -e' + + opts.separator '' + opts.separator 'options:' + + opts.on('-e email_addr', '--email email_addr', 'appoint email address') do |email_addr| + apply_info['email_addr'] = email_addr + end + + opts.on('-s pub_key', '--ssh-pubkey pub_key', 'ssh pub_key to enable keyless login') do |pub_key| + apply_info['pub_key'] = pub_key + end + + opts.on('-f email_file', '--raw-email email_file', 'email file') do |email_file| + init_info(email_file, apply_info) + end + + opts.on_tail('-h', '--help', 'show this message') do + puts opts + exit + end +end -def build_message(email, message_id, infos) +options.parse!(ARGV) + +def build_message(email, acct_infos) message = <<~EMAIL_MESSAGE To: #{email} - Message-ID: #{message_id} Subject: jumper account is ready - Dear #{email} + Dear user: Thank you for joining us. You can use the following command to login the jumper server: login command: - ssh -p #{infos['jumper_port']} #{infos['account']}@#{infos['jumper_ip']} + ssh -p #{acct_infos['jumper_port']} #{acct_infos['account']}@#{acct_infos['jumper_ip']} - account passwd: - account_password: #{infos['passwd']} + account password: + #{acct_infos['passwd']} regards compass-ci @@ -79,67 +95,26 @@ def build_message(email, message_id, infos) return message end -def email_addr(mail_content) - msg = 'not an applying account email' - - raise msg unless mail_content.subject =~ /apply ssh account/i - - email = mail_content.from.join(',') - - return email -end - -# def check_email_available(mail_content, email) -# oos_list = File.read('/c/upstream-repos/repo_list').split(/\n/) -# url = mail_content.body.decoded.split(/\n/).find { |line| line =~ /https?:\/\// } -# base_url = url.split('/')[0,5].join('/') -# message = 'The url is not in upstream repo_list' -# -# raise message unless oos_list.include? base_url -# -# url_fdback = %x(curl #{url}) -# email_index = url_fdback.index email -# -# message = 'No commit info found from the url for the email' -# raise message unless email_index -# end - -def email_message_id(mail_content) - message_id = mail_content.message_id - return message_id -end - -def pub_key_value(mail_content) - pub_key = mail_content.body.decoded.split(/\n/).find { |line| line =~ /ssh-rsa/ } - return pub_key -end - def account_info(pub_key) - account_info_str = %x(curl -XGET '#{JUMPER_IP}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") - account_info = JSON.parse account_info_str - - return account_info + account_info_str = if pub_key.nil? + %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account') + else + %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d "pub_key: #{pub_key}") + end + JSON.parse account_info_str end -def send_account(mail_content) - email = email_addr(mail_content) - message_id = email_message_id(mail_content) - # check_email_available(mail_content, email) +def send_account(apply_info) + message = "No email address specified\n" + message += "use -e email_addr add a email address\n" + message += 'or use -f to add a mail file' + raise message if apply_info['email_addr'].nil? - pub_key = pub_key_value(mail_content) - acct_info = account_info(pub_key) + acct_info = account_info(apply_info['pub_key']) - message = build_message(email, message_id, acct_info) + message = build_message(apply_info['email_addr'], acct_info) %x(curl -XPOST '#{CRYSTAL_INTRANET}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end -def read_mail_content(mail_file) - mail_content = Mail.read(mail_file) - - return mail_content -end - -mail_file = ARGV[0] -mail_content = read_mail_content(mail_file) -send_account(mail_content) +send_account(apply_info) -- 2.23.0

2 2

[PATCH v4 compass-ci 6/6] container/send-internet-mail: start
by Luan Shengde 21 Oct '20

21 Oct '20

Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/send-internet-mail/start | 76 ++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 container/send-internet-mail/start diff --git a/container/send-internet-mail/start b/container/send-internet-mail/start new file mode 100755 index 0000000..fd2267f --- /dev/null +++ b/container/send-internet-mail/start @@ -0,0 +1,76 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig' + +docker_rm 'send-internet-mail' + +names = Set.new %w[ + ROBOT_EMAIL_INTERNET_PASSWORD + ROBOT_EMAIL_INTERNET_ADDRESS + SEND_INTERNET_MAIL_PORT +] + +defaults = relevant_defaults(names) + +ROBOT_EMAIL_ADDRESS = defaults['ROBOT_EMAIL_INTERNET_ADDRESS'] +ROBOT_EMAIL_PASSWORD = defaults['ROBOT_EMAIL_INTERNET_PASSWORD'] +SEND_MAIL_PORT = defaults['SEND_INTERNET_MAIL_PORT'] +SMTP_REGION = 'internet-smtp' + +cmd = %W[ + docker run + --restart=always + --name=send-internet-mail + -d + -e ROBOT_EMAIL_ADDRESS=#{ROBOT_EMAIL_ADDRESS} + -e ROBOT_EMAIL_PASSWORD=#{ROBOT_EMAIL_PASSWORD} + -e CCI_SRC=/c/compass-ci + -e SEND_MAIL_PORT=#{SEND_MAIL_PORT} + -e SMTP_REGION=#{SMTP_REGION} + -p #{SEND_MAIL_PORT}:#{SEND_MAIL_PORT} + -v #{ENV['CCI_SRC']}:/c/compass-ci + -v /etc/localtime:/etc/localtime:ro + -w /c/compass-ci + --log-driver json-file + send-internet-mail +] +cmd += ['ruby', "container/send-internet-mail/run.rb"] + +system(*cmd) + +puts <<EOF +port: +--- +data='{ +"subject": "email subject", +"to": "email_to_addr", +"body": "email message" +}' + +or + +data=" +subject: email subject +to: email_to_addr +body: email message" + +Usage: + curl -XPOST 'send-mail-server:11312/send_mail_yaml' -d "$data" + +--- +data=" +To: email_to_addr +Subject: email_subject + +mail_msg_line1 +mail_msg_line2 +... +" + +Usage: + curl -XPOST 'send-mail-server:11312/send_mail_text' -d "$data" +EOF -- 2.23.0

1 0

[PATCH v4 compass-ci 5/6] container/send-internet-mail: Dockerfile
by Luan Shengde 21 Oct '20

21 Oct '20

Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/send-internet-mail/Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 container/send-internet-mail/Dockerfile diff --git a/container/send-internet-mail/Dockerfile b/container/send-internet-mail/Dockerfile new file mode 100644 index 0000000..dfa0d46 --- /dev/null +++ b/container/send-internet-mail/Dockerfile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM debian +MAINTAINER luanshd0525(a)163.com +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && \ + apt-get install -y ruby-mail ruby-json ruby-sinatra -- 2.23.0

1 0

[PATCH v4 compass-ci 4/6] container/send-internet-mail: build
by Luan Shengde 21 Oct '20

21 Oct '20

Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/send-internet-mail/build | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 container/send-internet-mail/build diff --git a/container/send-internet-mail/build b/container/send-internet-mail/build new file mode 100755 index 0000000..078adaf --- /dev/null +++ b/container/send-internet-mail/build @@ -0,0 +1,5 @@ +#!/bin/sh +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +docker build -t send-internet-mail:latest . -- 2.23.0

1 0

[PATCH v5 compass-ci] feat: build centos initramfs image with one command
by Wang Chenglong 21 Oct '20

21 Oct '20

Use the tool to create a new centos-$version initramfs image with one command. and we also can customize the pre-installed software by configuring ./bin/packages-to-install. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/osimage/centos/README.md | 15 ++++++++ container/osimage/centos/create-image | 37 +++++++++++++++++++ container/osimage/centos/files-to-exclude | 11 ++++++ container/osimage/centos/lib | 38 ++++++++++++++++++++ container/osimage/centos/packages-to-install | 9 +++++ container/osimage/centos/run | 27 ++++++++++++++ 6 files changed, 137 insertions(+) create mode 100644 container/osimage/centos/README.md create mode 100755 container/osimage/centos/create-image create mode 100644 container/osimage/centos/files-to-exclude create mode 100755 container/osimage/centos/lib create mode 100644 container/osimage/centos/packages-to-install create mode 100755 container/osimage/centos/run diff --git a/container/osimage/centos/README.md b/container/osimage/centos/README.md new file mode 100644 index 0000000..88166c8 --- /dev/null +++ b/container/osimage/centos/README.md @@ -0,0 +1,15 @@ +# Use the tool to create a new centos-${os_version} initramfs image. + +Usage: + cd ${CCI_SRC}/rootfs/initramfs/centos/aarch64/${os_version}/ + ./build + +Some configuration items: +./packages-to-install + If you want to pre-install the software, you can write the package names in ./packages-to-install. + +./files-to-exclude + If you want remove some unnecessary files, you can write the names in ./files-to-exclude + +$HOME/.config/compass-ci/rootfs-passwd + Set the password for the image into this file. diff --git a/container/osimage/centos/create-image b/container/osimage/centos/create-image new file mode 100755 index 0000000..61f3e1c --- /dev/null +++ b/container/osimage/centos/create-image @@ -0,0 +1,37 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +install_packages() +{ + yum repolist + yum -y install --skip-broken $(</tmp/packages-to-install) + yum clean all +} + +setup_rootfs() +{ + ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime +} + + +setup_login() { + [ -n "$ROOT_NEW_PASSWD" ] || return + echo "Changing root password" + passwd_md5=$(openssl passwd -1 "$ROOT_NEW_PASSWD") + sed -i -r "s/^root:[^:]*:(.*)/root:${passwd_md5//\//\\/}:\1/" "$ROOTFS_DIR/etc/shadow" + + sed -i 's/[# ]PermitRootLogin.*/PermitRootLogin yes/' "$ROOTFS_DIR/etc/ssh/sshd_config" +} + +pack_cgz() +{ + echo "Packing package. Please wait." + find / ! -path "/tmp/${image_name}" | grep -vf /tmp/files-to-exclude | cpio -o -Hnewc | gzip -9 > /tmp/${image_name} + chmod 644 /tmp/${image_name} +} + +install_packages +setup_rootfs +setup_login +pack_cgz diff --git a/container/osimage/centos/files-to-exclude b/container/osimage/centos/files-to-exclude new file mode 100644 index 0000000..3854947 --- /dev/null +++ b/container/osimage/centos/files-to-exclude @@ -0,0 +1,11 @@ +/.dockerenv +/lib/modules +/usr/lib/modules +/usr/share/doc +/usr/share/man +/usr/share/info +/usr/share/i18n +/usr/share/locale +/usr/share/terminfo +/tmp/files-to-exclude +/tmp/packages-to-install diff --git a/container/osimage/centos/lib b/container/osimage/centos/lib new file mode 100755 index 0000000..bb6d693 --- /dev/null +++ b/container/osimage/centos/lib @@ -0,0 +1,38 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +DIR="$(pwd)" +os_name="$(echo $DIR |awk -F "/" '{print $(NF - 2)}')" +os_version="$(echo $DIR |awk -F "/" '{print $NF}')" +image="${os_name}:${os_version}" +image_name="${os_name}-${os_version}-$(date +"%Y%m%d").cgz" + +check_passwd_file() { + root_pwd_file="$HOME/.config/compass-ci/rootfs-passwd" + export ROOT_NEW_PASSWD= + [ -f "$root_pwd_file" ] || { + echo "[INFO] Please set the password file." + echo "$HOME/.config/compass-ci/rootfs-passwd" + exit 1 + } + + export ROOT_NEW_PASSWD=$(cat "$root_pwd_file") +} + +pull_docker_image() +{ + docker pull $image + if [ $? = 0 ]; then + echo "finish downloading image" + else + echo "[INFO] Pull docker image failed." + exit + fi +} + +docker_cp_image() +{ + docker cp -a init_docker:/tmp/${image_name} $HOME/ + echo "result: $(ls $HOME/${image_name})" +} diff --git a/container/osimage/centos/packages-to-install b/container/osimage/centos/packages-to-install new file mode 100644 index 0000000..a09bc28 --- /dev/null +++ b/container/osimage/centos/packages-to-install @@ -0,0 +1,9 @@ +openssh-server +openssl +cifs-utils +hostname +curl +iproute +iputils +which +time diff --git a/container/osimage/centos/run b/container/osimage/centos/run new file mode 100755 index 0000000..3851429 --- /dev/null +++ b/container/osimage/centos/run @@ -0,0 +1,27 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. ${CCI_SRC}/container/osimage/centos/lib +. ${CCI_SRC}/container/defconfig.sh + +check_passwd_file + +DIR=$(dirname $(realpath $0)) +cmd=( + docker run + --name init_docker + -v $DIR/create-image:/root/bin/create-image + -v $DIR/packages-to-install:/tmp/packages-to-install + -v $DIR/files-to-exclude:/tmp/files-to-exclude + -e ROOT_NEW_PASSWD=$ROOT_NEW_PASSWD + -e image_name=$image_name + $image + /root/bin/create-image +) + +pull_docker_image +${cmd[@]} +docker_cp_image +docker_rm init_docker &> /dev/null +echo "build finished" -- 2.23.0

2 2

[PATCH v3 compass-ci] add exception handing for docker.rb
by Xiao Shenwei 21 Oct '20

21 Oct '20

[why] when run multi-docker job on physical machine, run 50 client to request job, if server down or error like Connection refused. these client will exit. when the server recovers, need to run these client again. [how] add exception handing to ensure all client does not exit --- providers/docker/docker.rb | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/providers/docker/docker.rb b/providers/docker/docker.rb index c241d3c..4e6bc2f 100755 --- a/providers/docker/docker.rb +++ b/providers/docker/docker.rb @@ -88,6 +88,20 @@ def main(hostname) run(hostname, load_path, hash) end +def loop_main(hostname) + loop do + begin + main(hostname) + rescue StandardError => e + puts e.backtrace + # if an exception happend, request the next time after 30 seconds + sleep 25 + ensure + sleep 5 + end + end +end + def save_pid(pids) FileUtils.cd("#{ENV['CCI_SRC']}/providers") f = File.new('dc.pid', 'a') @@ -99,10 +113,7 @@ def multi_docker(hostname, nr_container) pids = [] nr_container.to_i.times do |i| pid = Process.fork do - loop do - main("#{hostname}-#{i}") - sleep 5 - end + loop_main("#{hostname}-#{i}") end pids << pid end -- 2.23.0

1 0

[PATCH v3 lkp-tests] fix(iso2rootfs): support skipping iso that not used for dailybuild
by Yu Chuan 20 Oct '20

20 Oct '20

[Why] Openeuler release two kinds of iso type now, we need to skip the released iso type that not used for dailybuild. [How] 1. Confirm whether the purpose is dailybuild by iso name. 2. Exit 0 while the released iso is not used for dailybuild. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/iso2rootfs | 53 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/iso2rootfs b/tests/iso2rootfs index 89328f2f4a9b..bf1f995e78df 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -22,6 +22,12 @@ log_info() echo "[INFO] $*" } +exit_info() +{ + log_info "$@" + exit 0 +} + die() { echo "[ERROR] $*" >&2 @@ -56,6 +62,48 @@ check_yaml_vars() done } +check_iso_name() +{ + # Why need this step: + # + # 1. Our current strategy for detecting iso updates is as follows: + # - Monitor iso releases for different os (such as Openeuler). + # - Openeuler currently provides an http release_iso file that is + # updated each time a new iso is released. we can get the + # following two contents from this file: + # - Latest iso url; + # - Latest iso sha256sum; # Can be obtained by splicing: + # {latest_iso_url}. sha256sum + # + # 2. Regarding the iso name in the iso url, I consulted the support + # staff of openeuler's iso release and got the following reply: + # - There are currently two kinds of iso names: + # - openEuler-2.0-SP8-xxx.iso; + # - openEuler-20.03-xxx.iso; + # - openEuler-2.0.SP8-xxx. iso is not used for dailybuild; + # - Not sure if new uses of iso will be added in the future; + # + # 3. So we've prepared an array to fill in iso names to skip and + # exclude any additions later. + + local iso_prefixes_to_skip + case ${iso_os} in + "openeuler") + iso_prefixes_to_skip=("openEuler-2.0-SP8") + ;; + *) + return + ;; + esac + + local prefix + for prefix in "${iso_prefixes_to_skip[@]}" + do + [[ ${ISO_NAME} != ${prefix}* ]] || + exit_info "${iso_os} haven't release new iso for openEuler, no need to generate rootfs" + done +} + get_daily_iso_checksum() { ISO_URL="$(curl "${iso_url_file}")" @@ -63,8 +111,10 @@ get_daily_iso_checksum() local pub_ip=$(echo "${iso_url_file}" | grep -oEw "[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}") ISO_URL="$(curl "${iso_url_file}" |sed -r "s/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/${pub_ip}/g")" } + ISO_NAME=$(basename "$ISO_URL") ISO_CHECKSUM_URL="${ISO_URL}.sha256sum" + check_iso_name curl -o "${CHECKSUM_FILE_CACHE}.tmp" "${ISO_CHECKSUM_URL}" SHA256SUM_NET=$(awk '{print $1}' "${CHECKSUM_FILE_CACHE}.tmp") @@ -118,9 +168,8 @@ check_sha256sum_update() if [ "$SHA256SUM_CACHE" == "$SHA256SUM_NET" ] then - log_info "${iso_os} haven't release new iso, no need to generate rootfs" rm -f "${CHECKSUM_FILE_CACHE}.tmp" - exit 0 + exit_info "${iso_os} haven't release new iso, no need to generate rootfs" else log_info "${iso_os} release a new iso, start to generate rootfs ..." return -- 2.23.0

1 0