- Compass-ci - mailweb.openeuler.org

[PATCH lkp-tests] change service ports
by Wu Zhende 16 Mar '21

16 Mar '21

monitoring: 11310 to 20001 scheduler-https: 11306 to 20002 sshr: 50000-51999 to 21000-23999 Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- daemon/sshd | 2 +- include/lab/z9 | 2 +- lib/monitor.rb | 4 ++-- sbin/monitor | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/daemon/sshd b/daemon/sshd index 7ba471a88..50a7677ab 100755 --- a/daemon/sshd +++ b/daemon/sshd @@ -15,7 +15,7 @@ PROJECT_DIR=$(dirname $SCRIPT_DIR) sshr_ip=$LKP_SERVER [ -n "$sshr_port" ] || sshr_port=5050 [ -n "$sshr_port_len" ] || sshr_port_len=2000 -[ -n "$sshr_port_base" ] || sshr_port_base=50000 +[ -n "$sshr_port_base" ] || sshr_port_base=21000 generate_key_file() { diff --git a/include/lab/z9 b/include/lab/z9 index 0592fdb6a..68c7aafc1 100644 --- a/include/lab/z9 +++ b/include/lab/z9 @@ -1,2 +1,2 @@ SCHED_HOST: api.compass-ci.openeuler.org -SCHED_PORT: 11306 +SCHED_PORT: 20002 diff --git a/lib/monitor.rb b/lib/monitor.rb index 24191fa8d..b08fdf4c9 100755 --- a/lib/monitor.rb +++ b/lib/monitor.rb @@ -34,7 +34,7 @@ class Monitor return unless @monitor_url == '' if host = @job['SCHED_HOST'] - return @monitor_url = "ws://#{host}:11310/filter" + return @monitor_url = "ws://#{host}:20001/filter" end Dir.glob(['/etc/compass-ci/monitor/*.yaml', @@ -49,7 +49,7 @@ class Monitor revise_hash(@defaults, defaults, true) end - @monitor_url = @defaults['monitor_url'] || 'ws://localhost:11310/filter' + @monitor_url = @defaults['monitor_url'] || 'ws://localhost:20001/filter' end def merge_overrides diff --git a/sbin/monitor b/sbin/monitor index a2e677fc0..f3ec73469 100755 --- a/sbin/monitor +++ b/sbin/monitor @@ -26,7 +26,7 @@ options = OptionParser.new do |opts| opts.separator 'options:' opts.on('-u URL', '--url URL', - 'set Message Server API address, default is ws://localhost:11310/filter') do |monitor_url| + 'set Message Server API address, default is ws://localhost:20001/filter') do |monitor_url| opt_monitor_url = monitor_url end -- 2.23.0

1 0

[PATCH compass-ci] service: change port
by Wu Zhende 16 Mar '21

16 Mar '21

kibana: 11309 to 20000 monitoring: 11310 to 20001 scheduler-https: 11306 to 20002 sshr: 50000-51999 to 21000-23999 Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana/start | 2 +- container/monitoring/build | 2 +- container/monitoring/start | 2 +- container/scheduler-https/start | 4 ++-- container/ssh-r/Dockerfile | 2 +- container/ssh-r/start | 2 +- doc/manual/submit-job.en.md | 4 ++-- doc/manual/submit-job.zh.md | 4 ++-- src/delimiter/constants.rb | 2 +- src/monitoring/constants.cr | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/container/kibana/start b/container/kibana/start index dbb3c56..26a4944 100755 --- a/container/kibana/start +++ b/container/kibana/start @@ -24,7 +24,7 @@ cmd = %W[ -v /etc/localtime:/etc/localtime:ro -d -e ELASTICSEARCH_HOSTS=http://#{LOGGING_ES_HOST}:#{LOGGING_ES_PORT} - -p 11309:5601 + -p 20000:5601 kibana:7.6.2 ] diff --git a/container/monitoring/build b/container/monitoring/build index 121ede0..4865c70 100755 --- a/container/monitoring/build +++ b/container/monitoring/build @@ -9,7 +9,7 @@ require_relative '../defconfig.rb' names = Set.new ['MONITOR_PORT'] defaults = relevant_defaults(names) -MONITOR_PORT = (defaults['MONITOR_PORT'] || '11310') +MONITOR_PORT = (defaults['MONITOR_PORT'] || '20001') docker_skip_rebuild "monitoring-#{MONITOR_PORT}" diff --git a/container/monitoring/start b/container/monitoring/start index 2e2946c..f4214ed 100755 --- a/container/monitoring/start +++ b/container/monitoring/start @@ -17,7 +17,7 @@ env = docker_env(defaults) DEFAULT_CCI = '/c/cci' DEFAULT_LKP = '/c/lkp-tests' -MONITOR_PORT = defaults['MONITOR_PORT'] || '11310' +MONITOR_PORT = defaults['MONITOR_PORT'] || '20001' docker_rm "monitoring-#{MONITOR_PORT}" cmd = %W[ diff --git a/container/scheduler-https/start b/container/scheduler-https/start index 98940d7..ee93e79 100755 --- a/container/scheduler-https/start +++ b/container/scheduler-https/start @@ -39,12 +39,12 @@ names = Set.new %w[ defaults = relevant_defaults(names) defaults['SSHR_PORT'] ||= 5050 -defaults['SSHR_PORT_BASE'] ||= 50000 +defaults['SSHR_PORT_BASE'] ||= 21000 defaults['SSHR_PORT_LEN'] ||= 2000 defaults['SCHED_PORT'] ||= '3000' defaults['SCHED_HOST'] ||= '172.17.0.1' defaults['MASTER_FLUENTD_PORT'] ||= '24224' -defaults['SCHED_HTTPS_PORT'] ||= 11306 +defaults['SCHED_HTTPS_PORT'] ||= 20002 defaults['SCHED_HTTPS_HOST'] ||= 'api.compass-ci.openeuler.org' env = docker_env(defaults) diff --git a/container/ssh-r/Dockerfile b/container/ssh-r/Dockerfile index 65a13aa..416e0ef 100644 --- a/container/ssh-r/Dockerfile +++ b/container/ssh-r/Dockerfile @@ -21,7 +21,7 @@ RUN chown -R sshr:sshr /home/sshr/.ssh/authorized_keys \ && chmod 600 /home/sshr/.ssh/authorized_keys \ && passwd -u sshr -EXPOSE 50000-51999 +EXPOSE 21000-23999 EXPOSE 22 COPY entry.sh /entry.sh diff --git a/container/ssh-r/start b/container/ssh-r/start index 5c800f3..6c7a373 100755 --- a/container/ssh-r/start +++ b/container/ssh-r/start @@ -15,7 +15,7 @@ cmd=( -e GATEWAY_PORTS=true -e TCP_FORWARDING=true -d - -p 50000-51999:50000-51999 + -p 21000-23999:21000-23999 -p 5050:22 -v /etc/localtime:/etc/localtime:ro -v /srv/pub/sshr/keys/:/etc/ssh/keys diff --git a/doc/manual/submit-job.en.md b/doc/manual/submit-job.en.md index de73f8f..5106bd9 100644 --- a/doc/manual/submit-job.en.md +++ b/doc/manual/submit-job.en.md @@ -141,7 +141,7 @@ options: hi8109@account-vm ~% submit -m iperf.yaml submit iperf.yaml, got job_id=z9.173923 query=>{"job_id":["z9.173923"]} - connect to ws://172.168.131.2:11310/filter + connect to ws://172.168.131.2:20001/filter {"job_id":"z9.173923","message":"","job_state":"submit","result_root":"/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch6 {"job_id": "z9.173923", "result_root": "/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/tcp-30/z9.173923", "job_state {"job_id": "z9.173923", "job_state": "boot"} @@ -175,7 +175,7 @@ options: hi8109@account-vm ~% submit -m -c borrow-1h.yaml submit borrow-1h.yaml, got job_id=z9.173925 query=>{"job_id":["z9.173925"]} - connect to ws://172.168.131.2:11310/filter + connect to ws://172.168.131.2:20001/filter {"job_id":"z9.173925","message":"","job_state":"submit","result_root":"/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch {"job_id": "z9.173925", "result_root": "/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/3600/z9.173925", "job_state" {"job_id": "z9.173925", "job_state": "boot"} diff --git a/doc/manual/submit-job.zh.md b/doc/manual/submit-job.zh.md index 77e9566..5f1838b 100644 --- a/doc/manual/submit-job.zh.md +++ b/doc/manual/submit-job.zh.md @@ -140,7 +140,7 @@ options: hi8109@account-vm ~% submit -m iperf.yaml testbox=vm-2p8g submit iperf.yaml, got job_id=z9.173923 query=>{"job_id":["z9.173923"]} - connect to ws://172.168.131.2:11310/filter + connect to ws://172.168.131.2:20001/filter {"job_id":"z9.173923","message":"","job_state":"submit","result_root":"/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch6 {"job_id": "z9.173923", "result_root": "/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/tcp-30/z9.173923", "job_state {"job_id": "z9.173923", "job_state": "boot"} @@ -173,7 +173,7 @@ options: hi8109@account-vm ~% submit -m -c borrow-1h.yaml testbox=vm-2p8g submit borrow-1h.yaml, got job_id=z9.173925 query=>{"job_id":["z9.173925"]} - connect to ws://172.168.131.2:11310/filter + connect to ws://172.168.131.2:20001/filter {"job_id":"z9.173925","message":"","job_state":"submit","result_root":"/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch {"job_id": "z9.173925", "result_root": "/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/3600/z9.173925", "job_state" {"job_id": "z9.173925", "job_state": "boot"} diff --git a/src/delimiter/constants.rb b/src/delimiter/constants.rb index a790736..da3f075 100644 --- a/src/delimiter/constants.rb +++ b/src/delimiter/constants.rb @@ -4,7 +4,7 @@ LKP_SRC ||= ENV['LKP_SRC'] || '/c/lkp-tests' MONITOR_HOST ||= ENV['MONITOR_HOST'] || '172.17.0.1' -MONITOR_PORT ||= ENV['MONITOR_PORT'] || '11310' +MONITOR_PORT ||= ENV['MONITOR_PORT'] || '20001' GIT_MIRROR_HOST ||= ENV['GIT_MIRROR_HOST'] || '172.17.0.1' DELIMITER_EMAIL ||= ENV['DELIMITER_EMAIL'] || 'delimiter@localhost' BISECT_RUN_SCRIPT = "#{ENV['CCI_SRC']}/src/delimiter/find-commit/bisect_run_script.rb" diff --git a/src/monitoring/constants.cr b/src/monitoring/constants.cr index 154c536..9e080c8 100644 --- a/src/monitoring/constants.cr +++ b/src/monitoring/constants.cr @@ -1,4 +1,4 @@ MQ_HOST = (ENV.has_key?("MQ_HOST") ? ENV["MQ_HOST"] : "172.17.0.1") MQ_PORT = (ENV.has_key?("MQ_PORT") ? ENV["MQ_PORT"] : 5672).to_i32 -MONITOR_PORT = (ENV.has_key?("MONITOR_PORT") ? ENV["MONITOR_PORT"] : 11310).to_i32 +MONITOR_PORT = (ENV.has_key?("MONITOR_PORT") ? ENV["MONITOR_PORT"] : 20001).to_i32 -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] container/dracut-initrd: add e2fsck and basename into dracut module 00bash
by Xu Xijian 16 Mar '21

16 Mar '21

1. mke2fs is depended by e2fsck. 2. We've add basename into 98dracut module, but it's too late, so add it into an earlier module 00bash. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 494588a..616183f 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -17,7 +17,8 @@ cp -a /usr/local/bin/cifs-lib.sh /usr/lib/dracut/modules.d/95cifs/ cat overlay-lkp.sh >> /usr/lib/dracut/modules.d/90overlay-root/overlay-mount.sh sed -i "/install() {/a\ inst /usr/bin/awk" /usr/lib/dracut/modules.d/40network/module-setup.sh -sed -i "/install() {/a\ inst /sbin/mke2fs /usr/bin/basename" /usr/lib/dracut/modules.d/98dracut-systemd/module-setup.sh +sed -i "/install() {/a\ inst /sbin/e2fsck /sbin/mke2fs /usr/bin/basename" /usr/lib/dracut/modules.d/00bash/module-setup.sh + pre_mount_file="/usr/lib/dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh" [ "$(sed -n '$p' $pre_mount_file)" = "exit 0" ] && sed -i '$d' "$pre_mount_file" cat set-local-sysroot.sh >> "$pre_mount_file" -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] src/lib(scheduer): change function name "*pushlish*" to "*publish*" for mq
by Xu Xijian 16 Mar '21

16 Mar '21

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/lifecycle.cr | 4 ++-- src/lib/mq.cr | 4 ++-- src/lib/sched.cr | 4 ++-- src/scheduler/close_job.cr | 2 +- src/scheduler/find_job_boot.cr | 2 +- src/scheduler/update_job_parameter.cr | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 1eb6264..4bd2ac0 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -79,7 +79,7 @@ class Lifecycle "job_state" => "abnormal", "testbox" => testbox } - @mq.pushlish_confirm("job_mq", msg.to_json) + @mq.publish_confirm("job_mq", msg.to_json) @match[testbox].delete(id) end end @@ -313,7 +313,7 @@ class Lifecycle mq_queue = get_machine_reboot_queue(testbox) machine.as_h.delete("history") machine.as_h["testbox"] = JSON::Any.new(testbox) - @mq.pushlish_confirm(mq_queue, machine.to_json, durable: true) + @mq.publish_confirm(mq_queue, machine.to_json, durable: true) machine["state"] = "rebooting_queue" machine["time"] = Time.local.to_s("%Y-%m-%dT%H:%M:%S+0800") diff --git a/src/lib/mq.cr b/src/lib/mq.cr index 980585d..1c7f4b7 100644 --- a/src/lib/mq.cr +++ b/src/lib/mq.cr @@ -37,7 +37,7 @@ class MQClient Singleton::Of(self).instance end - def pushlish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish_confirm(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) @@ -46,7 +46,7 @@ class MQClient end end - def pushlish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 9528f54..ac677de 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -219,9 +219,9 @@ class Sched return nil end - def mq_pushlish_confirm(queue, msg) + def mq_publish_confirm(queue, msg) 3.times do - @mq.pushlish_confirm(queue, msg) + @mq.publish_confirm(queue, msg) break rescue e res = @mq.reconnect diff --git a/src/scheduler/close_job.cr b/src/scheduler/close_job.cr index e014361..a57b233 100644 --- a/src/scheduler/close_job.cr +++ b/src/scheduler/close_job.cr @@ -45,7 +45,7 @@ class Sched "job_state" => "close", "time" => get_time } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end end diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 4a8ea8a..416f8f4 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -32,7 +32,7 @@ class Sched "time" => get_time, "job_state" => "boot" } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end # auto submit a job to collect the host information. diff --git a/src/scheduler/update_job_parameter.cr b/src/scheduler/update_job_parameter.cr index d81f28b..92b905e 100644 --- a/src/scheduler/update_job_parameter.cr +++ b/src/scheduler/update_job_parameter.cr @@ -44,6 +44,6 @@ class Sched "job_state" => (@env.get?("job_state") || "update").to_s, "time" => @env.get?("time").to_s } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end -- 2.23.0

1 0

[PATCH compass-ci] container/dracut-initrd: add e2fsck and basename into dracut module 00bash
by Xu Xijian 16 Mar '21

16 Mar '21

1. mke2fs is depended by e2fsck. 2. We've add basename into 98dracut module, but it's too late, so add it into an earlier module 00bash. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 494588a..48e8bcc 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -18,6 +18,7 @@ cp -a /usr/local/bin/cifs-lib.sh /usr/lib/dracut/modules.d/95cifs/ cat overlay-lkp.sh >> /usr/lib/dracut/modules.d/90overlay-root/overlay-mount.sh sed -i "/install() {/a\ inst /usr/bin/awk" /usr/lib/dracut/modules.d/40network/module-setup.sh sed -i "/install() {/a\ inst /sbin/mke2fs /usr/bin/basename" /usr/lib/dracut/modules.d/98dracut-systemd/module-setup.sh +sed -i "/install() {/a\ inst /sbin/e2fsck /sbin/mke2fs /usr/bin/basename" /usr/lib/dracut/modules.d/00bash/module-setup.sh pre_mount_file="/usr/lib/dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh" [ "$(sed -n '$p' $pre_mount_file)" = "exit 0" ] && sed -i '$d' "$pre_mount_file" cat set-local-sysroot.sh >> "$pre_mount_file" -- 2.23.0

2 2

[PATCH v3 lkp-tests] add test script for dockerfile os migrating
by Bai Jing 16 Mar '21

16 Mar '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 151 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100755 tests/openeuler_docker diff --git a/tests/openeuler_docker b/tests/openeuler_docker new file mode 100755 index 000000000..b0b5336a3 --- /dev/null +++ b/tests/openeuler_docker @@ -0,0 +1,151 @@ +#!/bin/sh +# - dockerfile_url + +# exit 1 if openEuler failed +# exit 0 if +# - no centos support +# - centos failed + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/log.sh + +[ -n "$dockerfile_url" ] || die "dockerfile_url is empty" + +declare -i nr_openeuler_fail=0 +declare -i nr_openeuler_success=0 +declare -i is_openeuler_build_fail=0 +declare -i dockerfile_count=0 + +fit_repo() +{ + yum reinstall -y git + git clone "$dockerfile_url" "$BENCHMARK_ROOT/src/" + dockerfile_path="$(find "$BENCHMARK_ROOT/src/" -type f -name "*Dockerfile*" | xargs grep -HE '^FROM centos:(centos|)[78]')" + dockerfile_count=$(( $(echo $dockerfile_path | awk '{print NF}') / 2 )) + [ -n "$dockerfile_path" ] || { + echo "current-repo-not-support-centos: 1" + exit 0 + } +} + +openeuler_ready() +{ + docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar +} + +centos_docker() +{ + is_openeuler_build_fail=0 + index=1 + for file in $dockerfile_path + do + [[ $file =~ ^centos ]] && continue + file=$(echo "$file" | awk -F: '{print $1}') + dockerfile_name="${file##*/}" + cd "${file%/*}" || continue + image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') + [ -n "$image" ] && image=${image}_$index + if [ -f "build.sh" ] && [ -x "build.sh" ]; then + ./build.sh > /dev/null 2> /dev/null + else + docker_build > /dev/null 2> /dev/null + fi + + [ $? == 0 ] && { + handle_dockerfile + } + + let index++ + done +} + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" +} + +handle_epel() +{ + grep -qw "epel-release" "$dockerfile_name" && { + sed -i "s/epel-release/bash/g" "$dockerfile_name" + } +} + +handle_epelrpm() +{ + grep -qw "https://.*epel-release.*rpm" "$dockerfile_name" && { + sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" + } +} + +add_base_commands() +{ + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" +} + +handle_epel_repo() +{ + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" +} + +handle_dockerfile() +{ + is_openeuler_build_fail=1 + handle_FROM + handle_epel + add_base_commands + if docker_build "-openeuler"; then + docker_run "-openeuler" + let nr_openeuler_success++ + else + let nr_openeuler_fail++ + fi +} + +docker_build() +{ + docker build -f "$dockerfile_name" -t "${image}$1" . + [ $? != 0 ] && { + [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.fail: 1" + return 1 + } + [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.success: 1" + is_openeuler_build_fail=0 +} + +docker_run() +{ + docker run -d --name "${image}" "${image}$1" + docker_stats=$(docker inspect -f '{{.State.Running}}' ${image}) + [ $docker_stats == false ] && { + echo "${image}.run.fail: 1" + return 1 + } + + echo "${image}.run.success: 1" +} + +handle_build_result() +{ + if [ "$nr_openeuler_fail" != 0 ]; then + echo "openeuler build failed times: $nr_openeuler_fail" + echo "openeuler_valid_url: $dockerfile_url" + exit 1 + # centos all success, openeuler all success + elif [ "$nr_openeuler_success" == "$dockerfile_count" ]; then + echo "openeuler_valid_url: $dockerfile_url" + exit 0 + fi +} + +docker_test() +{ + fit_repo + openeuler_ready + centos_docker + handle_build_result +} + +docker_test -- 2.23.0

1 0

[PATCH lkp-tests] add test script for dockerfile os migrating
by Bai Jing 16 Mar '21

16 Mar '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 152 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100755 tests/openeuler_docker diff --git a/tests/openeuler_docker b/tests/openeuler_docker new file mode 100755 index 000000000..74087ab62 --- /dev/null +++ b/tests/openeuler_docker @@ -0,0 +1,152 @@ +#!/bin/sh +# - dockerfile_url + +# exit 1 if openEuler failed +# exit 0 if +# - no centos support +# - centos failed + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/log.sh + +[ -n "$dockerfile_url" ] || die "dockerfile_url is empty" + +declare -i nr_openeuler_fail=0 +declare -i nr_openeuler_success=0 +declare -i is_openeuler_build_fail=0 +declare -i dockerfile_count=0 + +fit_repo() +{ + yum reinstall -y git + git clone "$dockerfile_url" "$BENCHMARK_ROOT/src/" + dockerfile_path="$(find "$BENCHMARK_ROOT/src/" -type f -name "*Dockerfile*" | xargs grep -HE '^FROM centos:(centos|)[78]')" + dockerfile_count=$(( $(echo $dockerfile_path | awk '{print NF}') / 2 )) + [ -n "$dockerfile_path" ] || { + echo "current-repo-not-support-centos: 1" + exit 0 + } +} + +openeuler_ready() +{ + docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar +} + +centos_docker() +{ + is_openeuler_build_fail=0 + index=1 + for file in $dockerfile_path + do + [[ $file =~ ^centos ]] && continue + file=$(echo "$file" | awk -F: '{print $1}') + dockerfile_name="${file##*/}" + cd "${file%/*}" || continue + image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') + [ -n "$image" ] && image=${image}_$index + if [ -f "build.sh" ] && [ -x "build.sh" ]; then + ./build.sh > /dev/null 2> /dev/null + else + docker_build > /dev/null 2> /dev/null + fi + + [ $? == 0 ] && { + handle_dockerfile + } + + let index++ + done +} + + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" +} + +handle_epel() +{ + grep -qw "epel-release" "$dockerfile_name" && { + sed -i "s/epel-release/bash/g" "$dockerfile_name" + } +} + +handle_epelrpm() +{ + grep -qw "https://.*epel-release.*rpm" "$dockerfile_name" && { + sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" + } +} + +add_base_commands() +{ + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" +} + +handle_epel_repo() +{ + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" +} + +handle_dockerfile() +{ + is_openeuler_build_fail=1 + handle_FROM + handle_epel + add_base_commands + if docker_build "-openeuler"; then + docker_run "-openeuler" + let nr_openeuler_success++ + else + let nr_openeuler_fail++ + fi +} + +docker_build() +{ + docker build -f "$dockerfile_name" -t "${image}$1" . + [ $? != 0 ] && { + [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.fail: 1" + return 1 + } + [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.success: 1" + is_openeuler_build_fail=0 +} + +docker_run() +{ + docker run -d --name "${image}" "${image}$1" + docker_stats=$(docker inspect -f '{{.State.Running}}' ${image}) + [ $docker_stats == false ] && { + echo "${image}.run.fail: 1" + return 1 + } + + echo "${image}.run.success: 1" +} + +handle_build_result() +{ + if [ "$nr_openeuler_fail" != 0 ]; then + echo "openeuler build failed times: $nr_openeuler_fail" + echo "openeuler_valid_url: $dockerfile_url" + exit 1 + # centos all success, openeuler all success + elif [ "$nr_openeuler_success" == "$dockerfile_count" ]; then + echo "openeuler_valid_url: $dockerfile_url" + exit 0 + fi +} + +docker_test() +{ + fit_repo + openeuler_ready + centos_docker + handle_build_result +} + +docker_test -- 2.23.0

2 2

[PATCH compass-ci] sparrow: change service default ports
by Cao Xueliang 16 Mar '21

16 Mar '21

Reference: 6354ba631e0fd5f795a1dc28b65e2002bd3de668 Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/extract-stats/start | 2 +- sparrow/3-code/dev-env | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/container/extract-stats/start b/container/extract-stats/start index 5033754..621099a 100755 --- a/container/extract-stats/start +++ b/container/extract-stats/start @@ -25,7 +25,7 @@ env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' DEFAULT_CCI = '/c/compass-ci' SRV_HTTP_RESULT_HOST = defaults['SRV_HTTP_RESULT_HOST'] || Open3.capture3('curl ip.sb')[0].chomp -SRV_HTTP_RESULT_PORT = defaults['SRV_HTTP_RESULT_PORT'] || '11300' +SRV_HTTP_RESULT_PORT = defaults['SRV_HTTP_RESULT_PORT'] || '20007' docker_rm "extract-stats" cmd = %w[ diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index b740a15..65061d8 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -15,10 +15,10 @@ server_name=$(hostname | cut -f1 -d.) : ${SRV_HTTP_OS_HOST:=$server_ip} : ${SRV_HTTP_GIT_HOST:=$server_ip} : ${SRV_HTTP_CCI_HOST:=$server_ip} -: ${SRV_HTTP_RESULT_PORT:=11300} -: ${SRV_HTTP_OS_PORT:=11304} -: ${SRV_HTTP_GIT_PORT:=11305} -: ${SRV_HTTP_CCI_PORT:=11307} +: ${SRV_HTTP_RESULT_PORT:=20007} +: ${SRV_HTTP_OS_PORT:=20009} +: ${SRV_HTTP_GIT_PORT:=20010} +: ${SRV_HTTP_CCI_PORT:=20011} : ${LOGGING_ES_HOST:=$server_ip} : ${LOGGING_ES_PORT:=9202} : ${INITRD_HTTP_HOST:=$server_ip} -- 2.23.0

1 0

[PATCH compass-ci] src/lib(scheduer): change function name "*pushlish*" to "*publish*" for mq
by Xu Xijian 16 Mar '21

16 Mar '21

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/lifecycle.cr | 4 ++-- src/lib/mq.cr | 4 ++-- src/lib/sched.cr | 4 ++-- src/scheduler/close_job.cr | 2 +- src/scheduler/find_job_boot.cr | 2 +- src/scheduler/update_job_parameter.cr | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 1eb6264..4bd2ac0 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -79,7 +79,7 @@ class Lifecycle "job_state" => "abnormal", "testbox" => testbox } - @mq.pushlish_confirm("job_mq", msg.to_json) + @mq.publish_confirm("job_mq", msg.to_json) @match[testbox].delete(id) end end @@ -313,7 +313,7 @@ class Lifecycle mq_queue = get_machine_reboot_queue(testbox) machine.as_h.delete("history") machine.as_h["testbox"] = JSON::Any.new(testbox) - @mq.pushlish_confirm(mq_queue, machine.to_json, durable: true) + @mq.publish_confirm(mq_queue, machine.to_json, durable: true) machine["state"] = "rebooting_queue" machine["time"] = Time.local.to_s("%Y-%m-%dT%H:%M:%S+0800") diff --git a/src/lib/mq.cr b/src/lib/mq.cr index 980585d..1c7f4b7 100644 --- a/src/lib/mq.cr +++ b/src/lib/mq.cr @@ -37,7 +37,7 @@ class MQClient Singleton::Of(self).instance end - def pushlish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish_confirm(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) @@ -46,7 +46,7 @@ class MQClient end end - def pushlish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 9528f54..ac677de 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -219,9 +219,9 @@ class Sched return nil end - def mq_pushlish_confirm(queue, msg) + def mq_publish_confirm(queue, msg) 3.times do - @mq.pushlish_confirm(queue, msg) + @mq.publish_confirm(queue, msg) break rescue e res = @mq.reconnect diff --git a/src/scheduler/close_job.cr b/src/scheduler/close_job.cr index e014361..a57b233 100644 --- a/src/scheduler/close_job.cr +++ b/src/scheduler/close_job.cr @@ -45,7 +45,7 @@ class Sched "job_state" => "close", "time" => get_time } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end end diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 4a8ea8a..416f8f4 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -32,7 +32,7 @@ class Sched "time" => get_time, "job_state" => "boot" } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end # auto submit a job to collect the host information. diff --git a/src/scheduler/update_job_parameter.cr b/src/scheduler/update_job_parameter.cr index d81f28b..92b905e 100644 --- a/src/scheduler/update_job_parameter.cr +++ b/src/scheduler/update_job_parameter.cr @@ -44,6 +44,6 @@ class Sched "job_state" => (@env.get?("job_state") || "update").to_s, "time" => @env.get?("time").to_s } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end -- 2.23.0

1 0

[PATCH v3 lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 16 Mar '21

16 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 47 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..d6acad1ab 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,40 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + elsif error_msg.is_a? String + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/[^\w]/, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line.gsub!(/^-|-$/, '') + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

1 0