- Compass-ci - mailweb.openeuler.org

[PATCH lkp-tests] rpmbuild for openeuler
by Li Ping 16 Mar '21

16 Mar '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- jobs/rpmbuild-pkg.yaml | 2 +- tests/rpmbuild-pkg | 46 +++++++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/jobs/rpmbuild-pkg.yaml b/jobs/rpmbuild-pkg.yaml index 53e145b3b..a70590afe 100644 --- a/jobs/rpmbuild-pkg.yaml +++ b/jobs/rpmbuild-pkg.yaml @@ -1,5 +1,5 @@ suite: rpmbuild-pkg category: functional -rpm_repo: +upstream_repo: rpmbuild-pkg: diff --git a/tests/rpmbuild-pkg b/tests/rpmbuild-pkg index b641e97d1..034e4f3e1 100755 --- a/tests/rpmbuild-pkg +++ b/tests/rpmbuild-pkg @@ -1,12 +1,16 @@ #!/bin/bash -# - rpm_repo +# - upstream_repo +# - compatible_os . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/upload.sh -[ -n "$rpm_repo" ] || die "rpm_repo is empty" -package_name=${rpm_repo##*/} -sync_dest="/initrd/rpmbuild-pkg/${os}/${os_arch}/${os_version}/${package_name}" +: "${compatible_os:=compat-openeuler}" + +[ -n "$upstream_repo" ] || die "upstream_repo is empty" +package_name=${upstream_repo##*/} +rpm_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compatible_os}/${os_arch}/Packages" +source_dest="/initrd/rpmbuild-pkg/${os}-${os_version}/${compatible_os}/source/Packages" init_workspace() { @@ -14,21 +18,16 @@ init_workspace() rpmdev-setuptree } -get_pkgfile() +download_upstream_repo() { - curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-show", "HEAD:'$1'"]}' -o "${2}" -} - -download_rpm_repo() -{ - local filelist=$(curl -sS -H 'Content-Type: Application/json' -XPOST "$GIT_SERVER"':8100/git_command' \ - -d '{"git_repo": "'${rpm_repo}'", "git_command": ["git-ls-files", "."]}') - for pkgfile in "${filelist[@]}" + git clone "git://$GIT_SERVER/openeuler/${upstream_repo}" || die "clone git repo ${package_name} failed: git://$GIT_SERVER/openeuler/${upstream_repo}" + cd $package_name || exit + filelist=$(git ls-files) + for pkgfile in ${filelist[@]} do local dir="SOURCES" echo $pkgfile | egrep "\.spec$" && dir="SPECS" - get_pkgfile "$pkgfile" "${HOME}/rpmbuild/${dir}/$pkgfile" + mv "$pkgfile" "${HOME}/rpmbuild/${dir}/" done } @@ -50,19 +49,28 @@ build_rpm() show_rpm_files() { find ${HOME}/rpmbuild/RPMS -type f -name "*.rpm" +} + +show_src_rpm_files() +{ find ${HOME}/rpmbuild/SRPMS -type f -name "*.rpm" } upload_rpm_pkg() { - local file - for file in $(show_rpm_files) + local rpm_file + for rpm_file in $(show_rpm_files) + do + upload_one_curl ${rpm_file} ${rpm_dest} + done + local src_rpm_file + for src_rpm_file in $(show_src_rpm_files) do - upload_one_curl ${file} ${sync_dest} + upload_one_curl ${src_rpm_file} ${source_dest} done } init_workspace -download_rpm_repo +download_upstream_repo build_rpm upload_rpm_pkg -- 2.23.0

1 0

[PATCH compass-ci] src/scheduler/find_job_boot.cr: merge 2 similar functions
by Xu Xijian 16 Mar '21

16 Mar '21

ipxe_msg() and grub_msg() have the same function, they just diff at boot_type in ret msg. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/scheduler/find_job_boot.cr | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 416f8f4..78b10ad 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -126,16 +126,8 @@ class Sched return boot_content(job, boot_type) end - private def ipxe_msg(msg) - "#!ipxe - echo ... - echo #{msg} - echo ... - reboot" - end - - private def grub_msg(msg) - "#!grub + private def boot_msg(boot_type, msg) + "#!#{boot_type} echo ... echo #{msg} echo ... @@ -220,9 +212,9 @@ class Sched case boot_type when "ipxe" - return job ? get_boot_ipxe(job) : ipxe_msg("No job now") + return job ? get_boot_ipxe(job) : boot_msg(boot_type, "No job now") when "grub" - return job ? get_boot_grub(job) : grub_msg("No job now") + return job ? get_boot_grub(job) : boot_msg(boot_type, "No job now") when "container" return job ? get_boot_container(job) : Hash(String, String).new.to_json when "libvirt" -- 2.23.0

1 0

[PATCH lkp-tests 2/2] tests/monitor-job: refactor monitor-job submit job
by Wei Jihui 16 Mar '21

16 Mar '21

add my-token for submit job, so del $CCI_SRC/sbin/build-my-info Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- tests/monitor-job | 59 ++++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/tests/monitor-job b/tests/monitor-job index 5da992fa..8f8ac682 100755 --- a/tests/monitor-job +++ b/tests/monitor-job @@ -2,60 +2,67 @@ # - monitor_repo # - job_yaml # - job_testbox +# lab +# my_name +# my_email +# my_token . $LKP_SRC/lib/log.sh -set_scheduler_config() +set_submit_config() { - mkdir -p /etc/compass-ci/defaults - cat > /etc/compass-ci/defaults/$lab.yaml <<-EOF + mkdir -p ~/.config/compass-ci/defaults/ + cat > ~/.config/compass-ci/defaults/"$lab".yaml <<-EOF SCHED_HOST: $SCHED_HOST SCHED_PORT: $SCHED_PORT lab: $lab + EOF + + cat > ~/.config/compass-ci/defaults/account.yaml <<-EOF + my_name: $my_name my_email: $my_email EOF -} -install_gem_pkgs() -{ - gem sources -r http://rubygems.org -a https://gems.ruby-china.com/ - gem install git activesupport rest-client faye-websocket + my_token=${my_token:-$secrets_my_token} + mkdir -p ~/.config/compass-ci/include/lab/ + cat > ~/.config/compass-ci/include/lab/"$lab".yaml <<-EOF + my_token: $my_token + EOF } -build_my_info() +install_lkp() { - git clone git://$GIT_SERVER/gitee.com/wu_fengguang/compass-ci.git "$CCI_SRC" - ruby "$CCI_SRC"/sbin/build-my-info.rb -e "$my_email" -n "$my_name" -l "$lab" - git clone git://$GIT_SERVER/gitee.com/wu_fengguang/lkp-tests.git "$LKP_SRC" + cd "$LKP_SRC" && make install } download_monitor_repo() { - if [ -n "$monitor_repo" ]; then - git clone git://"$LKP_SERVER"/"$monitor_repo" - else - log_error "monitor_repo is not specified" && exit 1 - fi - - test_code=${monitor_repo##*/} - cp -r "$test_code"/* $LKP_SRC/ + git clone git://"$GIT_SERVER"/"$monitor_repo" "$repo_path" + cp -r "$repo_path"/* "$LKP_SRC"/ } submit_job() { - yaml=${job_yaml##*/} - "$LKP_SRC"/sbin/submit -a "$yaml" testbox="$job_testbox" + if [ -f "$LKP_SRC"/"$job_yaml" ]; then + job_yaml="$LKP_SRC"/"$job_yaml" + else + job_yaml=${job_yaml##*/} + fi + + "$LKP_SRC"/sbin/submit -a "$job_yaml" testbox="$job_testbox" } main() { - local CCI_SRC=/c/compass-ci + [ -z "$monitor_repo" ] && log_error "monitor_repo is not specified" && exit 1 + + local repo=${monitor_repo##*/} + local repo_path=/c/"$repo" local LKP_SRC=/c/lkp-tests - install_gem_pkgs - set_scheduler_config - build_my_info + set_submit_config + install_lkp download_monitor_repo submit_job } -- 2.23.0

1 0

[PATCH lkp-tests 1/2] distro/depends/monitor-job: add sudo
by Wei Jihui 16 Mar '21

16 Mar '21

monitor-job task need install lkp-tests, so sudo cmd is needed. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- distro/depends/monitor-job | 1 + 1 file changed, 1 insertion(+) diff --git a/distro/depends/monitor-job b/distro/depends/monitor-job index 5bfafa87..cb01a8c8 100644 --- a/distro/depends/monitor-job +++ b/distro/depends/monitor-job @@ -1,3 +1,4 @@ git +sudo ruby-dev build-essential -- 2.23.0

1 0

[PATCH lkp-tests] change service ports
by Wu Zhende 16 Mar '21

16 Mar '21

monitoring: 11310 to 20001 scheduler-https: 11306 to 20002 sshr: 50000-51999 to 21000-23999 Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- daemon/sshd | 2 +- include/lab/z9 | 2 +- lib/monitor.rb | 4 ++-- sbin/monitor | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/daemon/sshd b/daemon/sshd index 7ba471a88..50a7677ab 100755 --- a/daemon/sshd +++ b/daemon/sshd @@ -15,7 +15,7 @@ PROJECT_DIR=$(dirname $SCRIPT_DIR) sshr_ip=$LKP_SERVER [ -n "$sshr_port" ] || sshr_port=5050 [ -n "$sshr_port_len" ] || sshr_port_len=2000 -[ -n "$sshr_port_base" ] || sshr_port_base=50000 +[ -n "$sshr_port_base" ] || sshr_port_base=21000 generate_key_file() { diff --git a/include/lab/z9 b/include/lab/z9 index 0592fdb6a..68c7aafc1 100644 --- a/include/lab/z9 +++ b/include/lab/z9 @@ -1,2 +1,2 @@ SCHED_HOST: api.compass-ci.openeuler.org -SCHED_PORT: 11306 +SCHED_PORT: 20002 diff --git a/lib/monitor.rb b/lib/monitor.rb index 24191fa8d..b08fdf4c9 100755 --- a/lib/monitor.rb +++ b/lib/monitor.rb @@ -34,7 +34,7 @@ class Monitor return unless @monitor_url == '' if host = @job['SCHED_HOST'] - return @monitor_url = "ws://#{host}:11310/filter" + return @monitor_url = "ws://#{host}:20001/filter" end Dir.glob(['/etc/compass-ci/monitor/*.yaml', @@ -49,7 +49,7 @@ class Monitor revise_hash(@defaults, defaults, true) end - @monitor_url = @defaults['monitor_url'] || 'ws://localhost:11310/filter' + @monitor_url = @defaults['monitor_url'] || 'ws://localhost:20001/filter' end def merge_overrides diff --git a/sbin/monitor b/sbin/monitor index a2e677fc0..f3ec73469 100755 --- a/sbin/monitor +++ b/sbin/monitor @@ -26,7 +26,7 @@ options = OptionParser.new do |opts| opts.separator 'options:' opts.on('-u URL', '--url URL', - 'set Message Server API address, default is ws://localhost:11310/filter') do |monitor_url| + 'set Message Server API address, default is ws://localhost:20001/filter') do |monitor_url| opt_monitor_url = monitor_url end -- 2.23.0

1 0

[PATCH compass-ci] service: change port
by Wu Zhende 16 Mar '21

16 Mar '21

kibana: 11309 to 20000 monitoring: 11310 to 20001 scheduler-https: 11306 to 20002 sshr: 50000-51999 to 21000-23999 Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana/start | 2 +- container/monitoring/build | 2 +- container/monitoring/start | 2 +- container/scheduler-https/start | 4 ++-- container/ssh-r/Dockerfile | 2 +- container/ssh-r/start | 2 +- doc/manual/submit-job.en.md | 4 ++-- doc/manual/submit-job.zh.md | 4 ++-- src/delimiter/constants.rb | 2 +- src/monitoring/constants.cr | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/container/kibana/start b/container/kibana/start index dbb3c56..26a4944 100755 --- a/container/kibana/start +++ b/container/kibana/start @@ -24,7 +24,7 @@ cmd = %W[ -v /etc/localtime:/etc/localtime:ro -d -e ELASTICSEARCH_HOSTS=http://#{LOGGING_ES_HOST}:#{LOGGING_ES_PORT} - -p 11309:5601 + -p 20000:5601 kibana:7.6.2 ] diff --git a/container/monitoring/build b/container/monitoring/build index 121ede0..4865c70 100755 --- a/container/monitoring/build +++ b/container/monitoring/build @@ -9,7 +9,7 @@ require_relative '../defconfig.rb' names = Set.new ['MONITOR_PORT'] defaults = relevant_defaults(names) -MONITOR_PORT = (defaults['MONITOR_PORT'] || '11310') +MONITOR_PORT = (defaults['MONITOR_PORT'] || '20001') docker_skip_rebuild "monitoring-#{MONITOR_PORT}" diff --git a/container/monitoring/start b/container/monitoring/start index 2e2946c..f4214ed 100755 --- a/container/monitoring/start +++ b/container/monitoring/start @@ -17,7 +17,7 @@ env = docker_env(defaults) DEFAULT_CCI = '/c/cci' DEFAULT_LKP = '/c/lkp-tests' -MONITOR_PORT = defaults['MONITOR_PORT'] || '11310' +MONITOR_PORT = defaults['MONITOR_PORT'] || '20001' docker_rm "monitoring-#{MONITOR_PORT}" cmd = %W[ diff --git a/container/scheduler-https/start b/container/scheduler-https/start index 98940d7..ee93e79 100755 --- a/container/scheduler-https/start +++ b/container/scheduler-https/start @@ -39,12 +39,12 @@ names = Set.new %w[ defaults = relevant_defaults(names) defaults['SSHR_PORT'] ||= 5050 -defaults['SSHR_PORT_BASE'] ||= 50000 +defaults['SSHR_PORT_BASE'] ||= 21000 defaults['SSHR_PORT_LEN'] ||= 2000 defaults['SCHED_PORT'] ||= '3000' defaults['SCHED_HOST'] ||= '172.17.0.1' defaults['MASTER_FLUENTD_PORT'] ||= '24224' -defaults['SCHED_HTTPS_PORT'] ||= 11306 +defaults['SCHED_HTTPS_PORT'] ||= 20002 defaults['SCHED_HTTPS_HOST'] ||= 'api.compass-ci.openeuler.org' env = docker_env(defaults) diff --git a/container/ssh-r/Dockerfile b/container/ssh-r/Dockerfile index 65a13aa..416e0ef 100644 --- a/container/ssh-r/Dockerfile +++ b/container/ssh-r/Dockerfile @@ -21,7 +21,7 @@ RUN chown -R sshr:sshr /home/sshr/.ssh/authorized_keys \ && chmod 600 /home/sshr/.ssh/authorized_keys \ && passwd -u sshr -EXPOSE 50000-51999 +EXPOSE 21000-23999 EXPOSE 22 COPY entry.sh /entry.sh diff --git a/container/ssh-r/start b/container/ssh-r/start index 5c800f3..6c7a373 100755 --- a/container/ssh-r/start +++ b/container/ssh-r/start @@ -15,7 +15,7 @@ cmd=( -e GATEWAY_PORTS=true -e TCP_FORWARDING=true -d - -p 50000-51999:50000-51999 + -p 21000-23999:21000-23999 -p 5050:22 -v /etc/localtime:/etc/localtime:ro -v /srv/pub/sshr/keys/:/etc/ssh/keys diff --git a/doc/manual/submit-job.en.md b/doc/manual/submit-job.en.md index de73f8f..5106bd9 100644 --- a/doc/manual/submit-job.en.md +++ b/doc/manual/submit-job.en.md @@ -141,7 +141,7 @@ options: hi8109@account-vm ~% submit -m iperf.yaml submit iperf.yaml, got job_id=z9.173923 query=>{"job_id":["z9.173923"]} - connect to ws://172.168.131.2:11310/filter + connect to ws://172.168.131.2:20001/filter {"job_id":"z9.173923","message":"","job_state":"submit","result_root":"/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch6 {"job_id": "z9.173923", "result_root": "/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/tcp-30/z9.173923", "job_state {"job_id": "z9.173923", "job_state": "boot"} @@ -175,7 +175,7 @@ options: hi8109@account-vm ~% submit -m -c borrow-1h.yaml submit borrow-1h.yaml, got job_id=z9.173925 query=>{"job_id":["z9.173925"]} - connect to ws://172.168.131.2:11310/filter + connect to ws://172.168.131.2:20001/filter {"job_id":"z9.173925","message":"","job_state":"submit","result_root":"/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch {"job_id": "z9.173925", "result_root": "/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/3600/z9.173925", "job_state" {"job_id": "z9.173925", "job_state": "boot"} diff --git a/doc/manual/submit-job.zh.md b/doc/manual/submit-job.zh.md index 77e9566..5f1838b 100644 --- a/doc/manual/submit-job.zh.md +++ b/doc/manual/submit-job.zh.md @@ -140,7 +140,7 @@ options: hi8109@account-vm ~% submit -m iperf.yaml testbox=vm-2p8g submit iperf.yaml, got job_id=z9.173923 query=>{"job_id":["z9.173923"]} - connect to ws://172.168.131.2:11310/filter + connect to ws://172.168.131.2:20001/filter {"job_id":"z9.173923","message":"","job_state":"submit","result_root":"/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch6 {"job_id": "z9.173923", "result_root": "/srv/result/iperf/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/tcp-30/z9.173923", "job_state {"job_id": "z9.173923", "job_state": "boot"} @@ -173,7 +173,7 @@ options: hi8109@account-vm ~% submit -m -c borrow-1h.yaml testbox=vm-2p8g submit borrow-1h.yaml, got job_id=z9.173925 query=>{"job_id":["z9.173925"]} - connect to ws://172.168.131.2:11310/filter + connect to ws://172.168.131.2:20001/filter {"job_id":"z9.173925","message":"","job_state":"submit","result_root":"/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch {"job_id": "z9.173925", "result_root": "/srv/result/borrow/2020-11-30/vm-2p8g/openeuler-20.03-aarch64/3600/z9.173925", "job_state" {"job_id": "z9.173925", "job_state": "boot"} diff --git a/src/delimiter/constants.rb b/src/delimiter/constants.rb index a790736..da3f075 100644 --- a/src/delimiter/constants.rb +++ b/src/delimiter/constants.rb @@ -4,7 +4,7 @@ LKP_SRC ||= ENV['LKP_SRC'] || '/c/lkp-tests' MONITOR_HOST ||= ENV['MONITOR_HOST'] || '172.17.0.1' -MONITOR_PORT ||= ENV['MONITOR_PORT'] || '11310' +MONITOR_PORT ||= ENV['MONITOR_PORT'] || '20001' GIT_MIRROR_HOST ||= ENV['GIT_MIRROR_HOST'] || '172.17.0.1' DELIMITER_EMAIL ||= ENV['DELIMITER_EMAIL'] || 'delimiter@localhost' BISECT_RUN_SCRIPT = "#{ENV['CCI_SRC']}/src/delimiter/find-commit/bisect_run_script.rb" diff --git a/src/monitoring/constants.cr b/src/monitoring/constants.cr index 154c536..9e080c8 100644 --- a/src/monitoring/constants.cr +++ b/src/monitoring/constants.cr @@ -1,4 +1,4 @@ MQ_HOST = (ENV.has_key?("MQ_HOST") ? ENV["MQ_HOST"] : "172.17.0.1") MQ_PORT = (ENV.has_key?("MQ_PORT") ? ENV["MQ_PORT"] : 5672).to_i32 -MONITOR_PORT = (ENV.has_key?("MONITOR_PORT") ? ENV["MONITOR_PORT"] : 11310).to_i32 +MONITOR_PORT = (ENV.has_key?("MONITOR_PORT") ? ENV["MONITOR_PORT"] : 20001).to_i32 -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] container/dracut-initrd: add e2fsck and basename into dracut module 00bash
by Xu Xijian 16 Mar '21

16 Mar '21

1. mke2fs is depended by e2fsck. 2. We've add basename into 98dracut module, but it's too late, so add it into an earlier module 00bash. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 494588a..616183f 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -17,7 +17,8 @@ cp -a /usr/local/bin/cifs-lib.sh /usr/lib/dracut/modules.d/95cifs/ cat overlay-lkp.sh >> /usr/lib/dracut/modules.d/90overlay-root/overlay-mount.sh sed -i "/install() {/a\ inst /usr/bin/awk" /usr/lib/dracut/modules.d/40network/module-setup.sh -sed -i "/install() {/a\ inst /sbin/mke2fs /usr/bin/basename" /usr/lib/dracut/modules.d/98dracut-systemd/module-setup.sh +sed -i "/install() {/a\ inst /sbin/e2fsck /sbin/mke2fs /usr/bin/basename" /usr/lib/dracut/modules.d/00bash/module-setup.sh + pre_mount_file="/usr/lib/dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh" [ "$(sed -n '$p' $pre_mount_file)" = "exit 0" ] && sed -i '$d' "$pre_mount_file" cat set-local-sysroot.sh >> "$pre_mount_file" -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] src/lib(scheduer): change function name "*pushlish*" to "*publish*" for mq
by Xu Xijian 16 Mar '21

16 Mar '21

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/lifecycle.cr | 4 ++-- src/lib/mq.cr | 4 ++-- src/lib/sched.cr | 4 ++-- src/scheduler/close_job.cr | 2 +- src/scheduler/find_job_boot.cr | 2 +- src/scheduler/update_job_parameter.cr | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 1eb6264..4bd2ac0 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -79,7 +79,7 @@ class Lifecycle "job_state" => "abnormal", "testbox" => testbox } - @mq.pushlish_confirm("job_mq", msg.to_json) + @mq.publish_confirm("job_mq", msg.to_json) @match[testbox].delete(id) end end @@ -313,7 +313,7 @@ class Lifecycle mq_queue = get_machine_reboot_queue(testbox) machine.as_h.delete("history") machine.as_h["testbox"] = JSON::Any.new(testbox) - @mq.pushlish_confirm(mq_queue, machine.to_json, durable: true) + @mq.publish_confirm(mq_queue, machine.to_json, durable: true) machine["state"] = "rebooting_queue" machine["time"] = Time.local.to_s("%Y-%m-%dT%H:%M:%S+0800") diff --git a/src/lib/mq.cr b/src/lib/mq.cr index 980585d..1c7f4b7 100644 --- a/src/lib/mq.cr +++ b/src/lib/mq.cr @@ -37,7 +37,7 @@ class MQClient Singleton::Of(self).instance end - def pushlish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish_confirm(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) @@ -46,7 +46,7 @@ class MQClient end end - def pushlish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 9528f54..ac677de 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -219,9 +219,9 @@ class Sched return nil end - def mq_pushlish_confirm(queue, msg) + def mq_publish_confirm(queue, msg) 3.times do - @mq.pushlish_confirm(queue, msg) + @mq.publish_confirm(queue, msg) break rescue e res = @mq.reconnect diff --git a/src/scheduler/close_job.cr b/src/scheduler/close_job.cr index e014361..a57b233 100644 --- a/src/scheduler/close_job.cr +++ b/src/scheduler/close_job.cr @@ -45,7 +45,7 @@ class Sched "job_state" => "close", "time" => get_time } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end end diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 4a8ea8a..416f8f4 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -32,7 +32,7 @@ class Sched "time" => get_time, "job_state" => "boot" } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end # auto submit a job to collect the host information. diff --git a/src/scheduler/update_job_parameter.cr b/src/scheduler/update_job_parameter.cr index d81f28b..92b905e 100644 --- a/src/scheduler/update_job_parameter.cr +++ b/src/scheduler/update_job_parameter.cr @@ -44,6 +44,6 @@ class Sched "job_state" => (@env.get?("job_state") || "update").to_s, "time" => @env.get?("time").to_s } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end -- 2.23.0

1 0

[PATCH compass-ci] container/dracut-initrd: add e2fsck and basename into dracut module 00bash
by Xu Xijian 16 Mar '21

16 Mar '21

1. mke2fs is depended by e2fsck. 2. We've add basename into 98dracut module, but it's too late, so add it into an earlier module 00bash. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 494588a..48e8bcc 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -18,6 +18,7 @@ cp -a /usr/local/bin/cifs-lib.sh /usr/lib/dracut/modules.d/95cifs/ cat overlay-lkp.sh >> /usr/lib/dracut/modules.d/90overlay-root/overlay-mount.sh sed -i "/install() {/a\ inst /usr/bin/awk" /usr/lib/dracut/modules.d/40network/module-setup.sh sed -i "/install() {/a\ inst /sbin/mke2fs /usr/bin/basename" /usr/lib/dracut/modules.d/98dracut-systemd/module-setup.sh +sed -i "/install() {/a\ inst /sbin/e2fsck /sbin/mke2fs /usr/bin/basename" /usr/lib/dracut/modules.d/00bash/module-setup.sh pre_mount_file="/usr/lib/dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh" [ "$(sed -n '$p' $pre_mount_file)" = "exit 0" ] && sed -i '$d' "$pre_mount_file" cat set-local-sysroot.sh >> "$pre_mount_file" -- 2.23.0

2 2

[PATCH v3 lkp-tests] add test script for dockerfile os migrating
by Bai Jing 16 Mar '21

16 Mar '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 151 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100755 tests/openeuler_docker diff --git a/tests/openeuler_docker b/tests/openeuler_docker new file mode 100755 index 000000000..b0b5336a3 --- /dev/null +++ b/tests/openeuler_docker @@ -0,0 +1,151 @@ +#!/bin/sh +# - dockerfile_url + +# exit 1 if openEuler failed +# exit 0 if +# - no centos support +# - centos failed + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/log.sh + +[ -n "$dockerfile_url" ] || die "dockerfile_url is empty" + +declare -i nr_openeuler_fail=0 +declare -i nr_openeuler_success=0 +declare -i is_openeuler_build_fail=0 +declare -i dockerfile_count=0 + +fit_repo() +{ + yum reinstall -y git + git clone "$dockerfile_url" "$BENCHMARK_ROOT/src/" + dockerfile_path="$(find "$BENCHMARK_ROOT/src/" -type f -name "*Dockerfile*" | xargs grep -HE '^FROM centos:(centos|)[78]')" + dockerfile_count=$(( $(echo $dockerfile_path | awk '{print NF}') / 2 )) + [ -n "$dockerfile_path" ] || { + echo "current-repo-not-support-centos: 1" + exit 0 + } +} + +openeuler_ready() +{ + docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar +} + +centos_docker() +{ + is_openeuler_build_fail=0 + index=1 + for file in $dockerfile_path + do + [[ $file =~ ^centos ]] && continue + file=$(echo "$file" | awk -F: '{print $1}') + dockerfile_name="${file##*/}" + cd "${file%/*}" || continue + image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') + [ -n "$image" ] && image=${image}_$index + if [ -f "build.sh" ] && [ -x "build.sh" ]; then + ./build.sh > /dev/null 2> /dev/null + else + docker_build > /dev/null 2> /dev/null + fi + + [ $? == 0 ] && { + handle_dockerfile + } + + let index++ + done +} + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" +} + +handle_epel() +{ + grep -qw "epel-release" "$dockerfile_name" && { + sed -i "s/epel-release/bash/g" "$dockerfile_name" + } +} + +handle_epelrpm() +{ + grep -qw "https://.*epel-release.*rpm" "$dockerfile_name" && { + sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" + } +} + +add_base_commands() +{ + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" +} + +handle_epel_repo() +{ + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" +} + +handle_dockerfile() +{ + is_openeuler_build_fail=1 + handle_FROM + handle_epel + add_base_commands + if docker_build "-openeuler"; then + docker_run "-openeuler" + let nr_openeuler_success++ + else + let nr_openeuler_fail++ + fi +} + +docker_build() +{ + docker build -f "$dockerfile_name" -t "${image}$1" . + [ $? != 0 ] && { + [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.fail: 1" + return 1 + } + [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.success: 1" + is_openeuler_build_fail=0 +} + +docker_run() +{ + docker run -d --name "${image}" "${image}$1" + docker_stats=$(docker inspect -f '{{.State.Running}}' ${image}) + [ $docker_stats == false ] && { + echo "${image}.run.fail: 1" + return 1 + } + + echo "${image}.run.success: 1" +} + +handle_build_result() +{ + if [ "$nr_openeuler_fail" != 0 ]; then + echo "openeuler build failed times: $nr_openeuler_fail" + echo "openeuler_valid_url: $dockerfile_url" + exit 1 + # centos all success, openeuler all success + elif [ "$nr_openeuler_success" == "$dockerfile_count" ]; then + echo "openeuler_valid_url: $dockerfile_url" + exit 0 + fi +} + +docker_test() +{ + fit_repo + openeuler_ready + centos_docker + handle_build_result +} + +docker_test -- 2.23.0

1 0

[PATCH lkp-tests] add test script for dockerfile os migrating
by Bai Jing 16 Mar '21

16 Mar '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 152 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100755 tests/openeuler_docker diff --git a/tests/openeuler_docker b/tests/openeuler_docker new file mode 100755 index 000000000..74087ab62 --- /dev/null +++ b/tests/openeuler_docker @@ -0,0 +1,152 @@ +#!/bin/sh +# - dockerfile_url + +# exit 1 if openEuler failed +# exit 0 if +# - no centos support +# - centos failed + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/log.sh + +[ -n "$dockerfile_url" ] || die "dockerfile_url is empty" + +declare -i nr_openeuler_fail=0 +declare -i nr_openeuler_success=0 +declare -i is_openeuler_build_fail=0 +declare -i dockerfile_count=0 + +fit_repo() +{ + yum reinstall -y git + git clone "$dockerfile_url" "$BENCHMARK_ROOT/src/" + dockerfile_path="$(find "$BENCHMARK_ROOT/src/" -type f -name "*Dockerfile*" | xargs grep -HE '^FROM centos:(centos|)[78]')" + dockerfile_count=$(( $(echo $dockerfile_path | awk '{print NF}') / 2 )) + [ -n "$dockerfile_path" ] || { + echo "current-repo-not-support-centos: 1" + exit 0 + } +} + +openeuler_ready() +{ + docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar +} + +centos_docker() +{ + is_openeuler_build_fail=0 + index=1 + for file in $dockerfile_path + do + [[ $file =~ ^centos ]] && continue + file=$(echo "$file" | awk -F: '{print $1}') + dockerfile_name="${file##*/}" + cd "${file%/*}" || continue + image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') + [ -n "$image" ] && image=${image}_$index + if [ -f "build.sh" ] && [ -x "build.sh" ]; then + ./build.sh > /dev/null 2> /dev/null + else + docker_build > /dev/null 2> /dev/null + fi + + [ $? == 0 ] && { + handle_dockerfile + } + + let index++ + done +} + + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" +} + +handle_epel() +{ + grep -qw "epel-release" "$dockerfile_name" && { + sed -i "s/epel-release/bash/g" "$dockerfile_name" + } +} + +handle_epelrpm() +{ + grep -qw "https://.*epel-release.*rpm" "$dockerfile_name" && { + sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" + } +} + +add_base_commands() +{ + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" +} + +handle_epel_repo() +{ + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" +} + +handle_dockerfile() +{ + is_openeuler_build_fail=1 + handle_FROM + handle_epel + add_base_commands + if docker_build "-openeuler"; then + docker_run "-openeuler" + let nr_openeuler_success++ + else + let nr_openeuler_fail++ + fi +} + +docker_build() +{ + docker build -f "$dockerfile_name" -t "${image}$1" . + [ $? != 0 ] && { + [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.fail: 1" + return 1 + } + [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.success: 1" + is_openeuler_build_fail=0 +} + +docker_run() +{ + docker run -d --name "${image}" "${image}$1" + docker_stats=$(docker inspect -f '{{.State.Running}}' ${image}) + [ $docker_stats == false ] && { + echo "${image}.run.fail: 1" + return 1 + } + + echo "${image}.run.success: 1" +} + +handle_build_result() +{ + if [ "$nr_openeuler_fail" != 0 ]; then + echo "openeuler build failed times: $nr_openeuler_fail" + echo "openeuler_valid_url: $dockerfile_url" + exit 1 + # centos all success, openeuler all success + elif [ "$nr_openeuler_success" == "$dockerfile_count" ]; then + echo "openeuler_valid_url: $dockerfile_url" + exit 0 + fi +} + +docker_test() +{ + fit_repo + openeuler_ready + centos_docker + handle_build_result +} + +docker_test -- 2.23.0

2 2

[PATCH compass-ci] sparrow: change service default ports
by Cao Xueliang 16 Mar '21

16 Mar '21

Reference: 6354ba631e0fd5f795a1dc28b65e2002bd3de668 Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/extract-stats/start | 2 +- sparrow/3-code/dev-env | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/container/extract-stats/start b/container/extract-stats/start index 5033754..621099a 100755 --- a/container/extract-stats/start +++ b/container/extract-stats/start @@ -25,7 +25,7 @@ env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' DEFAULT_CCI = '/c/compass-ci' SRV_HTTP_RESULT_HOST = defaults['SRV_HTTP_RESULT_HOST'] || Open3.capture3('curl ip.sb')[0].chomp -SRV_HTTP_RESULT_PORT = defaults['SRV_HTTP_RESULT_PORT'] || '11300' +SRV_HTTP_RESULT_PORT = defaults['SRV_HTTP_RESULT_PORT'] || '20007' docker_rm "extract-stats" cmd = %w[ diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index b740a15..65061d8 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -15,10 +15,10 @@ server_name=$(hostname | cut -f1 -d.) : ${SRV_HTTP_OS_HOST:=$server_ip} : ${SRV_HTTP_GIT_HOST:=$server_ip} : ${SRV_HTTP_CCI_HOST:=$server_ip} -: ${SRV_HTTP_RESULT_PORT:=11300} -: ${SRV_HTTP_OS_PORT:=11304} -: ${SRV_HTTP_GIT_PORT:=11305} -: ${SRV_HTTP_CCI_PORT:=11307} +: ${SRV_HTTP_RESULT_PORT:=20007} +: ${SRV_HTTP_OS_PORT:=20009} +: ${SRV_HTTP_GIT_PORT:=20010} +: ${SRV_HTTP_CCI_PORT:=20011} : ${LOGGING_ES_HOST:=$server_ip} : ${LOGGING_ES_PORT:=9202} : ${INITRD_HTTP_HOST:=$server_ip} -- 2.23.0

1 0

[PATCH compass-ci] src/lib(scheduer): change function name "*pushlish*" to "*publish*" for mq
by Xu Xijian 16 Mar '21

16 Mar '21

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/lifecycle.cr | 4 ++-- src/lib/mq.cr | 4 ++-- src/lib/sched.cr | 4 ++-- src/scheduler/close_job.cr | 2 +- src/scheduler/find_job_boot.cr | 2 +- src/scheduler/update_job_parameter.cr | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lib/lifecycle.cr b/src/lib/lifecycle.cr index 1eb6264..4bd2ac0 100644 --- a/src/lib/lifecycle.cr +++ b/src/lib/lifecycle.cr @@ -79,7 +79,7 @@ class Lifecycle "job_state" => "abnormal", "testbox" => testbox } - @mq.pushlish_confirm("job_mq", msg.to_json) + @mq.publish_confirm("job_mq", msg.to_json) @match[testbox].delete(id) end end @@ -313,7 +313,7 @@ class Lifecycle mq_queue = get_machine_reboot_queue(testbox) machine.as_h.delete("history") machine.as_h["testbox"] = JSON::Any.new(testbox) - @mq.pushlish_confirm(mq_queue, machine.to_json, durable: true) + @mq.publish_confirm(mq_queue, machine.to_json, durable: true) machine["state"] = "rebooting_queue" machine["time"] = Time.local.to_s("%Y-%m-%dT%H:%M:%S+0800") diff --git a/src/lib/mq.cr b/src/lib/mq.cr index 980585d..1c7f4b7 100644 --- a/src/lib/mq.cr +++ b/src/lib/mq.cr @@ -37,7 +37,7 @@ class MQClient Singleton::Of(self).instance end - def pushlish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish_confirm(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish_confirm(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) @@ -46,7 +46,7 @@ class MQClient end end - def pushlish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) + def publish(queue, msg, passive = false, durable = false, exclusive = false, auto_delete = false) q = @ch.queue(queue, passive, durable, exclusive, auto_delete) if durable q.publish(msg, props: AMQ::Protocol::Properties.new(delivery_mode: 2)) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 9528f54..ac677de 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -219,9 +219,9 @@ class Sched return nil end - def mq_pushlish_confirm(queue, msg) + def mq_publish_confirm(queue, msg) 3.times do - @mq.pushlish_confirm(queue, msg) + @mq.publish_confirm(queue, msg) break rescue e res = @mq.reconnect diff --git a/src/scheduler/close_job.cr b/src/scheduler/close_job.cr index e014361..a57b233 100644 --- a/src/scheduler/close_job.cr +++ b/src/scheduler/close_job.cr @@ -45,7 +45,7 @@ class Sched "job_state" => "close", "time" => get_time } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end end diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 4a8ea8a..416f8f4 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -32,7 +32,7 @@ class Sched "time" => get_time, "job_state" => "boot" } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end # auto submit a job to collect the host information. diff --git a/src/scheduler/update_job_parameter.cr b/src/scheduler/update_job_parameter.cr index d81f28b..92b905e 100644 --- a/src/scheduler/update_job_parameter.cr +++ b/src/scheduler/update_job_parameter.cr @@ -44,6 +44,6 @@ class Sched "job_state" => (@env.get?("job_state") || "update").to_s, "time" => @env.get?("time").to_s } - spawn mq_pushlish_confirm(JOB_MQ, mq_msg.to_json) + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) end end -- 2.23.0

1 0

[PATCH v3 lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 16 Mar '21

16 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 47 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..d6acad1ab 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,40 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + elsif error_msg.is_a? String + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/[^\w]/, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line.gsub!(/^-|-$/, '') + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

1 0

[PATCH compass-ci] doc: update document
by Luan Shengde 16 Mar '21

16 Mar '21

update document for service new port: send-internet-mail: 49000 => 10001 Signed-off-by: Luan Shengde <shdluan(a)163.com> --- doc/send-mail.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/send-mail.md b/doc/send-mail.md index 5af8d17..ffa5651 100644 --- a/doc/send-mail.md +++ b/doc/send-mail.md @@ -8,7 +8,7 @@ The service is used to send mails with formatted mail data ## send mail port - send intranet mail: 11311 -- send internet mail: 49000 +- send internet mail: 10001 ## send mail host - host the service running on @@ -58,5 +58,5 @@ The service is used to send mails with formatted mail data test msg 1010 " - %x(curl -XPOST 'localhost:49000/send_mail_text' -d "#{data}") + %x(curl -XPOST 'localhost:10001/send_mail_text' -d "#{data}") ``` -- 2.23.0

1 0

[PATCH v3 lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 16 Mar '21

16 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 47 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..9e9e15fab 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,40 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + elsif error_msg.is_a? String + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/[^\w]/, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line.gsub!(/-$/, '') + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

1 1

[PATCH v2 lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 16 Mar '21

16 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 47 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..119dc1d2a 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,40 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + else + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/[^\w]/, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line.gsub!(/-$/, '') + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

2 2

[PATCH compass-ci] lib: update port
by Luan Shengde 16 Mar '21

16 Mar '21

service assigned new port: send-internet-mail: 49000 => 10001 assign-account: 29999 => 10000 Signed-off-by: Luan Shengde <shdluan(a)163.com> --- lib/assign_account_client.rb | 4 ++-- lib/constants.rb | 2 +- lib/mail-post.rb | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/assign_account_client.rb b/lib/assign_account_client.rb index 4831545..f551a9d 100644 --- a/lib/assign_account_client.rb +++ b/lib/assign_account_client.rb @@ -21,9 +21,9 @@ names = Set.new %w[ defaults = relevant_defaults(names) JUMPER_HOST = defaults['JUMPER_HOST'] -JUMPER_PORT = defaults['JUMPER_PORT'] || 29999 +JUMPER_PORT = defaults['JUMPER_PORT'] || 10000 SEND_MAIL_HOST = defaults['SEND_MAIL_HOST'] || 'localhost' -SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 49000 +SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] || 10001 LAB = ENV['lab'] # used for other codes calling to assign account for user diff --git a/lib/constants.rb b/lib/constants.rb index 6c30aaf..03b1e40 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -9,7 +9,7 @@ ES_HOST ||= config['ES_HOST'] || '172.17.0.1' ES_PORT ||= config['ES_PORT'] || 9200 SEND_MAIL_HOST ||= config['SEND_MAIL_HOST'] || '172.17.0.1' -SEND_MAIL_PORT ||= config['SEND_MAIL_PORT'] || 49000 +SEND_MAIL_PORT ||= config['SEND_MAIL_PORT'] || 10001 SRV_HTTP_RESULT_HOST ||= config['SRV_HTTP_RESULT_HOST'] || ENV['SRV_HTTP_RESULT_HOST'] || '172.17.0.1' SRV_HTTP_OS_HOST ||= config['SRV_HTTP_OS_HOST'] || ENV['SRV_HTTP_OS_HOST'] || '172.17.0.1' diff --git a/lib/mail-post.rb b/lib/mail-post.rb index 4eb5800..84fc3c1 100755 --- a/lib/mail-post.rb +++ b/lib/mail-post.rb @@ -76,7 +76,7 @@ def check_send_mail(mail_info) raise 'no/empty email_to address.' if mail_info['to'].nil? || mail_info['to'].empty? raise 'no/empty email content.' if mail_info['body'].nil? || mail_info['body'].empty? - return mail_info unless ENV['SEND_MAIL_PORT'].to_s == '49000' + return mail_info unless ENV['SEND_MAIL_PORT'].to_s == '10001' mail_info = check_email_mapping(mail_info.clone) mail_info = check_email_limit(mail_info.clone) @@ -101,7 +101,7 @@ def send_mail(mail_info) end def check_to_store_email(mail) - return if ENV['SEND_MAIL_PORT'].to_s != '49000' + return if ENV['SEND_MAIL_PORT'].to_s != '10001' return if ENV['HOST_SERVER'] != 'z9' store_email(mail) -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] container/web-backend-nginx: change service port
by Lu Weitao 16 Mar '21

16 Mar '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend-nginx/nginx.conf | 4 ++-- container/web-backend-nginx/start | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/container/web-backend-nginx/nginx.conf b/container/web-backend-nginx/nginx.conf index c9a4ec4..381888a 100644 --- a/container/web-backend-nginx/nginx.conf +++ b/container/web-backend-nginx/nginx.conf @@ -8,7 +8,7 @@ http { default_type application/octet-stream; include /etc/nginx/conf.d/*.conf; server { - listen 11320 ssl; + listen 20003 ssl; server_name api.compass-ci.openeuler.org; ssl_certificate /opt/cert/web-backend.crt; ssl_certificate_key /opt/cert/web-backend.key; @@ -20,7 +20,7 @@ http { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forward-For $proxy_add_x_forwarded_for; # web-backend server - proxy_pass http://172.17.0.1:32767; + proxy_pass http://172.17.0.1:10002; } location ~ ^/(result|pub) { diff --git a/container/web-backend-nginx/start b/container/web-backend-nginx/start index 4da4c8b..5360385 100755 --- a/container/web-backend-nginx/start +++ b/container/web-backend-nginx/start @@ -10,7 +10,7 @@ docker_rm web-backend-nginx cmd=( docker run --name web-backend-nginx - -p 11320:11320 + -p 20003:20003 -d -v /etc/ssl/certs:/opt/cert -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] container/web-backend: change service port
by Lu Weitao 16 Mar '21

16 Mar '21

Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend/start | 2 +- container/web-backend/web-backend | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/container/web-backend/start b/container/web-backend/start index 6731d27..e680255 100755 --- a/container/web-backend/start +++ b/container/web-backend/start @@ -18,7 +18,7 @@ cmd=( --name web-backend --restart=always -d - -p 32767:32767 + -p 10002:10002 -e ES_PORT=9200 -e SRV_HTTP_RESULT_HOST=$SRV_HTTP_RESULT_HOST diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 74eda85..33a3fae 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -9,7 +9,7 @@ CCI_SRC = ENV['CCI_SRC'] || '/c/compass-ci' require "#{CCI_SRC}/src/lib/web_backend.rb" set :bind, '0.0.0.0' -set :port, 32767 +set :port, 10002 set :show_exceptions, false set :environment, :production -- 2.23.0

1 0

[PATCH lkp-tests] lib/email.sh: change default send email port
by Wei Jihui 16 Mar '21

16 Mar '21

49000 --> 10001 Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/email.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/email.sh b/lib/email.sh index a0c99c24..63d5a51e 100755 --- a/lib/email.sh +++ b/lib/email.sh @@ -87,5 +87,5 @@ send_email() ${subject}_content - curl -XPOST "${MAIL_HOST:-$LKP_SERVER}:${MAIL_PORT:-49000}/send_mail_text" -d "$email_content" + curl -XPOST "${MAIL_HOST:-$LKP_SERVER}:${MAIL_PORT:-10001}/send_mail_text" -d "$email_content" } -- 2.23.0

1 0

[PATCH v3 lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 16 Mar '21

16 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 81 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 tests/ansible_test diff --git a/tests/ansible_test b/tests/ansible_test new file mode 100755 index 000000000..bf75b19b7 --- /dev/null +++ b/tests/ansible_test @@ -0,0 +1,81 @@ +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + ansible_host=localhost + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] || { + echo "Ansible failed to install." + exit 1 + } + } +} + +download_ansible_role() +{ + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git + if [ $? -eq 0 ]; then + echo "$benchmark is successfully cloned" + else + echo "Please check whether your ansible_repo is correct!" + exit 1 + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: $ansible_host + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook is running failed!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

2 3

[PATCH lkp-tests] stat/ansible_test: stats result for ansible_test
by Wang Chenglong 16 Mar '21

16 Mar '21

The result of ansible_test are counted and recorded in the ES. query it with the 'es-jobs' command. sum.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng: 1 raw.stats.ansible_test.error.Unsupported-pkg-mgr-This-role-is-depends-on-apt-yum-or-pkgng.message: ["{\"changed\": false, \"msg\": \"Unsupported pkg mgr. This role is depends on apt, yum, or pkgng.\"}"] Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 55 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..38d6e278c 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,48 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def output_error(error_msg) + return if error_msg.nil? + + if error_msg.is_a? Array + error_msg.each do |i| + error_id = common_error_id i + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" + end + else + error_id = common_error_id error_msg + puts "error.#{error_id}: 1" + puts "error.#{error_id}.message: #{@ansible_failed_info}" end end + +def common_error_id(line) + line.gsub!(/\n|\t/, '.') + line =`echo "#{line}"|awk -F'.' '{print $1}'` + line.gsub!(/:/, '') # error.:-'__mysql_packages'-is-undefined: 1 + line.gsub!(/\^/, '') + line.gsub!(/\.|,/, '') + line.gsub!(/<|>/, '') # error.Request-failed-<urlopen-error-timed-out>: 1 + line.gsub!(/'/, '') # Failed-to-download-metadata-for-repo-'dockerrepo: 1 + line.gsub!(/!/, '') # Distribution-openEuler-is-not-supported-by-this-role!: 1 + line.gsub!(/ /, '-') + line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one + line = line.chomp + line +end + +while (line = STDIN.gets) + next unless line =~ /(FAILED!|failed:).*=>(.*)/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.parse @ansible_failed_info + + output_error ansible_failed_json['msg'] + output_error ansible_failed_json['message'] + output_error ansible_failed_json['cmd'] + output_error ansible_failed_json['failures'] +end -- 2.23.0

2 2

[PATCH compass-ci] dev-env: update port
by Luan Shengde 16 Mar '21

16 Mar '21

send-internet-mail has been assigned new port: 49000 -> 10001 Signed-off-by: Luan Shengde <shdluan(a)163.com> --- sparrow/3-code/dev-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index b740a15..e448dcb 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -10,7 +10,7 @@ server_name=$(hostname | cut -f1 -d.) : ${ES_PORT:=9200} : ${OS_HTTP_HOST:=$server_ip} : ${SEND_MAIL_HOST:=$server_ip} -: ${SEND_MAIL_PORT:=49000} +: ${SEND_MAIL_PORT:=10001} : ${SRV_HTTP_RESULT_HOST:=$server_ip} : ${SRV_HTTP_OS_HOST:=$server_ip} : ${SRV_HTTP_GIT_HOST:=$server_ip} -- 2.23.0

1 0

[PATCH v2 compass-ci] src/lib/job.cr: change rpm_repo to upstream_repo
by Li Ping 16 Mar '21

16 Mar '21

src/lib/job.cr: open the path for uploading rpm packages. Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/job.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 9e18a4c..40aaa62 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -278,7 +278,7 @@ class Job mount_type = "nfs" if mount_type == "cifs" common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" - common_dir = "#{tmp_os}-#{tmp_os_version}/#{tmp_os_arch}" if @hash.has_key?("rpmbuild-pkg") + common_dir = "#{tmp_os}-#{tmp_os_version}" if @hash.has_key?("rpmbuild-pkg") return common_dir end @@ -293,7 +293,7 @@ class Job elsif @hash["cci-depends"]? package_dir = ",/initrd/deps/#{common_dir}/#{@hash["cci-depends"]["benchmark"]}" elsif @hash["rpmbuild-pkg"]? - package_dir = ",/initrd/rpmbuild-pkg/#{common_dir}/#{@hash["rpm_repo"].to_s.split("/")[-1]}" + package_dir = ",/initrd/rpmbuild-pkg/#{common_dir}" elsif @hash["build-pkg"]? if @hash["pkgbuild_repo"].to_s =~ /(packages|community)\/\// package_name = @hash["pkgbuild_repo"].to_s.split("/")[-2] -- 2.23.0

1 1

2024

2023

2022

2021

2020

Compass-ci