- Compass-ci - mailweb.openeuler.org

[PATCH v2 lkp-tests] tests: add httpress test script
by Wei Jihui 05 Jan '21

05 Jan '21

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- tests/httpress | 138 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100755 tests/httpress diff --git a/tests/httpress b/tests/httpress new file mode 100755 index 00000000..109ba7f9 --- /dev/null +++ b/tests/httpress @@ -0,0 +1,138 @@ +#!/bin/sh +# - nic +# - nic_ip +# - request_ip +# - request_port +# - request_index +# - core_nums +# - protocol +# - alive_type + +. $LKP_SRC/lib/env.sh + +: "${request_index:=index.html}" +: "${core_nums:=4}" +: "${protocol:=http}" +: "${alive_type:=short}" + + +core_range=0-15 + +http_long_requests() +{ + if [ "$core_nums" == 4 ]; then + requests_num=3000000 + threads_num=100 + concurrent_num=100 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 8 ]; then + requests_num=5000000 + threads_num=300 + concurrent_num=300 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=50000000 + threads_num=1200 + concurrent_num=1200 + ethtool -L $nic combined 16 && /lkp/lkp/src/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=50000000 + threads_num=400 + concurrent_num=400 + ethtool -L $nic combined 16 && /lkp/lkp/src/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + fi +} + + +http_short_requests() +{ + if [ "$core_nums" == 4 ]; then + requests_num=3000000 + threads_num=300 + concurrent_num=300 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 8 ]; then + requests_num=5000000 + threads_num=400 + concurrent_num=400 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=10000000 + threads_num=300 + concurrent_num=300 + ethtool -L $nic combined 16 && $LKP_SRC/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=10000000 + threads_num=600 + concurrent_num=600 + ethtool -L $nic combined 16 && $LKP_SRC/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + fi +} + +https_long_requests() +{ + if [ "$core_nums" == 4 ] || [ "$core_nums" == 8 ]; then + requests_num=5000000 + threads_num=100 + concurrent_num=100 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=5000000 + threads_num=600 + concurrent_num=600 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=10000000 + threads_num=600 + concurrent_num=600 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + fi +} + +https_short_requests() +{ + if [ "$core_nums" == 4 ] || [ "$core_nums" == 8 ]; then + requests_num=500000 + threads_num=100 + concurrent_num=100 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=500000 + threads_num=600 + concurrent_num=600 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=1000000 + threads_num=1200 + concurrent_num=1200 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + fi +} + +main() +{ + local request_url="$protocol://$request_ip:$request_port/$request_index" + + ip addr add $nic_ip/24 dev "$nic" + ip link set "$nic" up + + if [ "$protocol" == 'http' ]; then + if [ "$alive_type" == 'long' ]; then + http_long_requests + else + http_short_requests + fi + else + if [ "$alive_type" == 'long' ]; then + https_long_requests + else + https_short_requests + fi + fi +} + +main -- 2.23.0

2 2

[PATCH compass-ci] service/monitoring: print json logs
by Wu Zhende 05 Jan '21

05 Jan '21

Replace "puts" with JSONLogger. It's easier to query and locate faults. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/monitoring.cr | 12 +++++++++++- src/monitoring/amqp.cr | 6 ++++-- src/monitoring/filter.cr | 4 +++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/monitoring.cr b/src/monitoring.cr index b1b534b..0f88be4 100644 --- a/src/monitoring.cr +++ b/src/monitoring.cr @@ -1,6 +1,16 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +require "./lib/json_logger" require "monitoring/monitoring" require "monitoring/constants" module Monitoring - Kemal.run(MONITOR_PORT) + log = JSONLogger.new + + begin + Kemal.run(MONITOR_PORT) + rescue e + log.error(e) + end end diff --git a/src/monitoring/amqp.cr b/src/monitoring/amqp.cr index b0341ce..27ad3d2 100644 --- a/src/monitoring/amqp.cr +++ b/src/monitoring/amqp.cr @@ -3,10 +3,12 @@ require "amqp-client" require "./monitoring" require "./filter" require "./constants" +require "../lib/json_logger" class MessageQueueClient def initialize(host = MQ_HOST, port = MQ_PORT) @client = AMQP::Client.new("amqp://#{host}:#{port}") + @log = JSONLogger.new end private def start @@ -24,7 +26,7 @@ class MessageQueueClient begin filter.filter_msg(msg.body_io) rescue e - puts "filter message error: #{e}" + @log.warn("filter message failed: #{e}") end end end @@ -37,7 +39,7 @@ class MessageQueueClient filter_msg(conn, filter, exchange_name, queue_name) end rescue e - puts "monitoring_message_queue error: #{e}" + @log.warn("monitoring_message_queue failed: #{e}") sleep 5 end end diff --git a/src/monitoring/filter.cr b/src/monitoring/filter.cr index 79af3ab..2bd1d1a 100644 --- a/src/monitoring/filter.cr +++ b/src/monitoring/filter.cr @@ -2,6 +2,7 @@ require "set" require "json" require "./parse_serial_logs" +require "../lib/json_logger" class Filter def initialize @@ -9,6 +10,7 @@ class Filter # like {query => [socket1, socket2]} @hash = Hash(JSON::Any, Array(HTTP::WebSocket)).new @sp = SerialParser.new + @log = JSONLogger.new end def add_filter_rule(query : JSON::Any, socket : HTTP::WebSocket) @@ -42,7 +44,7 @@ class Filter @hash[query].each do |socket| socket.send msg.to_json rescue e - puts "send msg failed: #{e}" + @log.warn("send msg failed: #{e}") remove_filter_rule(query, socket) end end -- 2.23.0

1 0

[PATCH v2 compass-ci 1/7] send-internet-mail/Dockerfile: add install gem resources
by Luan Shengde 05 Jan '21

05 Jan '21

[why] the service need to install redis to store email counts to redis queue the service need to install rufus-scheduler to reset the queue as a timing task Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/send-internet-mail/Dockerfile b/container/send-internet-mail/Dockerfile index d151bda..03e3327 100644 --- a/container/send-internet-mail/Dockerfile +++ b/container/send-internet-mail/Dockerfile @@ -6,4 +6,4 @@ MAINTAINER shdluan(a)163.com ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && \ - apt-get install -y ruby-mail ruby-json ruby-sinatra + apt-get install -y ruby-mail ruby-json ruby-sinatra ruby-redis ruby-rufus-scheduler -- 2.23.0

2 2

[PATCH v2 compass-ci 2/7] send-internet-mail: optimize
by Luan Shengde 05 Jan '21

05 Jan '21

add init redis host/port add invoke code files Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/send-internet-mail.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/container/send-internet-mail/send-internet-mail.rb b/container/send-internet-mail/send-internet-mail.rb index 8fdaa0d..90571f8 100755 --- a/container/send-internet-mail/send-internet-mail.rb +++ b/container/send-internet-mail/send-internet-mail.rb @@ -6,7 +6,13 @@ require 'mail' require 'sinatra' -require "#{ENV['CCI_SRC']}/lib/mail-post" +REDIS_HOST = %x(/sbin/ip route |awk '/default/ {print $3}').chomp +REDIS_PORT = ENV['REDIS_PORT'] + +require_relative 'email_init' +require_relative 'email_limit_queue' +require_relative 'email_mapping' +require_relative '../../lib/mail-post' smtp = { address: 'smtp.qq.com', -- 2.23.0

2 2

[PATCH v3 compass-ci] providers/qemu: adapt for x86_64 and riscv64
by Liu Yinsi 05 Jan '21

05 Jan '21

before: support kvm for aarch64 after: support kvm for aarch64, x86_64, riscv64 [how] put public options like $kernel, $initrd, $smp together, list different options like $machine, $bios, $nic according to different qemu command, then merge all options. test: /srv/result/host-info/2021-01-04/vm-2p8g/debian-sid-aarch64/z9.280302 /srv/result/host-info/2021-01-04/vm-2p8g/debian-sid-riscv64/z9.280305 /srv/result/host-info/2021-01-04/vm-2p8g/debian-sid-x86_64/z9.280303 Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 106 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 93 insertions(+), 13 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index c926d3d..8749568 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -18,12 +18,6 @@ check_logfile() } } -qemu_command() -{ - qemu=qemu-system-aarch64 - command -v $qemu >/dev/null || qemu=qemu-kvm -} - write_logfile() { ipxe_script=ipxe_script @@ -72,6 +66,50 @@ get_initrd() cat $initrds > $initrd } +get_bios() +{ + bios=/usr/share/qemu-efi-aarch64/QEMU_EFI.fd + [ -f "$bios" ] || bios=/usr/share/ovmf/OVMF.fd +} + +get_helper() +{ + helper=/usr/libexec/qemu-bridge-helper + [ -f "$helper" ] || helper=/usr/lib/qemu/qemu-bridge-helper +} + +get_nic() +{ + nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" +} + +get_device() +{ + device="virtio-net-device,netdev=net0,mac=${mac}" +} + +get_netdev() +{ + netdev="bridge,br=br0,id=net0,helper=${helper}" +} + +get_qemu() +{ + qemu=( + qemu-system-aarch64 + qemu-kvm + qemu-system-x86_64 + ) + + for qemu in "${qemu[@]}" + do + [ -n "$(command -v ${qemu})" ] && break + done + + # debian has both qemu-system-x86_64 and qemu-system-riscv64 command + [[ $kernel =~ 'riscv64' ]] && qemu=qemu-system-riscv64 +} + print_message() { echo $SCHED_PORT @@ -83,36 +121,78 @@ print_message() sleep 5 } -run_qemu() +public_option() { kvm=( $qemu - -machine virt-4.0,accel=kvm,gic-version=3 -kernel $kernel -initrd $initrd -smp $nr_cpu -m $memory - -cpu Kunpeng-920 - -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd - -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} -k en-us -no-reboot -nographic -serial file:${log_file} -monitor null ) +} - "${kvm[@]}" --append "${append}" +run_qemu() +{ + case "$qemu" in + qemu-system-aarch64) + arch_option=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) + ;; + qemu-kvm) + [ "$(arch)" == "aarch64" ] && arch_option=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) + [ "$(arch)" == "x86_64" ] && arch_option=( + -bios $bios + -nic $nic + ) + ;; + qemu-system-x86_64) + arch_option=( + -bios $bios + -nic $nic + ) + ;; + qemu-system-riscv64) + arch_option=( + -machine virt + -device virtio-net-device,netdev=net0,mac=${mac} + -netdev bridge,br=br0,id=net0,helper=${helper} + ) + ;; + esac + + "${kvm[@]}" "${arch_option[@]}" --append "${append}" } check_logfile write_logfile -qemu_command parse_ipxe_script check_option_value + get_initrd +get_bios +get_helper +get_nic +get_device +get_netdev +get_qemu print_message +public_option run_qemu -- 2.23.0

2 2

[PATCH compass-ci] provicers/docker: copy rootfs/addon/usr/* to container
by Xiao Shenwei 05 Jan '21

05 Jan '21

[why] under LKP_SRC/addon/usr we have some customized tools such as: /usr/bin/my_curl, these resource should be used by container Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- providers/docker/bin/entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/providers/docker/bin/entrypoint.sh b/providers/docker/bin/entrypoint.sh index feb5edf..4388e19 100755 --- a/providers/docker/bin/entrypoint.sh +++ b/providers/docker/bin/entrypoint.sh @@ -4,4 +4,5 @@ mkdir -p /root/.ssh cp /lkp/lkp/src/rootfs/addon/root/.ssh/id_rsa /root/.ssh/ +cp -r /lkp/lkp/src/rootfs/addon/usr/* /usr sh /lkp/lkp/src/rootfs/addon/etc/init.d/lkp-bootstrap -- 2.23.0

1 0

[PATCH compass-ci] providers/docker: read only mount of host resources
by Xiao Shenwei 05 Jan '21

05 Jan '21

Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- providers/docker/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/docker/run.sh b/providers/docker/run.sh index 127cf5b..62a1f9f 100755 --- a/providers/docker/run.sh +++ b/providers/docker/run.sh @@ -27,7 +27,7 @@ cmd=( -e CCI_SRC=/c/compass-ci -v ${load_path}/lkp:/lkp -v ${DIR}/bin:/root/bin:ro - -v $CCI_SRC:/c/compass-ci + -v $CCI_SRC:/c/compass-ci:ro -v /srv/git:/srv/git:ro -v /srv/result:/srv/result:ro -v ${busybox_path}:/usr/local/bin/busybox -- 2.23.0

1 0

[PATCH v2 compass-ci] 3-code/git: modify the git download path
by Hu Xuejiao 05 Jan '21

05 Jan '21

[why] if git path is not specified, lkp-tests will be downloaded in /c/compass-ci Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sparrow/3-code/git | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sparrow/3-code/git b/sparrow/3-code/git index 48741ec..85db0de 100755 --- a/sparrow/3-code/git +++ b/sparrow/3-code/git @@ -9,10 +9,10 @@ umask 002 cd /c || exit -# git clone https://gitee.com/openeuler/compass-ci.git +# git clone compass-ci,lkp-tests,upstream-repos # modify and manual run for now: [ -d "compass-ci/.git" ] || { - git clone https://gitee.com/wu_fengguang/compass-ci.git + git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci cd /c/compass-ci || exit [ -n "$compass_commit_id" ] && git reset --hard "$compass_commit_id" @@ -21,14 +21,14 @@ cd /c || exit } [ -d "lkp-tests/.git" ] || { - git clone https://gitee.com/wu_fengguang/lkp-tests.git + git clone https://gitee.com/wu_fengguang/lkp-tests.git /c/lkp-tests cd /c/lkp-tests || exit [ -n "$lkp_commit_id" ] && git reset --hard "$lkp_commit_id" } [ -d "upstream-repos/.git" ] || { - git clone https://gitee.com/wu_fengguang/upstream-repos.git + git clone https://gitee.com/wu_fengguang/upstream-repos.git /c/upstream-repos cd /c/upstream-repos || exit [ -n "$upstream_commit_id" ] && git reset --hard "$upstream_commit_id" -- 2.23.0

1 0

[PATCH v5 compass-ci 2/2] lib/es_query.rb: parse es_result to a Hash result
by Lu Kaiyi 05 Jan '21

05 Jan '21

[why] make es result more easy access. [example] input: es_result = [{"key"=>"build-pkg", "doc_count"=>354526, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>5708}, {"key"=>"finished", "doc_count"=>1033}, {"key"=>"incomplete", "doc_count"=>204}, {"key"=>"submit", "doc_count"=>136}, ... output: result_hash = {"build-pkg"=> {"failed"=>5708, "finished"=>1033, "incomplete"=>204, "submit"=>136, "OOM"=>11, "post_run"=>2}, "cci-depends"=> {"finished"=>1785, "failed"=>675, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 91aa6ab..207aaae 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -128,7 +128,7 @@ class ESQuery result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] return nil if result.empty? - result + parse_fields(result) end end @@ -216,3 +216,47 @@ def build_aggs_from_fields(fields) aggs_hash['aggs']["all_#{field}"].merge!(sub_aggs) if sub_aggs aggs_hash end + +# input: +# es_result = +# [{"key"=>"build-pkg", +# "doc_count"=>354526, +# "all_job_state"=> +# {"doc_count_error_upper_bound"=>0, +# "sum_other_doc_count"=>0, +# "buckets"=> +# [{"key"=>"failed", "doc_count"=>5708}, +# {"key"=>"finished", "doc_count"=>1033}, +# {"key"=>"incomplete", "doc_count"=>204}, +# {"key"=>"submit", "doc_count"=>136}, +# ... +# +# output: +# result_hash = +# {"build-pkg"=> +# {"failed"=>5708, +# "finished"=>1033, +# "incomplete"=>204, +# "submit"=>136, +# "OOM"=>11, +# "post_run"=>2}, +# "cci-depends"=> +# {"finished"=>1785, +# "failed"=>675, +# ... +def parse_fields(es_result) + result_hash ||= {} + es_result.each do |result| + key = result['key'] + sub_field = result.keys.detect { |field| field.start_with?('all_') } + + if sub_field + all_field = result[sub_field]['buckets'] + result_hash[key] = parse_fields(all_field) + else + result_hash[key] = result['doc_count'] + end + end + + result_hash +end -- 2.23.0

1 0

[PATCH v2 compass-ci 7/7] send-internet-mail/start: add key REDIS_PORT
by Luan Shengde 05 Jan '21

05 Jan '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/send-internet-mail/start b/container/send-internet-mail/start index 02c6ab6..136435d 100755 --- a/container/send-internet-mail/start +++ b/container/send-internet-mail/start @@ -13,6 +13,7 @@ names = Set.new %w[ ROBOT_EMAIL_PASSWORD SEND_MAIL_PORT SENT_MAILDIR + REDIS_PORT ] defaults = relevant_defaults(names) -- 2.23.0

1 0