- Compass-ci - mailweb.openeuler.org

[PATCH v5 compass-ci 1/2] lib/es_query.rb: get multi-field aggregation count result
by Lu Kaiyi 05 Jan '21

05 Jan '21

[why] get multi-field aggregation count for overview of report page. [input]: fields, query_items (optional for query_items, default no scope limitation) [example] input: fields = ['suite'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186214}, {"key"=>"cci-depends", "doc_count"=>1230}, {"key"=>"iperf", "doc_count"=>1103}, {"key"=>"spinlock", "doc_count"=>1000}, {"key"=>"cci-makepkg", "doc_count"=>949}, {"key"=>"iperf-walk-os-test", "doc_count"=>474}, {"key"=>"bisect", "doc_count"=>381}, ... input: fields = ['suite', 'job_state'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186227, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3830}, {"key"=>"finished", "doc_count"=>803}, {"key"=>"incomplete", "doc_count"=>196}, {"key"=>"submit", "doc_count"=>136}, ... input: fields = ['os', 'os_version', 'job_state'] query_items = {} output: [{"key"=>"openeuler", "doc_count"=>193287, "all_os_version"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"20.03", "doc_count"=>191789, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3909}, {"key"=>"finished", "doc_count"=>2401}, {"key"=>"incomplete", "doc_count"=>300}, {"key"=>"submit", "doc_count"=>136}, {"key"=>"OOM", "doc_count"=>23}, {"key"=>"post_run", "doc_count"=>1}]}}, {"key"=>"20.03-SP1", "doc_count"=>904, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>641}, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 70 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/lib/es_query.rb b/lib/es_query.rb index 88a420f..91aa6ab 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -87,6 +87,49 @@ class ESQuery result end + + # input: + # fields, query_items + # + # fields => fields: Array(keyword) + # eg: + # ['suite', 'job_state'] + # + # query_items => query_items: Hash(keyword, value) + # eg: + # { + # 'os' => 'openeuler', + # ... + # } + # (optional for query_items, default no scope limitation) + # output: + # [{"key"=>"build-pkg", + # "doc_count"=>186175, + # "all_job_state"=> + # {"doc_count_error_upper_bound"=>0, + # "sum_other_doc_count"=>0, + # "buckets"=> + # [{"key"=>"failed", "doc_count"=>3830}, + # {"key"=>"finished", "doc_count"=>803}, + # {"key"=>"incomplete", "doc_count"=>196}, + # ... + def query_fields(fields, query_items = {}) + field1 = fields.first + aggs_hash = build_aggs_from_fields(fields) + query = { + query: { + bool: { + must: build_multi_field_subquery_body(query_items) + } + }, + aggs: aggs_hash['aggs'], + size: 0 + } + result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] + return nil if result.empty? + + result + end end # Range Query Example: @@ -146,3 +189,30 @@ def assign_desc_body(keyword) }] } end + +# input: +# fields = ['os', 'os_version', 'job_state'] +# output: +# aggs_hash = { +# "aggs"=> +# {"all_os"=> +# {"terms"=>{:field=>"os", :size=>1000}, +# "aggs"=> +# {"all_os_version"=> +# {"terms"=>{:field=>"os_version", :size=>1000}, +# "aggs"=> +# {"all_job_state"=>{"terms"=>{:field=>"job_state", :size=>1000}}}}}}} +# } +def build_aggs_from_fields(fields) + aggs_hash = {} + return if fields.empty? + + field = fields.shift + aggs_hash['aggs'] ||= {} + aggs_hash['aggs']["all_#{field}"] = { + 'terms' => { field: field, size: 1000 } + } + sub_aggs = build_aggs_from_fields(fields) + aggs_hash['aggs']["all_#{field}"].merge!(sub_aggs) if sub_aggs + aggs_hash +end -- 2.23.0

2 1

[PATCH lkp-tests] lib/bootstrap.sh: fix the problem caused by the lack of double quotes
by Yu Chuan 05 Jan '21

05 Jan '21

[Why] In dash, we must add double quotes when calling a method using `$()`. + local ycb= asf + echo ycb: : ycb: : The lack of double quotes in our current code leads to a wrong judgement. [Error Msg] + local net_devices= enp0s1 + [ -z ] + warn_no_eth0 Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 94b94c28e9a8..30e4af828bcd 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -73,7 +73,7 @@ get_net_devices() net_devices_link() { local operation=$1 - local net_devices=$(get_net_devices) + local net_devices="$(get_net_devices)" local ndev for ndev in $net_devices do @@ -163,7 +163,7 @@ setup_network() $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig network_ok && return - local net_devices=$(get_net_devices) + local net_devices="$(get_net_devices)" if [ -z "$net_devices" ]; then warn_no_eth0 -- 2.23.0

1 0

[PATCH compass-ci 1/2] sbin/run_job: registering user when submit-job
by Hu Xuejiao 05 Jan '21

05 Jan '21

Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sbin/run_job | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbin/run_job b/sbin/run_job index c84df0d..690e5d6 100755 --- a/sbin/run_job +++ b/sbin/run_job @@ -1,5 +1,8 @@ #!/bin/bash +# register user information +ruby build-my-info.rb $my_email $USER $lab + cci_makepkg_file="$LKP_SRC/jobs/cci-makepkg.yaml" grep "atomic" $cci_makepkg_file || sed -i 's|benchmark:|& atomic|g' $cci_makepkg_file -- 2.23.0

2 1

[PATCH compass-ci 2/2] sbin/build-my-info.rb: modify the method of inputting variables
by Hu Xuejiao 05 Jan '21

05 Jan '21

Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sbin/build-my-info.rb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sbin/build-my-info.rb b/sbin/build-my-info.rb index 40193af..197425c 100755 --- a/sbin/build-my-info.rb +++ b/sbin/build-my-info.rb @@ -3,16 +3,12 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true -require 'io/console' require_relative '../lib/es_client' require_relative '../lib/build_my_info_client' -print 'email: ' -my_email = $stdin.echo = gets.chomp -print 'name: ' -my_name = $stdin.echo = gets.chomp -print 'lab: ' -lab = $stdin.echo = gets.chomp +my_email = ARGV[0] +my_name = ARGV[1] +lab = ARGV[2] build_my_info = BuildMyInfo.new(my_email, my_name, lab) build_my_info.config_my_info -- 2.23.0

2 1

[PATCH v4 compass-ci] docker-rootfs/common: add comment for set password file.
by Wang Chenglong 05 Jan '21

05 Jan '21

[Why] add password file is easy to ignore. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container/docker-rootfs/common b/container/docker-rootfs/common index fe37da9..e908079 100755 --- a/container/docker-rootfs/common +++ b/container/docker-rootfs/common @@ -33,7 +33,8 @@ usage() dst_rootfs_new_abspath: destination absolute path to create for rootfs Example: - ./run /tmp/openEuler-docker/openEuler-docker.${aarch}.tar.xz /tmp/openeuler-rootfs/ + ./run /tmp/openEuler-docker/openEuler-docker.${aarch}.tar.xz /tmp/openeuler-rootfs/ + # Please ensure $HOME/.config/compass-ci/rootfs-passwd exists. " } -- 2.23.0

2 1

[PATCH v2 lkp-tests] tests: add httpress test script
by Wei Jihui 05 Jan '21

05 Jan '21

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- tests/httpress | 138 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100755 tests/httpress diff --git a/tests/httpress b/tests/httpress new file mode 100755 index 00000000..109ba7f9 --- /dev/null +++ b/tests/httpress @@ -0,0 +1,138 @@ +#!/bin/sh +# - nic +# - nic_ip +# - request_ip +# - request_port +# - request_index +# - core_nums +# - protocol +# - alive_type + +. $LKP_SRC/lib/env.sh + +: "${request_index:=index.html}" +: "${core_nums:=4}" +: "${protocol:=http}" +: "${alive_type:=short}" + + +core_range=0-15 + +http_long_requests() +{ + if [ "$core_nums" == 4 ]; then + requests_num=3000000 + threads_num=100 + concurrent_num=100 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 8 ]; then + requests_num=5000000 + threads_num=300 + concurrent_num=300 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=50000000 + threads_num=1200 + concurrent_num=1200 + ethtool -L $nic combined 16 && /lkp/lkp/src/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=50000000 + threads_num=400 + concurrent_num=400 + ethtool -L $nic combined 16 && /lkp/lkp/src/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + fi +} + + +http_short_requests() +{ + if [ "$core_nums" == 4 ]; then + requests_num=3000000 + threads_num=300 + concurrent_num=300 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 8 ]; then + requests_num=5000000 + threads_num=400 + concurrent_num=400 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=10000000 + threads_num=300 + concurrent_num=300 + ethtool -L $nic combined 16 && $LKP_SRC/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=10000000 + threads_num=600 + concurrent_num=600 + ethtool -L $nic combined 16 && $LKP_SRC/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + fi +} + +https_long_requests() +{ + if [ "$core_nums" == 4 ] || [ "$core_nums" == 8 ]; then + requests_num=5000000 + threads_num=100 + concurrent_num=100 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=5000000 + threads_num=600 + concurrent_num=600 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=10000000 + threads_num=600 + concurrent_num=600 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + fi +} + +https_short_requests() +{ + if [ "$core_nums" == 4 ] || [ "$core_nums" == 8 ]; then + requests_num=500000 + threads_num=100 + concurrent_num=100 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=500000 + threads_num=600 + concurrent_num=600 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=1000000 + threads_num=1200 + concurrent_num=1200 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + fi +} + +main() +{ + local request_url="$protocol://$request_ip:$request_port/$request_index" + + ip addr add $nic_ip/24 dev "$nic" + ip link set "$nic" up + + if [ "$protocol" == 'http' ]; then + if [ "$alive_type" == 'long' ]; then + http_long_requests + else + http_short_requests + fi + else + if [ "$alive_type" == 'long' ]; then + https_long_requests + else + https_short_requests + fi + fi +} + +main -- 2.23.0

2 2

[PATCH compass-ci] service/monitoring: print json logs
by Wu Zhende 05 Jan '21

05 Jan '21

Replace "puts" with JSONLogger. It's easier to query and locate faults. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/monitoring.cr | 12 +++++++++++- src/monitoring/amqp.cr | 6 ++++-- src/monitoring/filter.cr | 4 +++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/monitoring.cr b/src/monitoring.cr index b1b534b..0f88be4 100644 --- a/src/monitoring.cr +++ b/src/monitoring.cr @@ -1,6 +1,16 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +require "./lib/json_logger" require "monitoring/monitoring" require "monitoring/constants" module Monitoring - Kemal.run(MONITOR_PORT) + log = JSONLogger.new + + begin + Kemal.run(MONITOR_PORT) + rescue e + log.error(e) + end end diff --git a/src/monitoring/amqp.cr b/src/monitoring/amqp.cr index b0341ce..27ad3d2 100644 --- a/src/monitoring/amqp.cr +++ b/src/monitoring/amqp.cr @@ -3,10 +3,12 @@ require "amqp-client" require "./monitoring" require "./filter" require "./constants" +require "../lib/json_logger" class MessageQueueClient def initialize(host = MQ_HOST, port = MQ_PORT) @client = AMQP::Client.new("amqp://#{host}:#{port}") + @log = JSONLogger.new end private def start @@ -24,7 +26,7 @@ class MessageQueueClient begin filter.filter_msg(msg.body_io) rescue e - puts "filter message error: #{e}" + @log.warn("filter message failed: #{e}") end end end @@ -37,7 +39,7 @@ class MessageQueueClient filter_msg(conn, filter, exchange_name, queue_name) end rescue e - puts "monitoring_message_queue error: #{e}" + @log.warn("monitoring_message_queue failed: #{e}") sleep 5 end end diff --git a/src/monitoring/filter.cr b/src/monitoring/filter.cr index 79af3ab..2bd1d1a 100644 --- a/src/monitoring/filter.cr +++ b/src/monitoring/filter.cr @@ -2,6 +2,7 @@ require "set" require "json" require "./parse_serial_logs" +require "../lib/json_logger" class Filter def initialize @@ -9,6 +10,7 @@ class Filter # like {query => [socket1, socket2]} @hash = Hash(JSON::Any, Array(HTTP::WebSocket)).new @sp = SerialParser.new + @log = JSONLogger.new end def add_filter_rule(query : JSON::Any, socket : HTTP::WebSocket) @@ -42,7 +44,7 @@ class Filter @hash[query].each do |socket| socket.send msg.to_json rescue e - puts "send msg failed: #{e}" + @log.warn("send msg failed: #{e}") remove_filter_rule(query, socket) end end -- 2.23.0

1 0

[PATCH v2 compass-ci 1/7] send-internet-mail/Dockerfile: add install gem resources
by Luan Shengde 05 Jan '21

05 Jan '21

[why] the service need to install redis to store email counts to redis queue the service need to install rufus-scheduler to reset the queue as a timing task Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/send-internet-mail/Dockerfile b/container/send-internet-mail/Dockerfile index d151bda..03e3327 100644 --- a/container/send-internet-mail/Dockerfile +++ b/container/send-internet-mail/Dockerfile @@ -6,4 +6,4 @@ MAINTAINER shdluan(a)163.com ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && \ - apt-get install -y ruby-mail ruby-json ruby-sinatra + apt-get install -y ruby-mail ruby-json ruby-sinatra ruby-redis ruby-rufus-scheduler -- 2.23.0

2 2

[PATCH v2 compass-ci 2/7] send-internet-mail: optimize
by Luan Shengde 05 Jan '21

05 Jan '21

add init redis host/port add invoke code files Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/send-internet-mail.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/container/send-internet-mail/send-internet-mail.rb b/container/send-internet-mail/send-internet-mail.rb index 8fdaa0d..90571f8 100755 --- a/container/send-internet-mail/send-internet-mail.rb +++ b/container/send-internet-mail/send-internet-mail.rb @@ -6,7 +6,13 @@ require 'mail' require 'sinatra' -require "#{ENV['CCI_SRC']}/lib/mail-post" +REDIS_HOST = %x(/sbin/ip route |awk '/default/ {print $3}').chomp +REDIS_PORT = ENV['REDIS_PORT'] + +require_relative 'email_init' +require_relative 'email_limit_queue' +require_relative 'email_mapping' +require_relative '../../lib/mail-post' smtp = { address: 'smtp.qq.com', -- 2.23.0

2 2

[PATCH v3 compass-ci] providers/qemu: adapt for x86_64 and riscv64
by Liu Yinsi 05 Jan '21

05 Jan '21

before: support kvm for aarch64 after: support kvm for aarch64, x86_64, riscv64 [how] put public options like $kernel, $initrd, $smp together, list different options like $machine, $bios, $nic according to different qemu command, then merge all options. test: /srv/result/host-info/2021-01-04/vm-2p8g/debian-sid-aarch64/z9.280302 /srv/result/host-info/2021-01-04/vm-2p8g/debian-sid-riscv64/z9.280305 /srv/result/host-info/2021-01-04/vm-2p8g/debian-sid-x86_64/z9.280303 Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 106 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 93 insertions(+), 13 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index c926d3d..8749568 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -18,12 +18,6 @@ check_logfile() } } -qemu_command() -{ - qemu=qemu-system-aarch64 - command -v $qemu >/dev/null || qemu=qemu-kvm -} - write_logfile() { ipxe_script=ipxe_script @@ -72,6 +66,50 @@ get_initrd() cat $initrds > $initrd } +get_bios() +{ + bios=/usr/share/qemu-efi-aarch64/QEMU_EFI.fd + [ -f "$bios" ] || bios=/usr/share/ovmf/OVMF.fd +} + +get_helper() +{ + helper=/usr/libexec/qemu-bridge-helper + [ -f "$helper" ] || helper=/usr/lib/qemu/qemu-bridge-helper +} + +get_nic() +{ + nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" +} + +get_device() +{ + device="virtio-net-device,netdev=net0,mac=${mac}" +} + +get_netdev() +{ + netdev="bridge,br=br0,id=net0,helper=${helper}" +} + +get_qemu() +{ + qemu=( + qemu-system-aarch64 + qemu-kvm + qemu-system-x86_64 + ) + + for qemu in "${qemu[@]}" + do + [ -n "$(command -v ${qemu})" ] && break + done + + # debian has both qemu-system-x86_64 and qemu-system-riscv64 command + [[ $kernel =~ 'riscv64' ]] && qemu=qemu-system-riscv64 +} + print_message() { echo $SCHED_PORT @@ -83,36 +121,78 @@ print_message() sleep 5 } -run_qemu() +public_option() { kvm=( $qemu - -machine virt-4.0,accel=kvm,gic-version=3 -kernel $kernel -initrd $initrd -smp $nr_cpu -m $memory - -cpu Kunpeng-920 - -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd - -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} -k en-us -no-reboot -nographic -serial file:${log_file} -monitor null ) +} - "${kvm[@]}" --append "${append}" +run_qemu() +{ + case "$qemu" in + qemu-system-aarch64) + arch_option=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) + ;; + qemu-kvm) + [ "$(arch)" == "aarch64" ] && arch_option=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) + [ "$(arch)" == "x86_64" ] && arch_option=( + -bios $bios + -nic $nic + ) + ;; + qemu-system-x86_64) + arch_option=( + -bios $bios + -nic $nic + ) + ;; + qemu-system-riscv64) + arch_option=( + -machine virt + -device virtio-net-device,netdev=net0,mac=${mac} + -netdev bridge,br=br0,id=net0,helper=${helper} + ) + ;; + esac + + "${kvm[@]}" "${arch_option[@]}" --append "${append}" } check_logfile write_logfile -qemu_command parse_ipxe_script check_option_value + get_initrd +get_bios +get_helper +get_nic +get_device +get_netdev +get_qemu print_message +public_option run_qemu -- 2.23.0

2 2