- Compass-ci - mailweb.openeuler.org

[PATCH v2 compass-ci] tool/es-jobs: a new tool for summarizing jobs information
by Wu Zhende 13 Jan '21

13 Jan '21

Use es-jobs to summarizes information about a group of related jobs. [Example] es-jobs submit_id=xxx -f 'job_state,dmesg.timestamp:last' -r '-1' -s 'on_fail' means: 1. query the job whose submit_id is xxx 2. I want to get job_state,dmesg.timestamp:last 3. don't refine jobs 4. -s 'on_fail' to get data in stats which contain 'on_fail' [Output] { "job_state": [ "crystal.605716.failed" ], "dmesg.timestamp:last": [ 657.181408 ], "stats_filter_result": { "crystal.605716.openeuler_docker.\u001b[0mwordpress_build_on_fail": 1 }, "stats.count": { "stats.has_error": 1, "stats.has_error_jobs": [ "crystal.605716" ] } } Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/es_jobs.rb | 139 +++++++++++++++++++++++++++++++++++++++++++++++++ sbin/es-jobs | 58 +++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 lib/es_jobs.rb create mode 100755 sbin/es-jobs diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb new file mode 100644 index 0000000..a90b1ba --- /dev/null +++ b/lib/es_jobs.rb @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: GPL-2.0-only +# frozen_string_literal: true + +LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' + +require "#{LKP_SRC}/lib/stats" +require_relative './es_query' + +# deal jobs search from es +class ESJobs + def initialize(es_query, my_refine = [], fields = [], stats_filter = []) + @es_query = es_query + @es = ESQuery.new(ES_HOST, ES_PORT) + @refine = my_refine + @fields = fields + @stats_filter = stats_filter + @stats_filter_result = {} + @refine_jobs = [] + @jobs = {} + @stats_level = { + 0 => 'stats.success', + 1 => 'stats.unknown', + 2 => 'stats.warning', + 3 => 'stats.has_error' + } + set_defaults + deal_jobs + end + + def set_defaults + query_result = @es.multi_field_query(@es_query) + query_result['hits']['hits'].each do |job| + @jobs[job['_id']] = job['_source'] + end + + @stats = { + 'stats.count' => Hash.new(0), + 'stats.sum' => Hash.new(0), + 'stats.avg' => Hash.new(0) + } + @result = {} + @fields.each do |field| + @result[field] = [] + end + end + + def add_result_fields(job, level) + return unless @refine.include?(level) || @refine.include?(-1) + + @refine_jobs << job['id'] + @fields.each do |field| + value = job[field] + if value + value = job['id'] + '.' + value if field == 'job_state' + @result[field] << value + end + + next unless job['stats'] + + @result[field] << job['stats'][field] if job['stats'][field] + end + end + + def deal_jobs + stats_count = Hash.new(0) + stats_jobs = {} + + @jobs.each do |job_id, job| + level = deal_stats(job) + add_result_fields(job, level) + + stat_key = @stats_level[level] + stat_jobs_key = stat_key + '_jobs' + + stats_count[stat_key] += 1 + stats_jobs[stat_jobs_key] ||= [] + stats_jobs[stat_jobs_key] << job_id + end + + @stats['stats.count'].merge!(stats_count) + @stats['stats.count'].merge!(stats_jobs) + end + + def deal_stats(job, level = 0) + return 1 unless job['stats'] + + job['stats'].each do |key, value| + match_stats_filter(key, value, job['id']) + calculate_stat(key, value) + level = get_stat_level(key, level) + end + return level + end + + def match_stats_filter(key, value, job_id) + @stats_filter.each do |filter| + next unless key.include?(filter) + + key = job_id + '.' + key + @stats_filter_result[key] = value + + break + end + end + + def calculate_stat(key, value) + if function_stat?(key) + return unless @fields.include?('stats.sum') + + @stats['stats.sum'][key] += value + else + return unless @fields.include?('stats.avg') + + @stats['stats.avg'][key] = (@stats['stats.avg'][key] + value) / 2 + end + end + + def get_stat_level(stat, level) + return level if level >= 3 + return 3 if stat.match(/error|fail/i) + return 2 if stat.match(/warn/i) + + return 0 + end + + def output + result = { + 'stats.count' => @stats['stats.count'] + } + + @stats.each do |key, value| + result[key] = value if @fields.include?(key) + end + + @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? + @result.merge!(result) + puts JSON.pretty_generate(@result) + end +end diff --git a/sbin/es-jobs b/sbin/es-jobs new file mode 100755 index 0000000..25cbe9c --- /dev/null +++ b/sbin/es-jobs @@ -0,0 +1,58 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'optparse' +require 'json' + +require_relative '../lib/es_jobs' + +def parse_argv + items = {} + ARGV.each do |item| + key, value = item.split('=') + if key && value + value_list = value.split(',') + items[key] = value_list.length > 1 ? value_list : value + end + end + items +end +opt_refine = [-1] +opt_fields = [] +opt_stats_filter = [] +opt_parser = OptionParser.new do |opts| + opts.banner = 'Usage: es-jobs [options] search_key1=val1[,val2..] ..' + + opts.separator 'search_key can be id, suite, os, etc.' + opts.separator 'how to use -r' + opts.separator 'like es-jobs submit_id=xxx -r "-1,0,1,2,3"' + opts.separator '-1 means not refine' + opts.separator '{ + -1 => null, + 0 => stats.succes, + 1 => stats.unknow, + 2 => stats.warning, + 3 => stats.has_error +}' + + opts.on('-r fields', '--refine fields', 'refine jobs') do |fields| + opt_refine = fields.split(',').map(&:to_i) + end + + opts.on('-f fields', '--fields fields', 'fields you want to see') do |fields| + opt_fields = fields.split(',') + end + + opts.on('-s fields', '--stats-filter fields', 'return data contains fields') do |fields| + opt_stats_filter = fields.split(',') + end +end + +opt_parser.parse!(ARGV) +items = parse_argv +raise 'Please enter a query' if items.empty? + +es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter) +es_jobs.output -- 2.23.0

2 1

[PATCH v8 compass-ci] providers/qemu: adapt for x86_64 and riscv64
by Liu Yinsi 13 Jan '21

13 Jan '21

before: support kvm for aarch64 after: support kvm for aarch64, x86_64, riscv64 test: /srv/result/host-info/2021-01-12/vm-2p8g/debian-sid-aarch64/z9.318111 /srv/result/host-info/2021-01-12/vm-2p8g/debian-sid-x86_64/z9.318142 /srv/result/host-info/2021-01-12/vm-2p8g/debian-sid-riscv64/z9.319551 [how] put public options like $kernel, $initrd, $smp together, set individual options like $machine, $bios, $nic according to different qemu command, then merge all options to run qemu. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 121 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 106 insertions(+), 15 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index 577bca1..a61acab 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -18,12 +18,6 @@ check_logfile() } } -qemu_command() -{ - qemu=qemu-system-aarch64 - command -v $qemu >/dev/null || qemu=qemu-kvm -} - write_logfile() { ipxe_script=ipxe_script @@ -66,12 +60,57 @@ check_option_value() [ -n "$initrds" ] || exit } -get_initrd() +set_initrd() { initrd=initrd cat $initrds > $initrd } +set_bios() +{ + bios=/usr/share/qemu-efi-aarch64/QEMU_EFI.fd + [ -f "$bios" ] || bios=/usr/share/ovmf/OVMF.fd +} + +set_helper() +{ + helper=/usr/libexec/qemu-bridge-helper + [ -f "$helper" ] || helper=/usr/lib/qemu/qemu-bridge-helper +} + +set_nic() +{ + nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" +} + +set_device() +{ + device="virtio-net-device,netdev=net0,mac=${mac}" +} + +set_netdev() +{ + netdev="bridge,br=br0,id=net0,helper=${helper}" +} + +set_qemu() +{ + qemus=( + qemu-system-aarch64 + qemu-kvm + qemu-system-x86_64 + qemu-system-riscv64 + ) + + for qemu in "${qemus[@]}" + do + [ -n "$(command -v ${qemu})" ] && break + done + + # debian has both qemu-system-x86_64 and qemu-system-riscv64 command + [[ $kernel =~ 'riscv64' ]] && qemu=qemu-system-riscv64 +} + print_message() { echo $SCHED_PORT @@ -83,36 +122,88 @@ print_message() sleep 5 } -run_qemu() +public_option() { kvm=( $qemu - -machine virt-4.0,accel=kvm,gic-version=3 -kernel $kernel -initrd $initrd -smp $nr_cpu -m $memory - -cpu Kunpeng-920 - -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd - -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} -k en-us -no-reboot -nographic -serial file:${log_file} -monitor null ) +} + +individual_option() +{ + case "$qemu" in + qemu-system-aarch64) + arch_option=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) + ;; + qemu-kvm) + [ "$(arch)" == "aarch64" ] && arch_option=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) + [ "$(arch)" == "x86_64" ] && arch_option=( + -bios $bios + -nic $nic + ) + ;; + qemu-system-x86_64) + arch_option=( + -bios $bios + -nic $nic + ) + ;; + qemu-system-riscv64) + arch_option=( + -machine virt + -device $device + -netdev $netdev + ) + ;; + esac +} - "${kvm[@]}" --append "${append}" +run_qemu() +{ + "${kvm[@]}" "${arch_option[@]}" --append "${append}" +} + +set_options() +{ + set_initrd + set_bios + set_helper + set_nic + set_device + set_netdev + set_qemu } check_logfile write_logfile -qemu_command parse_ipxe_script check_option_value -get_initrd + +set_options print_message +public_option +individual_option + run_qemu -- 2.23.0

2 2

[PATCH v2 lkp-tests] tests/*: add makepkg -s operation in archlinux
by Wang Yong 13 Jan '21

13 Jan '21

use makepkg -s to auto install depends in archlinux ref: https://git.archlinux.org/pacman.git/plain/scripts/makepkg.sh.in Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/build-pkg | 11 +++++++++-- tests/cci-makepkg | 14 +++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/build-pkg b/tests/build-pkg index 31762384d..db4007b20 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -92,7 +92,9 @@ create_cgz_name() cgz_path="$PKG_MNT/${pack_to}/${pkgname}" cgz_name="${cgz_path}/${upstream_commit}.cgz" + pkg_args="-A --check --skippgpcheck" + [ "${os}" == "archlinux" ] && pkg_args="${pkg_args} -s --needed --noconfirm" [ -n "$git_tag" ] && { cgz_name="${cgz_path}/${git_tag}-${pkgrel}.cgz" @@ -153,8 +155,13 @@ build_source_pkg() create_cgz_name get_config - PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ - $LKP_SRC/sbin/makepkg ${pkg_args} --config $LKP_SRC/etc/makepkg.conf + if [ "${os}" == "archlinux" ]; then + BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ + $LKP_SRC/sbin/makepkg ${pkg_args} --config $LKP_SRC/etc/makepkg.conf + else + PACMAN=true BUILDDIR=$TMP CARCH=$os_arch PKGEXT=.cgz CGZDEST="$cgz_name" \ + $LKP_SRC/sbin/makepkg ${pkg_args} --config $LKP_SRC/etc/makepkg.conf + fi local build_state="$?" diff --git a/tests/cci-makepkg b/tests/cci-makepkg index 7d1e5cb15..e4bab41eb 100755 --- a/tests/cci-makepkg +++ b/tests/cci-makepkg @@ -133,10 +133,18 @@ update_softlink() build_source_pkg() { - local makepkg="$LKP_SRC/sbin/makepkg -A --skippgpcheck --config $LKP_SRC/etc/makepkg.conf" - echo "$makepkg" + if [ "${DISTRO}" == "archlinux" ]; then + local makepkg="$LKP_SRC/sbin/makepkg -A -s --needed --noconfirm --skippgpcheck --config $LKP_SRC/etc/makepkg.conf" + echo "$makepkg" - PACMAN=true BUILDDIR=$TMP CARCH=$arch PKGEXT=.cgz CGZDEST="$sync_dest/$cgz_name" $makepkg + BUILDDIR=$TMP CARCH=$arch PKGEXT=.cgz CGZDEST="$sync_dest/$cgz_name" $makepkg + + else + local makepkg="$LKP_SRC/sbin/makepkg -A --skippgpcheck --config $LKP_SRC/etc/makepkg.conf" + echo "$makepkg" + + PACMAN=true BUILDDIR=$TMP CARCH=$arch PKGEXT=.cgz CGZDEST="$sync_dest/$cgz_name" $makepkg + fi [ "$?" == 0 ] || exit 1 update_softlink -- 2.23.0

2 2

[PATCH v3 lkp-tests] tests/send_email: refactor send_email
by Wei Jihui 13 Jan '21

13 Jan '21

It is convenient for other tasks reuse to send email [usage] job.yaml test_task: on_fail: send_email: job_failed sleep: 1h This will send email to user when job is failed. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/email.sh | 26 ++++++++++++++++++++++++++ tests/email | 41 ----------------------------------------- tests/send_email | 20 ++++++++++++++++++++ 3 files changed, 46 insertions(+), 41 deletions(-) create mode 100755 lib/email.sh delete mode 100755 tests/email create mode 100755 tests/send_email diff --git a/lib/email.sh b/lib/email.sh new file mode 100755 index 00000000..d7c2c44a --- /dev/null +++ b/lib/email.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +job_failed_content() +{ + email_content="To: $my_email +Subject: [NOTIFY Compass-CI] Test job failed: $id + +Dear $my_name: + + Sorry to inform you that your test job is failed, you can click the following link to view details. + http://api.compass-ci.openeuler.org:${SRV_HTTP_PORT:-11300}$result_root + +Regards +Compass-CI +" +} + +send_email() +{ + local subject=$1 + local email_content + + ${subject}_content + + curl -XPOST "${MAIL_HOST:-$LKP_SERVER}:${MAIL_PORT:-49000}/send_mail_text" -d "$email_content" +} diff --git a/tests/email b/tests/email deleted file mode 100755 index a5f87d42..00000000 --- a/tests/email +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh -# - my_email -# - info - -: ${info:=failure} - -SCRIPT_DIR=$(dirname $(realpath $0)) -PROJECT_DIR=$(dirname $SCRIPT_DIR) - -. $PROJECT_DIR/lib/http.sh -. $PROJECT_DIR/lib/env.sh - -data_failure() -{ - data="To: $my_email -Subject: [NOTIFY Compass-ci] Test job failed - -Dear $my_username: - - Sorry to inform you that your test job is failed. you can click the following link to view details. - http://api.compass-ci.openeuler.org:$SRV_HTTP_PORT/$result_root - -Regards -Compass-Ci -" -} - -compose_email() -{ - [ "$info" == "failure" ] && data_failure -} - -run_email() -{ - [ -n "$my_email" ] || return 0 - my_username=$(echo "$my_email" | awk -F '@' '{print $1}') - compose_email - curl -XPOST "${MAIL_HOST:-$LKP_SERVER}:${MAIL_PORT:-49000}/send_mail_text" -d "$data" -} - -run_email diff --git a/tests/send_email b/tests/send_email new file mode 100755 index 00000000..0132c3c8 --- /dev/null +++ b/tests/send_email @@ -0,0 +1,20 @@ +#!/bin/sh +# - subject +# my_email + +SCRIPT_DIR=$(dirname $(realpath $0)) +PROJECT_DIR=$(dirname $SCRIPT_DIR) + +. $PROJECT_DIR/lib/email.sh + +[ -n "$my_email" ] || { + echo "Can not get my_email, send email failed!" + exit 0 +} + +[ -n "$subject" ] || { + echo "Can not get email subject, send email failed!" + exit 0 +} + +send_email "$subject" -- 2.23.0

1 0

[PATCH v2 lkp-tests 1/2] reset-tbox to reboot machine testbox
by Zhang Dewan 13 Jan '21

13 Jan '21

[why] sumbit a job to reboot machine testbox is better. [how] cp ~/lkp-test/jobs/reset-tbox-yaml . submit reset-tbox.yaml host_name="full-name-of-machine-testbox" Signed-off-by: Zhang Dewan <513619464(a)qq.com> --- jobs/reset-tbox.yaml | 10 ++++++++++ tests/reset-tbox | 17 +++++++++++++++++ tests/reset-tbox.yaml | 12 ++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 jobs/reset-tbox.yaml create mode 100755 tests/reset-tbox create mode 100644 tests/reset-tbox.yaml diff --git a/jobs/reset-tbox.yaml b/jobs/reset-tbox.yaml new file mode 100644 index 00000000..9c3f4f01 --- /dev/null +++ b/jobs/reset-tbox.yaml @@ -0,0 +1,10 @@ +suite: reset-tbox +testcase: reset-tbox + +testbox: dc-8g +os_mount: container +docker_image: 172.168.131.2:5001/reset-tbox +queue: reset + +host_name: +reset-tbox: diff --git a/tests/reset-tbox b/tests/reset-tbox new file mode 100755 index 00000000..b0f645d7 --- /dev/null +++ b/tests/reset-tbox @@ -0,0 +1,17 @@ +#!/bin/bash +# host_name + +source "/etc/lab/.account.info" +hosts_file=/etc/lab/hosts_file + +# require $host_name +host_ibmc_ip=$(grep -w $host_name $hosts_file | awk '{print $1}') + +if [[ "$host_ibmc_ip" =~ ^[0-9]+.* ]]; then + echo ">>> exec ipmitool commamd!" + ipmitool -I lanplus -H $host_ibmc_ip -U $iBMC_user -P $iBMC_passwd power reset + [ $? -eq 0 ] && echo "succeed!" +else + echo "specify host cannot found0!" + exit +fi diff --git a/tests/reset-tbox.yaml b/tests/reset-tbox.yaml new file mode 100644 index 00000000..f557085f --- /dev/null +++ b/tests/reset-tbox.yaml @@ -0,0 +1,12 @@ +short_description: > + To reboot machine testbox like taishan200-2280-2s64p-256g--aN etc. + +description: | + reset-tbox executed by a special docker which running on ibmc-mgnt, where you can access all server's iBMC-os by ipmitool, and then reboot the host. + +homepage: + +parameters: + host_name: see it on the first line about 'latbox aN' + +results: -- 2.23.0

1 0

[PATCH compass-ci] sparrow/0-package: install libvirt
by Xiao Shenwei 13 Jan '21

13 Jan '21

Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- sparrow/0-package/common | 1 + sparrow/0-package/os/openEuler | 1 + 2 files changed, 2 insertions(+) diff --git a/sparrow/0-package/common b/sparrow/0-package/common index 4a9da60..2d7c9e3 100755 --- a/sparrow/0-package/common +++ b/sparrow/0-package/common @@ -51,3 +51,4 @@ sed '/^Environment=/a\TimeoutSec=0' /usr/lib/systemd/system/docker.service systemctl daemon-reload systemctl restart docker +systemctl start libvirtd diff --git a/sparrow/0-package/os/openEuler b/sparrow/0-package/os/openEuler index a63964f..37c74d0 100755 --- a/sparrow/0-package/os/openEuler +++ b/sparrow/0-package/os/openEuler @@ -21,6 +21,7 @@ pkgs=( busybox iproute hostname + libvirt ) yum install -y --skip-broken "${pkgs[@]}" -- 2.23.0

1 0

[PATCH compass-ci] sparrow/3-code: set default values
by Liu Yinsi 13 Jan '21

13 Jan '21

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index f2366c2..ca91da5 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -20,6 +20,8 @@ server_ip=$(ip route get 1.2.3.4 | awk '{print $7; exit}') : ${LOCAL_ROBOT_EMAIL_ADDRESS:=Crystal TEAM} : ${ASSIST_RESULT_PORT:=8102} : ${RESULT_WEBDAV_PORT:=3080} +: ${ES_HOST:=$server_ip} +: ${ES_PORT:=9200} mkdir -p /etc/compass-ci/defaults cat > /etc/compass-ci/defaults/$server_name.yaml <<EOF @@ -45,6 +47,9 @@ LOCAL_ROBOT_EMAIL_ADDRESS: $LOCAL_ROBOT_EMAIL_ADDRESS MAILDIR: /srv/cci/Maildir/.compass-ci ASSIST_RESULT_PORT: $ASSIST_RESULT_PORT RESULT_WEBDAV_PORT: $RESULT_WEBDAV_PORT +lab: nolab +ES_HOST: $ES_HOST +ES_PORT: $ES_PORT EOF mkdir -p /etc/compass-ci/account -- 2.23.0

1 0

[PATCH lkp-tests] tests/send_email: refactor send_email
by Wei Jihui 13 Jan '21

13 Jan '21

It is convenient for other tasks reuse to send email [usage] job.yaml test_task: on_fail: send_email: job_failed sleep: 1h This will send email to user when job is failed. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/email.sh | 26 ++++++++++++++++++++++++++ tests/email | 41 ----------------------------------------- tests/send_email | 21 +++++++++++++++++++++ 3 files changed, 47 insertions(+), 41 deletions(-) create mode 100755 lib/email.sh delete mode 100755 tests/email create mode 100755 tests/send_email diff --git a/lib/email.sh b/lib/email.sh new file mode 100755 index 00000000..a407ad40 --- /dev/null +++ b/lib/email.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +job_failed_content() +{ + email_content="To: $my_email +Subject: [NOTIFY Compass-CI] Test job failed: $id + +Dear $my_name: + + Sorry to inform you that your test job is failed. you can click the following link to view details. + http://api.compass-ci.openeuler.org:${SRV_HTTP_PORT:-11300}$result_root + +Regards +Compass-CI +" +} + +send_email() +{ + local subject=$1 + local email_content + + ${subject}_content + + curl -XPOST "${MAIL_HOST:-$LKP_SERVER}:${MAIL_PORT:-49000}/send_mail_text" -d "$email_content" +} diff --git a/tests/email b/tests/email deleted file mode 100755 index a5f87d42..00000000 --- a/tests/email +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh -# - my_email -# - info - -: ${info:=failure} - -SCRIPT_DIR=$(dirname $(realpath $0)) -PROJECT_DIR=$(dirname $SCRIPT_DIR) - -. $PROJECT_DIR/lib/http.sh -. $PROJECT_DIR/lib/env.sh - -data_failure() -{ - data="To: $my_email -Subject: [NOTIFY Compass-ci] Test job failed - -Dear $my_username: - - Sorry to inform you that your test job is failed. you can click the following link to view details. - http://api.compass-ci.openeuler.org:$SRV_HTTP_PORT/$result_root - -Regards -Compass-Ci -" -} - -compose_email() -{ - [ "$info" == "failure" ] && data_failure -} - -run_email() -{ - [ -n "$my_email" ] || return 0 - my_username=$(echo "$my_email" | awk -F '@' '{print $1}') - compose_email - curl -XPOST "${MAIL_HOST:-$LKP_SERVER}:${MAIL_PORT:-49000}/send_mail_text" -d "$data" -} - -run_email diff --git a/tests/send_email b/tests/send_email new file mode 100755 index 00000000..2911d3b8 --- /dev/null +++ b/tests/send_email @@ -0,0 +1,21 @@ +#!/bin/sh +# - subject +# my_email + +SCRIPT_DIR=$(dirname $(realpath $0)) +PROJECT_DIR=$(dirname $SCRIPT_DIR) + +. $PROJECT_DIR/lib/email.sh + +[ -n "$my_email" ] || { + echo "Can not get my_email, send email failed!" + exit 0 +} + +[ -n "$subject" ] || { + echo "Can not get email subject, send email failed!" + exit 0 +} + + +send_email $subject -- 2.23.0

3 2

[PATCH v2 lkp-tests 1/3] stats/crystal/memory--stat.cr: create memory--stat.cr with memory--stat
by Lu Kaiyi 13 Jan '21

13 Jan '21

create memory--stat.cr as same content with memory--stat Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- stats/crystal/memory--stat.cr | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 stats/crystal/memory--stat.cr diff --git a/stats/crystal/memory--stat.cr b/stats/crystal/memory--stat.cr new file mode 100644 index 000000000..1a39e4075 --- /dev/null +++ b/stats/crystal/memory--stat.cr @@ -0,0 +1,22 @@ +#!/usr/bin/env ruby + +# time: 123456 +# cgroup: pmbench.1 +# cache 0 + +def parse + cg = 'nocg' + $stdin.each_line do |line| + case line + when /^time:/ + puts line + when /^cgroup: (\S+)/ + cg = $1 + when /(\S+)\s*(\S+)/ + k, v, = line.split + puts "#{cg}.#{k}: #{v}" + end + end +end + +parse -- 2.23.0

2 1

[PATCH v8 compass-ci 2/2] lib/es_query.rb: parse es_result to a Hash result
by Lu Kaiyi 13 Jan '21

13 Jan '21

[why] make es result more easy access. [example] input: es_result = [{"key"=>"build-pkg", "doc_count"=>354526, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>5708}, {"key"=>"finished", "doc_count"=>1033}, {"key"=>"incomplete", "doc_count"=>204}, {"key"=>"submit", "doc_count"=>136}, ... output: result_hash = {"build-pkg"=> {"failed"=>5708, "finished"=>1033, "incomplete"=>204, "submit"=>136, "OOM"=>11, "post_run"=>2}, "cci-depends"=> {"finished"=>1785, "failed"=>675, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 91aa6ab..928bd68 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -128,7 +128,7 @@ class ESQuery result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] return nil if result.empty? - result + parse_fields(result) end end @@ -216,3 +216,47 @@ def build_aggs_from_fields(fields) aggs_hash['aggs']["all_#{field}"].merge!(sub_aggs) if sub_aggs aggs_hash end + +# input: +# es_result = +# [{"key"=>"build-pkg", +# "doc_count"=>354526, +# "all_job_state"=> +# {"doc_count_error_upper_bound"=>0, +# "sum_other_doc_count"=>0, +# "buckets"=> +# [{"key"=>"failed", "doc_count"=>5708}, +# {"key"=>"finished", "doc_count"=>1033}, +# {"key"=>"incomplete", "doc_count"=>204}, +# {"key"=>"submit", "doc_count"=>136}, +# ... +# +# output: +# result_hash = +# {"build-pkg"=> +# {"failed"=>5708, +# "finished"=>1033, +# "incomplete"=>204, +# "submit"=>136, +# "OOM"=>11, +# "post_run"=>2}, +# "cci-depends"=> +# {"finished"=>1785, +# "failed"=>675, +# ... +def parse_fields(es_result) + result_hash = {} + es_result.each do |result| + key = result['key'] + sub_field = result.keys.detect { |field| field.start_with?('all_') } + + if sub_field + all_field = result[sub_field]['buckets'] + result_hash[key] = parse_fields(all_field) + else + result_hash[key] = result['doc_count'] + end + end + + result_hash +end -- 2.23.0

3 3