- Compass-ci - mailweb.openeuler.org

[PATCH v4 compass-ci 2/2] lib/es_query.rb: parse es_result to a Hash result
by Lu Kaiyi 04 Jan '21

04 Jan '21

[why] make es result more easy access. [example] input: result_hash = {} es_result = [{"key"=>"build-pkg", "doc_count"=>354526, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>5708}, {"key"=>"finished", "doc_count"=>1033}, {"key"=>"incomplete", "doc_count"=>204}, {"key"=>"submit", "doc_count"=>136}, ... output: result_hash = {"build-pkg"=> {"failed"=>5708, "finished"=>1033, "incomplete"=>204, "submit"=>136, "OOM"=>11, "post_run"=>2}, "cci-depends"=> {"finished"=>1785, "failed"=>675, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 29d50e9..63d32ce 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -129,7 +129,7 @@ class ESQuery result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] return nil if result.empty? - result + parse_fields(result) end end @@ -217,3 +217,47 @@ def build_aggs_from_fields(fields, aggs_hash) build_aggs_from_fields(fields, aggs_hash['aggs']["all_#{field}"]) end + +# input: +# es_result = +# [{"key"=>"build-pkg", +# "doc_count"=>354526, +# "all_job_state"=> +# {"doc_count_error_upper_bound"=>0, +# "sum_other_doc_count"=>0, +# "buckets"=> +# [{"key"=>"failed", "doc_count"=>5708}, +# {"key"=>"finished", "doc_count"=>1033}, +# {"key"=>"incomplete", "doc_count"=>204}, +# {"key"=>"submit", "doc_count"=>136}, +# ... +# +# output: +# result_hash = +# {"build-pkg"=> +# {"failed"=>5708, +# "finished"=>1033, +# "incomplete"=>204, +# "submit"=>136, +# "OOM"=>11, +# "post_run"=>2}, +# "cci-depends"=> +# {"finished"=>1785, +# "failed"=>675, +# ... +def parse_fields(es_result) + result_hash ||= {} + es_result.each do |result| + key = result['key'] + sub_field = result.keys.detect { |field| field.start_with?('all_') } + + if sub_field + all_field = result[sub_field]['buckets'] + result_hash[key] = parse_fields(all_field) + else + result_hash[key] = result['doc_count'] + end + end + + result_hash +end -- 2.23.0

1 0

[PATCH v4 compass-ci 1/2] lib/es_query.rb: get multi-field aggregation count result
by Lu Kaiyi 04 Jan '21

04 Jan '21

[why] get multi-field aggregation count for overview of report page. [input]: fields, query_items (optional for query_items, default no scope limitation) [example] input: fields = ['suite'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186214}, {"key"=>"cci-depends", "doc_count"=>1230}, {"key"=>"iperf", "doc_count"=>1103}, {"key"=>"spinlock", "doc_count"=>1000}, {"key"=>"cci-makepkg", "doc_count"=>949}, {"key"=>"iperf-walk-os-test", "doc_count"=>474}, {"key"=>"bisect", "doc_count"=>381}, ... input: fields = ['suite', 'job_state'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186227, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3830}, {"key"=>"finished", "doc_count"=>803}, {"key"=>"incomplete", "doc_count"=>196}, {"key"=>"submit", "doc_count"=>136}, ... input: fields = ['os', 'os_version', 'job_state'] query_items = {} output: [{"key"=>"openeuler", "doc_count"=>193287, "all_os_version"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"20.03", "doc_count"=>191789, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3909}, {"key"=>"finished", "doc_count"=>2401}, {"key"=>"incomplete", "doc_count"=>300}, {"key"=>"submit", "doc_count"=>136}, {"key"=>"OOM", "doc_count"=>23}, {"key"=>"post_run", "doc_count"=>1}]}}, {"key"=>"20.03-SP1", "doc_count"=>904, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>641}, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/lib/es_query.rb b/lib/es_query.rb index 88a420f..29d50e9 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -87,6 +87,50 @@ class ESQuery result end + + # input: + # fields, query_items + # + # fields => fields: Array(keyword) + # eg: + # ['suite', 'job_state'] + # + # query_items => query_items: Hash(keyword, value) + # eg: + # { + # 'os' => 'openeuler', + # ... + # } + # (optional for query_items, default no scope limitation) + # output: + # [{"key"=>"build-pkg", + # "doc_count"=>186175, + # "all_job_state"=> + # {"doc_count_error_upper_bound"=>0, + # "sum_other_doc_count"=>0, + # "buckets"=> + # [{"key"=>"failed", "doc_count"=>3830}, + # {"key"=>"finished", "doc_count"=>803}, + # {"key"=>"incomplete", "doc_count"=>196}, + # ... + def query_fields(fields, query_items = {}) + field1 = fields.first + aggs_hash = {} + build_aggs_from_fields(fields, aggs_hash) + query = { + query: { + bool: { + must: build_multi_field_subquery_body(query_items) + } + }, + aggs: aggs_hash['aggs'], + size: 0 + } + result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] + return nil if result.empty? + + result + end end # Range Query Example: @@ -146,3 +190,30 @@ def assign_desc_body(keyword) }] } end + +# input: +# fields = ['os', 'os_version', 'job_state'] +# aggs_hash = {} +# output: +# aggs_hash = { +# "aggs"=> +# {"all_os"=> +# {"terms"=>{:field=>"os", :size=>1000}, +# "aggs"=> +# {"all_os_version"=> +# {"terms"=>{:field=>"os_version", :size=>1000}, +# "aggs"=> +# {"all_job_state"=>{"terms"=>{:field=>"job_state", :size=>1000}}}}}}} +# } +def build_aggs_from_fields(fields, aggs_hash) + return if fields.empty? + + field = fields.shift + aggs_hash['aggs'] = { + "all_#{field}" => { + 'terms' => { field: field, size: 1000 } + } + } + + build_aggs_from_fields(fields, aggs_hash['aggs']["all_#{field}"]) +end -- 2.23.0

1 0

[PATCH v2 compass-ci] docker-rootfs/common: add comment for set password file.
by Wang Chenglong 04 Jan '21

04 Jan '21

[Why] add password file is easy to ignore. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container/docker-rootfs/common b/container/docker-rootfs/common index fe37da9..2ee16c8 100755 --- a/container/docker-rootfs/common +++ b/container/docker-rootfs/common @@ -33,7 +33,8 @@ usage() dst_rootfs_new_abspath: destination absolute path to create for rootfs Example: - ./run /tmp/openEuler-docker/openEuler-docker.${aarch}.tar.xz /tmp/openeuler-rootfs/ + ./run /tmp/openEuler-docker/openEuler-docker.${aarch}.tar.xz /tmp/openeuler-rootfs/ + # please ensure $HOME/.config/compass-ci/rootfs-passwd exists " } -- 2.23.0

3 4

[PATCH v3 compass-ci] docker-rootfs: add README.md
by Wang Chenglong 04 Jan '21

04 Jan '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 container/docker-rootfs/README.md diff --git a/container/docker-rootfs/README.md b/container/docker-rootfs/README.md new file mode 100644 index 0000000..1b6f42f --- /dev/null +++ b/container/docker-rootfs/README.md @@ -0,0 +1,18 @@ +## Use the tool to create a new nfsroot for os + +## Usage: + ./run <src_docker_file_abspath> <dst_rootfs_new_abspath> + + src_docker_file_abspath: source .tar.xz file absolute path with suffix: [ tar.xz ]. + dst_rootfs_new_abspath: destination absolute path to create for rootfs. + +## Example: + ./run /tmp/openEuler-docker/openEuler-docker.aarch64.tar.xz /tmp/openeuler-rootfs/ + # please ensure $HOME/.config/compass-ci/rootfs-passwd exists + +## Some configuration items: + ./packages-to-install + # If you want to pre-install the software, you can write the package names in ./packages-to-install. + + $HOME/.config/compass-ci/rootfs-passwd + # Set the password for the image into this file. -- 2.23.0

2 2

[PATCH v4 compass-ci] docker-rootfs: add README.md
by Wang Chenglong 04 Jan '21

04 Jan '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 container/docker-rootfs/README.md diff --git a/container/docker-rootfs/README.md b/container/docker-rootfs/README.md new file mode 100644 index 0000000..e7b5084 --- /dev/null +++ b/container/docker-rootfs/README.md @@ -0,0 +1,18 @@ +# Use the tool to create a new nfsroot for os + +## Usage: + ./run <src_docker_file_abspath> <dst_rootfs_new_abspath> + + src_docker_file_abspath: source .tar.xz file absolute path with suffix: [ tar.xz ]. + dst_rootfs_new_abspath: destination absolute path to create for rootfs. + +## Example: + ./run /tmp/openEuler-docker/openEuler-docker.aarch64.tar.xz /tmp/openeuler-rootfs/ + # Please ensure $HOME/.config/compass-ci/rootfs-passwd exists. + +## Some configuration items: + ./packages-to-install + # If you want to pre-install the software, you can write the package names in ./packages-to-install. + + $HOME/.config/compass-ci/rootfs-passwd + # Set the password for the image into this file. -- 2.23.0

1 0

[PATCH compass-ci 2/2] qcow2rootfs: fix rootfs-dir permission error
by Yu Chuan 03 Jan '21

03 Jan '21

[Error Msg] '01/03/21_04:25:49' Compressing file to /root/extract/rootfs-dir/qcow2-out.tar.gz Please wait until command returned... <r.gz compress:gzip numericowner:true xattrs:true selinux:true acls:true libguestfs: error: /root/extract/rootfs-dir/qcow2-out.tar.gz: Permission denied libguestfs: error: file receive cancelled by daemon Signed-off-by: Yu Chuan <13186087857(a)163.com> --- container/qcow2rootfs/bin/main | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/qcow2rootfs/bin/main b/container/qcow2rootfs/bin/main index b6ef7238f24a..c7527b794b89 100755 --- a/container/qcow2rootfs/bin/main +++ b/container/qcow2rootfs/bin/main @@ -34,7 +34,7 @@ unzip_qcow2_xz() { extract_rootfs() { echo "[WARNNING] Temporarily change owner of qcow2 file" chown "$EXTRACT_USER":"$EXTRACT_USER" "$QCOW2_FILE" - chmod 775 "$ROOTFS_DIR" + chmod 777 "$ROOTFS_DIR" echo "[$(date +'%D %T')] Converting .qcow2 to .tar.gz, please wait several minutes..." su - "$EXTRACT_USER" -c " -- 2.23.0

1 0

[PATCH compass-ci 1/2] qcow2rootfs: generate pkgs record file for result rootfs
by Yu Chuan 03 Jan '21

03 Jan '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- container/qcow2rootfs/bin/cci-generate-pkgs | 15 +++++++++++++++ container/qcow2rootfs/bin/main | 6 ++++++ 2 files changed, 21 insertions(+) create mode 100755 container/qcow2rootfs/bin/cci-generate-pkgs diff --git a/container/qcow2rootfs/bin/cci-generate-pkgs b/container/qcow2rootfs/bin/cci-generate-pkgs new file mode 100755 index 000000000000..e6a974cd0f41 --- /dev/null +++ b/container/qcow2rootfs/bin/cci-generate-pkgs @@ -0,0 +1,15 @@ +#!/bin/bash + +RESULT_FILE=/etc/pkgs-$(date "+%Y-%m-%d-%H-%M-%S").list + +command -v rpm >/dev/null && { + rpm -qa > $RESULT_FILE +} + +command -v dpkg >/dev/null && { + dpkg -l > $RESULT_FILE +} + +chmod 644 $RESULT_FILE + +exit 0 diff --git a/container/qcow2rootfs/bin/main b/container/qcow2rootfs/bin/main index b70ec7766649..b6ef7238f24a 100755 --- a/container/qcow2rootfs/bin/main +++ b/container/qcow2rootfs/bin/main @@ -74,6 +74,12 @@ pre_config_rootfs() { if [ -f ${ROOTFS_DIR}/etc/selinux/config ]; then sed -i -r 's/SELINUX=enforcing/SELINUX=disabled/g' "$ROOTFS_DIR/etc/selinux/config" fi + + # generate /etc/pkgs-$(date "+%Y-%m-%d-%H-%M-%S").list file for result rootfs + echo "generate /etc/pkgs-$(date "+%Y-%m-%d-%H-%M-%S").list" + cp $EXTRACT_ROOT/bin/cci-generate-pkgs $ROOTFS_DIR + chroot $ROOTFS_DIR /cci-generate-pkgs + rm /$ROOTFS_DIR/cci-generate-pkgs } clean_environment() { -- 2.23.0

1 0

[PATCH v2 lkp-tests] lib/bootstrap.sh: 'run-ipconfig' step should be before the check step
by Yu Chuan 03 Jan '21

03 Jan '21

[Why] The `run-ipconfig` step is the IP configuration step for initramfs, which should be executed immediately after checking the initramfs-tools file for existence. [Error Msg] + [ ! -e /usr/share/initramfs-tools/scripts/functions ] + get_net_devices + local net_devices + local i + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + [ p0s1/ != /sys/class/net/enp0s1/ ] + basename /sys/class/net/enp0s1/ + net_devices= enp0s1 + [ != /sys/class/net/lo/ ] + continue + echo enp0s1 + local net_devices= enp0s1 + [ -z ] + warn_no_eth0 + [ -f /proc/config.gz ] + return + echo ls /sys/class/net + ls /sys/class/net + is_virt + [ -n ] + [ -n ] + has_cmd virt-what + command -v virt-what + grep -q -w hypervisor /proc/cpuinfo + return 0 + export NO_NETWORK=1 + echo export NO_NETWORK=1 due to no net devices + return Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 7d15bb370fa9..770d550724f1 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -160,6 +160,9 @@ setup_network() return } + $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig + network_ok && return + local net_devices=$(get_net_devices) if [ -z "$net_devices" ]; then @@ -181,9 +184,6 @@ setup_network() fi fi - $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig - network_ok && return - local err_msg='IP-Config: Auto-configuration of network failed' dmesg | grep -q -F "$err_msg" || { # Include $err_msg in the error message so that it matches -- 2.23.0

1 0

[PATCH lkp-tests] lib/bootstrap.sh: 'run-ipconfig' step should be before the check step
by Yu Chuan 03 Jan '21

03 Jan '21

[Before] test job: result/borrow/2020-12-31/vm-2p8g/debian-sid-aarch64/3600/z9.275666 key log: + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + cat /sys/class/net/enp0s1//operstate + [ down = up ] [After] test job: result/borrow/2020-12-31/vm-2p8g/debian-sid-aarch64/3600/z9.275673 key log: + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + cat /sys/class/net/enp0s1//operstate + [ up = up ] Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 7d15bb370fa9..770d550724f1 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -160,6 +160,9 @@ setup_network() return } + $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig + network_ok && return + local net_devices=$(get_net_devices) if [ -z "$net_devices" ]; then @@ -181,9 +184,6 @@ setup_network() fi fi - $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig - network_ok && return - local err_msg='IP-Config: Auto-configuration of network failed' dmesg | grep -q -F "$err_msg" || { # Include $err_msg in the error message so that it matches -- 2.23.0

2 2

git count
by Wu Fengguang 31 Dec '20

31 Dec '20

Cao Xueliang is our code star again, thanks xueliang! 22 Cao Xueliang 21 Luan Shengde 16 Wu Zhende 13 Wei Jihui 13 Li Yuanchao 11 Hu Xuejiao 11 Ren Wen 10 Lu Weitao 9 Liu Shaofei 7 Lin Jiaxin 7 Lu Kaiyi 6 Liu Yinsi 6 Cui Yili 6 Wang Yong 5 Li Ping 4 Xiao Shenwei 3 Wu Fengguang 3 Zhang Yu 3 Yu Chuan 2 Xu Xijian 2 Wang Chenglong 1 Zhang Yale 0 Bai Jing 0 Tong Qunfeng 0 Du Kaitian 0 Sun Yukui 0 Zhang Dewan 0 Shi Zhichao

1 0