January 2021 - Compass-ci - mailweb.openeuler.org

[PATCH v4 compass-ci 2/2] lib/es_query.rb: parse es_result to a Hash result
by Lu Kaiyi 04 Jan '21

04 Jan '21

[why] make es result more easy access. [example] input: result_hash = {} es_result = [{"key"=>"build-pkg", "doc_count"=>354526, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>5708}, {"key"=>"finished", "doc_count"=>1033}, {"key"=>"incomplete", "doc_count"=>204}, {"key"=>"submit", "doc_count"=>136}, ... output: result_hash = {"build-pkg"=> {"failed"=>5708, "finished"=>1033, "incomplete"=>204, "submit"=>136, "OOM"=>11, "post_run"=>2}, "cci-depends"=> {"finished"=>1785, "failed"=>675, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 29d50e9..63d32ce 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -129,7 +129,7 @@ class ESQuery result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] return nil if result.empty? - result + parse_fields(result) end end @@ -217,3 +217,47 @@ def build_aggs_from_fields(fields, aggs_hash) build_aggs_from_fields(fields, aggs_hash['aggs']["all_#{field}"]) end + +# input: +# es_result = +# [{"key"=>"build-pkg", +# "doc_count"=>354526, +# "all_job_state"=> +# {"doc_count_error_upper_bound"=>0, +# "sum_other_doc_count"=>0, +# "buckets"=> +# [{"key"=>"failed", "doc_count"=>5708}, +# {"key"=>"finished", "doc_count"=>1033}, +# {"key"=>"incomplete", "doc_count"=>204}, +# {"key"=>"submit", "doc_count"=>136}, +# ... +# +# output: +# result_hash = +# {"build-pkg"=> +# {"failed"=>5708, +# "finished"=>1033, +# "incomplete"=>204, +# "submit"=>136, +# "OOM"=>11, +# "post_run"=>2}, +# "cci-depends"=> +# {"finished"=>1785, +# "failed"=>675, +# ... +def parse_fields(es_result) + result_hash ||= {} + es_result.each do |result| + key = result['key'] + sub_field = result.keys.detect { |field| field.start_with?('all_') } + + if sub_field + all_field = result[sub_field]['buckets'] + result_hash[key] = parse_fields(all_field) + else + result_hash[key] = result['doc_count'] + end + end + + result_hash +end -- 2.23.0

1 0

[PATCH v4 compass-ci 1/2] lib/es_query.rb: get multi-field aggregation count result
by Lu Kaiyi 04 Jan '21

04 Jan '21

[why] get multi-field aggregation count for overview of report page. [input]: fields, query_items (optional for query_items, default no scope limitation) [example] input: fields = ['suite'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186214}, {"key"=>"cci-depends", "doc_count"=>1230}, {"key"=>"iperf", "doc_count"=>1103}, {"key"=>"spinlock", "doc_count"=>1000}, {"key"=>"cci-makepkg", "doc_count"=>949}, {"key"=>"iperf-walk-os-test", "doc_count"=>474}, {"key"=>"bisect", "doc_count"=>381}, ... input: fields = ['suite', 'job_state'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186227, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3830}, {"key"=>"finished", "doc_count"=>803}, {"key"=>"incomplete", "doc_count"=>196}, {"key"=>"submit", "doc_count"=>136}, ... input: fields = ['os', 'os_version', 'job_state'] query_items = {} output: [{"key"=>"openeuler", "doc_count"=>193287, "all_os_version"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"20.03", "doc_count"=>191789, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3909}, {"key"=>"finished", "doc_count"=>2401}, {"key"=>"incomplete", "doc_count"=>300}, {"key"=>"submit", "doc_count"=>136}, {"key"=>"OOM", "doc_count"=>23}, {"key"=>"post_run", "doc_count"=>1}]}}, {"key"=>"20.03-SP1", "doc_count"=>904, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>641}, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/lib/es_query.rb b/lib/es_query.rb index 88a420f..29d50e9 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -87,6 +87,50 @@ class ESQuery result end + + # input: + # fields, query_items + # + # fields => fields: Array(keyword) + # eg: + # ['suite', 'job_state'] + # + # query_items => query_items: Hash(keyword, value) + # eg: + # { + # 'os' => 'openeuler', + # ... + # } + # (optional for query_items, default no scope limitation) + # output: + # [{"key"=>"build-pkg", + # "doc_count"=>186175, + # "all_job_state"=> + # {"doc_count_error_upper_bound"=>0, + # "sum_other_doc_count"=>0, + # "buckets"=> + # [{"key"=>"failed", "doc_count"=>3830}, + # {"key"=>"finished", "doc_count"=>803}, + # {"key"=>"incomplete", "doc_count"=>196}, + # ... + def query_fields(fields, query_items = {}) + field1 = fields.first + aggs_hash = {} + build_aggs_from_fields(fields, aggs_hash) + query = { + query: { + bool: { + must: build_multi_field_subquery_body(query_items) + } + }, + aggs: aggs_hash['aggs'], + size: 0 + } + result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] + return nil if result.empty? + + result + end end # Range Query Example: @@ -146,3 +190,30 @@ def assign_desc_body(keyword) }] } end + +# input: +# fields = ['os', 'os_version', 'job_state'] +# aggs_hash = {} +# output: +# aggs_hash = { +# "aggs"=> +# {"all_os"=> +# {"terms"=>{:field=>"os", :size=>1000}, +# "aggs"=> +# {"all_os_version"=> +# {"terms"=>{:field=>"os_version", :size=>1000}, +# "aggs"=> +# {"all_job_state"=>{"terms"=>{:field=>"job_state", :size=>1000}}}}}}} +# } +def build_aggs_from_fields(fields, aggs_hash) + return if fields.empty? + + field = fields.shift + aggs_hash['aggs'] = { + "all_#{field}" => { + 'terms' => { field: field, size: 1000 } + } + } + + build_aggs_from_fields(fields, aggs_hash['aggs']["all_#{field}"]) +end -- 2.23.0

1 0

[PATCH v2 compass-ci] docker-rootfs/common: add comment for set password file.
by Wang Chenglong 04 Jan '21

04 Jan '21

[Why] add password file is easy to ignore. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container/docker-rootfs/common b/container/docker-rootfs/common index fe37da9..2ee16c8 100755 --- a/container/docker-rootfs/common +++ b/container/docker-rootfs/common @@ -33,7 +33,8 @@ usage() dst_rootfs_new_abspath: destination absolute path to create for rootfs Example: - ./run /tmp/openEuler-docker/openEuler-docker.${aarch}.tar.xz /tmp/openeuler-rootfs/ + ./run /tmp/openEuler-docker/openEuler-docker.${aarch}.tar.xz /tmp/openeuler-rootfs/ + # please ensure $HOME/.config/compass-ci/rootfs-passwd exists " } -- 2.23.0

3 4

[PATCH v3 compass-ci] docker-rootfs: add README.md
by Wang Chenglong 04 Jan '21

04 Jan '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 container/docker-rootfs/README.md diff --git a/container/docker-rootfs/README.md b/container/docker-rootfs/README.md new file mode 100644 index 0000000..1b6f42f --- /dev/null +++ b/container/docker-rootfs/README.md @@ -0,0 +1,18 @@ +## Use the tool to create a new nfsroot for os + +## Usage: + ./run <src_docker_file_abspath> <dst_rootfs_new_abspath> + + src_docker_file_abspath: source .tar.xz file absolute path with suffix: [ tar.xz ]. + dst_rootfs_new_abspath: destination absolute path to create for rootfs. + +## Example: + ./run /tmp/openEuler-docker/openEuler-docker.aarch64.tar.xz /tmp/openeuler-rootfs/ + # please ensure $HOME/.config/compass-ci/rootfs-passwd exists + +## Some configuration items: + ./packages-to-install + # If you want to pre-install the software, you can write the package names in ./packages-to-install. + + $HOME/.config/compass-ci/rootfs-passwd + # Set the password for the image into this file. -- 2.23.0

2 2

[PATCH v4 compass-ci] docker-rootfs: add README.md
by Wang Chenglong 04 Jan '21

04 Jan '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 container/docker-rootfs/README.md diff --git a/container/docker-rootfs/README.md b/container/docker-rootfs/README.md new file mode 100644 index 0000000..e7b5084 --- /dev/null +++ b/container/docker-rootfs/README.md @@ -0,0 +1,18 @@ +# Use the tool to create a new nfsroot for os + +## Usage: + ./run <src_docker_file_abspath> <dst_rootfs_new_abspath> + + src_docker_file_abspath: source .tar.xz file absolute path with suffix: [ tar.xz ]. + dst_rootfs_new_abspath: destination absolute path to create for rootfs. + +## Example: + ./run /tmp/openEuler-docker/openEuler-docker.aarch64.tar.xz /tmp/openeuler-rootfs/ + # Please ensure $HOME/.config/compass-ci/rootfs-passwd exists. + +## Some configuration items: + ./packages-to-install + # If you want to pre-install the software, you can write the package names in ./packages-to-install. + + $HOME/.config/compass-ci/rootfs-passwd + # Set the password for the image into this file. -- 2.23.0

1 0

[PATCH compass-ci 2/2] qcow2rootfs: fix rootfs-dir permission error
by Yu Chuan 03 Jan '21

03 Jan '21

[Error Msg] '01/03/21_04:25:49' Compressing file to /root/extract/rootfs-dir/qcow2-out.tar.gz Please wait until command returned... <r.gz compress:gzip numericowner:true xattrs:true selinux:true acls:true libguestfs: error: /root/extract/rootfs-dir/qcow2-out.tar.gz: Permission denied libguestfs: error: file receive cancelled by daemon Signed-off-by: Yu Chuan <13186087857(a)163.com> --- container/qcow2rootfs/bin/main | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/qcow2rootfs/bin/main b/container/qcow2rootfs/bin/main index b6ef7238f24a..c7527b794b89 100755 --- a/container/qcow2rootfs/bin/main +++ b/container/qcow2rootfs/bin/main @@ -34,7 +34,7 @@ unzip_qcow2_xz() { extract_rootfs() { echo "[WARNNING] Temporarily change owner of qcow2 file" chown "$EXTRACT_USER":"$EXTRACT_USER" "$QCOW2_FILE" - chmod 775 "$ROOTFS_DIR" + chmod 777 "$ROOTFS_DIR" echo "[$(date +'%D %T')] Converting .qcow2 to .tar.gz, please wait several minutes..." su - "$EXTRACT_USER" -c " -- 2.23.0

1 0

[PATCH compass-ci 1/2] qcow2rootfs: generate pkgs record file for result rootfs
by Yu Chuan 03 Jan '21

03 Jan '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- container/qcow2rootfs/bin/cci-generate-pkgs | 15 +++++++++++++++ container/qcow2rootfs/bin/main | 6 ++++++ 2 files changed, 21 insertions(+) create mode 100755 container/qcow2rootfs/bin/cci-generate-pkgs diff --git a/container/qcow2rootfs/bin/cci-generate-pkgs b/container/qcow2rootfs/bin/cci-generate-pkgs new file mode 100755 index 000000000000..e6a974cd0f41 --- /dev/null +++ b/container/qcow2rootfs/bin/cci-generate-pkgs @@ -0,0 +1,15 @@ +#!/bin/bash + +RESULT_FILE=/etc/pkgs-$(date "+%Y-%m-%d-%H-%M-%S").list + +command -v rpm >/dev/null && { + rpm -qa > $RESULT_FILE +} + +command -v dpkg >/dev/null && { + dpkg -l > $RESULT_FILE +} + +chmod 644 $RESULT_FILE + +exit 0 diff --git a/container/qcow2rootfs/bin/main b/container/qcow2rootfs/bin/main index b70ec7766649..b6ef7238f24a 100755 --- a/container/qcow2rootfs/bin/main +++ b/container/qcow2rootfs/bin/main @@ -74,6 +74,12 @@ pre_config_rootfs() { if [ -f ${ROOTFS_DIR}/etc/selinux/config ]; then sed -i -r 's/SELINUX=enforcing/SELINUX=disabled/g' "$ROOTFS_DIR/etc/selinux/config" fi + + # generate /etc/pkgs-$(date "+%Y-%m-%d-%H-%M-%S").list file for result rootfs + echo "generate /etc/pkgs-$(date "+%Y-%m-%d-%H-%M-%S").list" + cp $EXTRACT_ROOT/bin/cci-generate-pkgs $ROOTFS_DIR + chroot $ROOTFS_DIR /cci-generate-pkgs + rm /$ROOTFS_DIR/cci-generate-pkgs } clean_environment() { -- 2.23.0

1 0

[PATCH v2 lkp-tests] lib/bootstrap.sh: 'run-ipconfig' step should be before the check step
by Yu Chuan 03 Jan '21

03 Jan '21

[Why] The `run-ipconfig` step is the IP configuration step for initramfs, which should be executed immediately after checking the initramfs-tools file for existence. [Error Msg] + [ ! -e /usr/share/initramfs-tools/scripts/functions ] + get_net_devices + local net_devices + local i + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + [ p0s1/ != /sys/class/net/enp0s1/ ] + basename /sys/class/net/enp0s1/ + net_devices= enp0s1 + [ != /sys/class/net/lo/ ] + continue + echo enp0s1 + local net_devices= enp0s1 + [ -z ] + warn_no_eth0 + [ -f /proc/config.gz ] + return + echo ls /sys/class/net + ls /sys/class/net + is_virt + [ -n ] + [ -n ] + has_cmd virt-what + command -v virt-what + grep -q -w hypervisor /proc/cpuinfo + return 0 + export NO_NETWORK=1 + echo export NO_NETWORK=1 due to no net devices + return Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 7d15bb370fa9..770d550724f1 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -160,6 +160,9 @@ setup_network() return } + $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig + network_ok && return + local net_devices=$(get_net_devices) if [ -z "$net_devices" ]; then @@ -181,9 +184,6 @@ setup_network() fi fi - $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig - network_ok && return - local err_msg='IP-Config: Auto-configuration of network failed' dmesg | grep -q -F "$err_msg" || { # Include $err_msg in the error message so that it matches -- 2.23.0

1 0

[PATCH lkp-tests] lib/bootstrap.sh: 'run-ipconfig' step should be before the check step
by Yu Chuan 03 Jan '21

03 Jan '21

[Before] test job: result/borrow/2020-12-31/vm-2p8g/debian-sid-aarch64/3600/z9.275666 key log: + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + cat /sys/class/net/enp0s1//operstate + [ down = up ] [After] test job: result/borrow/2020-12-31/vm-2p8g/debian-sid-aarch64/3600/z9.275673 key log: + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + cat /sys/class/net/enp0s1//operstate + [ up = up ] Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 7d15bb370fa9..770d550724f1 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -160,6 +160,9 @@ setup_network() return } + $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig + network_ok && return + local net_devices=$(get_net_devices) if [ -z "$net_devices" ]; then @@ -181,9 +184,6 @@ setup_network() fi fi - $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig - network_ok && return - local err_msg='IP-Config: Auto-configuration of network failed' dmesg | grep -q -F "$err_msg" || { # Include $err_msg in the error message so that it matches -- 2.23.0

2 2