- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci 2/2] compare: remove not latest failed metrics
by Lu Weitao 06 Jan '21

06 Jan '21

[Why] User just need the failed metrics belong latest job in compare result [How] 1. query job_list and order by start_time desc 2. auto group job_list - the first job of each dimension is the latest job { group_key => { dim1 => [job1, ...] ... } } 3. get latest job_list for each group 4. in compare result, ignore the failed metric that not belongs the latest job_list Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/compare.rb | 4 ++-- lib/compare_matrixes.rb | 32 ++++++++++++++++++++++++-------- lib/matrix2.rb | 24 +++++++++++++++++------- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/lib/compare.rb b/lib/compare.rb index 6a2f515..21e24df 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -62,8 +62,8 @@ end def compare_group(argv, dimensions, options) conditions = parse_conditions(argv) dims = dimensions.split(' ') - groups_matrices, suites_hash = create_groups_matrices_list(conditions, dims) - compare_group_matrices(groups_matrices, suites_hash, options) + groups_matrices, suites_hash, latest_jobs_hash = create_groups_matrices_list(conditions, dims) + compare_group_matrices(groups_matrices, suites_hash, latest_jobs_hash, options) end def create_groups_matrices_list(conditions, dims) diff --git a/lib/compare_matrixes.rb b/lib/compare_matrixes.rb index 21bd5c9..7747ea1 100644 --- a/lib/compare_matrixes.rb +++ b/lib/compare_matrixes.rb @@ -159,7 +159,7 @@ def get_matrixes_fields(matrixes_list) matrixes_fields end -def get_matrixes_values(matrixes_list, options) +def get_matrixes_values(matrixes_list, latest_jobs, options) # get all matrixes all field values # matrixes_values = { false => {}, true => {} } @@ -168,6 +168,10 @@ def get_matrixes_values(matrixes_list, options) next if field == 'stats_source' success = successful?(field) + unless success + next unless latest_failure?(field, latest_jobs) + end + matrixes_values[success][field] = get_values_by_field( matrixes_list, field, matrixes_size, success, options @@ -176,6 +180,18 @@ def get_matrixes_values(matrixes_list, options) matrixes_values end +def latest_failure?(field, latest_jobs) + flag = false + latest_jobs.each do |job| + if job['stats'][field] + flag = true + break + end + end + + flag +end + def remove_unchanged_field(matrixes_values, suite_list) # remove unchanged field from matrixes values and remove :changed key # @@ -197,7 +213,7 @@ def matrixes_empty?(matrixes_list) return matrixes_list.any?(&:empty?) end -def compare_matrixes(matrixes_list, suite_list, matrixes_titles = nil, group_key = nil, options: {}) +def compare_matrixes(matrixes_list, suite_list, latest_jobs, matrixes_titles = nil, group_key = nil, options: {}) # compare matrix in matrixes_list and print info # @matrixes_list: list consisting of matrix # @matrixes_titles: number or dimension of matrix @@ -206,7 +222,7 @@ def compare_matrixes(matrixes_list, suite_list, matrixes_titles = nil, group_key return warn('Matrix cannot be empty!') || '' if matrixes_empty?(matrixes_list) options = { 'perf-profile': 5, theme: :none, no_print: false }.merge(options) - matrixes_values = get_matrixes_values(matrixes_list, options) + matrixes_values = get_matrixes_values(matrixes_list, latest_jobs, options) remove_unchanged_field(matrixes_values, suite_list) if matrixes_list.length > 1 no_print = options[:no_print] result_str = group_key ? "\n\n\n\n\n" + group_key : '' @@ -387,28 +403,28 @@ end # input: group matrices # output: pre compare result of each group # the result with more comparison objects first -def compare_group_matrices(group_matrices, suites_hash, options) +def compare_group_matrices(group_matrices, suites_hash, latest_jobs_hash, options) result_str = '' group_matrices_array = sort_by_matrix_size(group_matrices) have_multi_member = group_matrices_array[0][1].size > 1 group_matrices_array.each do |matrice_kv| next if have_multi_member && matrice_kv[1].size < 2 - result_str += get_matrix_str(matrice_kv[0], matrice_kv[1], suites_hash[matrice_kv[0]], options) + result_str += get_matrix_str(matrice_kv[0], matrice_kv[1], suites_hash[matrice_kv[0]], latest_jobs_hash[matrice_kv[0]], options) end result_str end -def get_matrix_str(matrice_key, matrice_value, suite_list, options) +def get_matrix_str(matrice_key, matrice_value, suite_list, latest_jobs, options) m_list = [] m_titles = [] matrice_value.each do |dim, matrix| m_titles << dim m_list << matrix end - return compare_matrixes(m_list, suite_list, m_titles, matrice_key, options: options) if options[:no_print] + return compare_matrixes(m_list, suite_list, latest_jobs, m_titles, matrice_key, options: options) if options[:no_print] - print compare_matrixes(m_list, suite_list, m_titles, matrice_key, options: options) + print compare_matrixes(m_list, suite_list, latest_jobs, m_titles, matrice_key, options: options) return '' end diff --git a/lib/matrix2.rb b/lib/matrix2.rb index 09573ec..e3ae580 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -115,27 +115,37 @@ end # } def combine_group_query_data(job_list, dims) suites_hash = {} + latest_jobs_hash = {} groups = auto_group(job_list, dims) have_multi_member = multi_member?(groups) one_size_count = 0 groups.each do |group_key, value| if value.size < 2 one_size_count += 1 - if have_multi_member || one_size_count > 3 - groups.delete(group_key) - next - end + next if delete_group?(groups, group_key, have_multi_member, one_size_count) end - get_groups_matrix(groups, group_key, value, suites_hash) + get_groups_matrix(groups, group_key, value, suites_hash, latest_jobs_hash) end - return groups, suites_hash + return groups, suites_hash, latest_jobs_hash end -def get_groups_matrix(groups, group_key, value, suites_hash) +def delete_group?(groups, group_key, have_multi_member, one_size_count) + was_deleted = false + if have_multi_member || one_size_count > 3 + groups.delete(group_key) + was_deleted = true + end + + was_deleted +end + +def get_groups_matrix(groups, group_key, value, suites_hash, latest_jobs_hash) suite_list = [] + latest_jobs_hash[group_key] = [] value.each do |dimension_key, jobs| groups[group_key][dimension_key], suites = create_matrix(jobs) suite_list.concat(suites) + latest_jobs_hash[group_key] << jobs[0] end suites_hash[group_key] = suite_list end -- 2.23.0

1 0

[PATCH compass-ci 1/2] lib/compare.rb: order query result with desc
by Lu Weitao 06 Jan '21

06 Jan '21

[Why] 1. When query result include more than 10,000 jobs, we still need the latest jobs use to compare 2. Support we recognize the latest job that have failure messages from each group-dimension-job_list Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/compare.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/compare.rb b/lib/compare.rb index 9090c3b..6a2f515 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -40,7 +40,7 @@ def create_matrices_list(conditions) suite_list = [] es = ESQuery.new(ES_HOST, ES_PORT) conditions.each do |condition| - query_results = es.multi_field_query(condition) + query_results = es.multi_field_query(condition, desc_keyword: 'start_time') matrix, suites = combine_query_data(query_results) matrices_list << matrix suite_list.concat(suites) @@ -68,7 +68,7 @@ end def create_groups_matrices_list(conditions, dims) es = ESQuery.new(ES_HOST, ES_PORT) - query_results = es.multi_field_query(conditions) + query_results = es.multi_field_query(conditions, desc_keyword: 'start_time') combine_group_query_data(query_results['hits']['hits'], dims) end -- 2.23.0

1 0

[PATCH compass-ci] git_mirror.rb: git clone only one commit at first
by Li Yuanchao 05 Jan '21

05 Jan '21

Some repositories have many commits which make it big, and cause overtime when git clone. Clone only one commit at first, and pull all commits later, can help clone large repositories. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 156d5bb..7f265d9 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -42,7 +42,7 @@ class GitMirror ret = false url = get_url(Array(url)[0]) 10.times do - stderr = %x(git clone --mirror #{url} #{mirror_dir} 2>&1) + stderr = %x(git clone --mirror --depth 1 #{url} #{mirror_dir} 2>&1) ret = !stderr.include?('fatal') break if ret end @@ -51,6 +51,11 @@ class GitMirror end def git_fetch(mirror_dir) + if File.exist?("#{mirror_dir}/shallow") + %x(git -C #{mirror_dir} fetch --unshallow 2>&1) + return false + end + fetch_info = %x(git -C #{mirror_dir} fetch 2>&1) # Check whether mirror_dir is a good git repository by 2 conditions. If not, delete it. if fetch_info.include?(ERR_MESSAGE) && Dir.empty?(mirror_dir) -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] sbin/run_job: registering user when submit-job
by Hu Xuejiao 05 Jan '21

05 Jan '21

Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sbin/run_job | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbin/run_job b/sbin/run_job index c84df0d..325fa9f 100755 --- a/sbin/run_job +++ b/sbin/run_job @@ -1,5 +1,8 @@ #!/bin/bash +# register user information +ruby build-my-info.rb -e "$my_email" -n "$USER" -l "$lab" + cci_makepkg_file="$LKP_SRC/jobs/cci-makepkg.yaml" grep "atomic" $cci_makepkg_file || sed -i 's|benchmark:|& atomic|g' $cci_makepkg_file -- 2.23.0

1 0

[PATCH v4 compass-ci] docker-rootfs: add README.md
by Wang Chenglong 05 Jan '21

05 Jan '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 container/docker-rootfs/README.md diff --git a/container/docker-rootfs/README.md b/container/docker-rootfs/README.md new file mode 100644 index 0000000..e7b5084 --- /dev/null +++ b/container/docker-rootfs/README.md @@ -0,0 +1,18 @@ +# Use the tool to create a new nfsroot for os + +## Usage: + ./run <src_docker_file_abspath> <dst_rootfs_new_abspath> + + src_docker_file_abspath: source .tar.xz file absolute path with suffix: [ tar.xz ]. + dst_rootfs_new_abspath: destination absolute path to create for rootfs. + +## Example: + ./run /tmp/openEuler-docker/openEuler-docker.aarch64.tar.xz /tmp/openeuler-rootfs/ + # Please ensure $HOME/.config/compass-ci/rootfs-passwd exists. + +## Some configuration items: + ./packages-to-install + # If you want to pre-install the software, you can write the package names in ./packages-to-install. + + $HOME/.config/compass-ci/rootfs-passwd + # Set the password for the image into this file. -- 2.23.0

1 0

[PATCH v2 compass-ci] docker-rootfs/run: fix the incorrect image names
by Wang Chenglong 05 Jan '21

05 Jan '21

[Why] When docker's image name contains multilevel addresses, it needs to intercept again. image name: docker.io/openeuler20.03-LTS package name: openeuler20.03-LTS.img Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container/docker-rootfs/run b/container/docker-rootfs/run index cf91337..ce51ba3 100755 --- a/container/docker-rootfs/run +++ b/container/docker-rootfs/run @@ -11,7 +11,8 @@ check_cmd_input "$@" check_passwd_file "$root_pwd_file" load_docker_img "$1" echo $docker_name_tag -IMAGE_PACK=$(echo ${docker_name_tag%%:*}.cgz) +IMAGE_NAME=${docker_name_tag%%:*} +IMAGE_PACK=${IMAGE_NAME##*/}.cgz start=( docker run -- 2.23.0

1 0

[PATCH compass-ci] docker-rootfs/run: fix the incorrect image names
by Wang Chenglong 05 Jan '21

05 Jan '21

[Why] When docker's image name contains multilevel addresses, it needs to intercept again. image name: docker.io/openeuler20.03-LTS package name: openeuler20.03-LTS.img Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container/docker-rootfs/run b/container/docker-rootfs/run index cf91337..8a7b315 100755 --- a/container/docker-rootfs/run +++ b/container/docker-rootfs/run @@ -11,7 +11,8 @@ check_cmd_input "$@" check_passwd_file "$root_pwd_file" load_docker_img "$1" echo $docker_name_tag -IMAGE_PACK=$(echo ${docker_name_tag%%:*}.cgz) +IMAGE_NAME=${docker_name_tag%%:*} +IMAGE_PACK=$(echo ${IMAGE_NAME##*/}.cgz) start=( docker run -- 2.23.0

2 2

[PATCH v5 compass-ci 1/2] lib/es_query.rb: get multi-field aggregation count result
by Lu Kaiyi 05 Jan '21

05 Jan '21

[why] get multi-field aggregation count for overview of report page. [input]: fields, query_items (optional for query_items, default no scope limitation) [example] input: fields = ['suite'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186214}, {"key"=>"cci-depends", "doc_count"=>1230}, {"key"=>"iperf", "doc_count"=>1103}, {"key"=>"spinlock", "doc_count"=>1000}, {"key"=>"cci-makepkg", "doc_count"=>949}, {"key"=>"iperf-walk-os-test", "doc_count"=>474}, {"key"=>"bisect", "doc_count"=>381}, ... input: fields = ['suite', 'job_state'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186227, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3830}, {"key"=>"finished", "doc_count"=>803}, {"key"=>"incomplete", "doc_count"=>196}, {"key"=>"submit", "doc_count"=>136}, ... input: fields = ['os', 'os_version', 'job_state'] query_items = {} output: [{"key"=>"openeuler", "doc_count"=>193287, "all_os_version"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"20.03", "doc_count"=>191789, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3909}, {"key"=>"finished", "doc_count"=>2401}, {"key"=>"incomplete", "doc_count"=>300}, {"key"=>"submit", "doc_count"=>136}, {"key"=>"OOM", "doc_count"=>23}, {"key"=>"post_run", "doc_count"=>1}]}}, {"key"=>"20.03-SP1", "doc_count"=>904, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>641}, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 70 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/lib/es_query.rb b/lib/es_query.rb index 88a420f..91aa6ab 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -87,6 +87,49 @@ class ESQuery result end + + # input: + # fields, query_items + # + # fields => fields: Array(keyword) + # eg: + # ['suite', 'job_state'] + # + # query_items => query_items: Hash(keyword, value) + # eg: + # { + # 'os' => 'openeuler', + # ... + # } + # (optional for query_items, default no scope limitation) + # output: + # [{"key"=>"build-pkg", + # "doc_count"=>186175, + # "all_job_state"=> + # {"doc_count_error_upper_bound"=>0, + # "sum_other_doc_count"=>0, + # "buckets"=> + # [{"key"=>"failed", "doc_count"=>3830}, + # {"key"=>"finished", "doc_count"=>803}, + # {"key"=>"incomplete", "doc_count"=>196}, + # ... + def query_fields(fields, query_items = {}) + field1 = fields.first + aggs_hash = build_aggs_from_fields(fields) + query = { + query: { + bool: { + must: build_multi_field_subquery_body(query_items) + } + }, + aggs: aggs_hash['aggs'], + size: 0 + } + result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] + return nil if result.empty? + + result + end end # Range Query Example: @@ -146,3 +189,30 @@ def assign_desc_body(keyword) }] } end + +# input: +# fields = ['os', 'os_version', 'job_state'] +# output: +# aggs_hash = { +# "aggs"=> +# {"all_os"=> +# {"terms"=>{:field=>"os", :size=>1000}, +# "aggs"=> +# {"all_os_version"=> +# {"terms"=>{:field=>"os_version", :size=>1000}, +# "aggs"=> +# {"all_job_state"=>{"terms"=>{:field=>"job_state", :size=>1000}}}}}}} +# } +def build_aggs_from_fields(fields) + aggs_hash = {} + return if fields.empty? + + field = fields.shift + aggs_hash['aggs'] ||= {} + aggs_hash['aggs']["all_#{field}"] = { + 'terms' => { field: field, size: 1000 } + } + sub_aggs = build_aggs_from_fields(fields) + aggs_hash['aggs']["all_#{field}"].merge!(sub_aggs) if sub_aggs + aggs_hash +end -- 2.23.0

2 1

[PATCH lkp-tests] lib/bootstrap.sh: fix the problem caused by the lack of double quotes
by Yu Chuan 05 Jan '21

05 Jan '21

[Why] In dash, we must add double quotes when calling a method using `$()`. + local ycb= asf + echo ycb: : ycb: : The lack of double quotes in our current code leads to a wrong judgement. [Error Msg] + local net_devices= enp0s1 + [ -z ] + warn_no_eth0 Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 94b94c28e9a8..30e4af828bcd 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -73,7 +73,7 @@ get_net_devices() net_devices_link() { local operation=$1 - local net_devices=$(get_net_devices) + local net_devices="$(get_net_devices)" local ndev for ndev in $net_devices do @@ -163,7 +163,7 @@ setup_network() $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig network_ok && return - local net_devices=$(get_net_devices) + local net_devices="$(get_net_devices)" if [ -z "$net_devices" ]; then warn_no_eth0 -- 2.23.0

1 0

[PATCH compass-ci 1/2] sbin/run_job: registering user when submit-job
by Hu Xuejiao 05 Jan '21

05 Jan '21

Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sbin/run_job | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sbin/run_job b/sbin/run_job index c84df0d..690e5d6 100755 --- a/sbin/run_job +++ b/sbin/run_job @@ -1,5 +1,8 @@ #!/bin/bash +# register user information +ruby build-my-info.rb $my_email $USER $lab + cci_makepkg_file="$LKP_SRC/jobs/cci-makepkg.yaml" grep "atomic" $cci_makepkg_file || sed -i 's|benchmark:|& atomic|g' $cci_makepkg_file -- 2.23.0

2 1

[PATCH compass-ci 2/2] sbin/build-my-info.rb: modify the method of inputting variables
by Hu Xuejiao 05 Jan '21

05 Jan '21

Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sbin/build-my-info.rb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/sbin/build-my-info.rb b/sbin/build-my-info.rb index 40193af..197425c 100755 --- a/sbin/build-my-info.rb +++ b/sbin/build-my-info.rb @@ -3,16 +3,12 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true -require 'io/console' require_relative '../lib/es_client' require_relative '../lib/build_my_info_client' -print 'email: ' -my_email = $stdin.echo = gets.chomp -print 'name: ' -my_name = $stdin.echo = gets.chomp -print 'lab: ' -lab = $stdin.echo = gets.chomp +my_email = ARGV[0] +my_name = ARGV[1] +lab = ARGV[2] build_my_info = BuildMyInfo.new(my_email, my_name, lab) build_my_info.config_my_info -- 2.23.0

2 1

[PATCH v4 compass-ci] docker-rootfs/common: add comment for set password file.
by Wang Chenglong 05 Jan '21

05 Jan '21

[Why] add password file is easy to ignore. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container/docker-rootfs/common b/container/docker-rootfs/common index fe37da9..e908079 100755 --- a/container/docker-rootfs/common +++ b/container/docker-rootfs/common @@ -33,7 +33,8 @@ usage() dst_rootfs_new_abspath: destination absolute path to create for rootfs Example: - ./run /tmp/openEuler-docker/openEuler-docker.${aarch}.tar.xz /tmp/openeuler-rootfs/ + ./run /tmp/openEuler-docker/openEuler-docker.${aarch}.tar.xz /tmp/openeuler-rootfs/ + # Please ensure $HOME/.config/compass-ci/rootfs-passwd exists. " } -- 2.23.0

2 1

[PATCH v2 lkp-tests] tests: add httpress test script
by Wei Jihui 05 Jan '21

05 Jan '21

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- tests/httpress | 138 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100755 tests/httpress diff --git a/tests/httpress b/tests/httpress new file mode 100755 index 00000000..109ba7f9 --- /dev/null +++ b/tests/httpress @@ -0,0 +1,138 @@ +#!/bin/sh +# - nic +# - nic_ip +# - request_ip +# - request_port +# - request_index +# - core_nums +# - protocol +# - alive_type + +. $LKP_SRC/lib/env.sh + +: "${request_index:=index.html}" +: "${core_nums:=4}" +: "${protocol:=http}" +: "${alive_type:=short}" + + +core_range=0-15 + +http_long_requests() +{ + if [ "$core_nums" == 4 ]; then + requests_num=3000000 + threads_num=100 + concurrent_num=100 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 8 ]; then + requests_num=5000000 + threads_num=300 + concurrent_num=300 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=50000000 + threads_num=1200 + concurrent_num=1200 + ethtool -L $nic combined 16 && /lkp/lkp/src/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=50000000 + threads_num=400 + concurrent_num=400 + ethtool -L $nic combined 16 && /lkp/lkp/src/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + fi +} + + +http_short_requests() +{ + if [ "$core_nums" == 4 ]; then + requests_num=3000000 + threads_num=300 + concurrent_num=300 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 8 ]; then + requests_num=5000000 + threads_num=400 + concurrent_num=400 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=10000000 + threads_num=300 + concurrent_num=300 + ethtool -L $nic combined 16 && $LKP_SRC/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=10000000 + threads_num=600 + concurrent_num=600 + ethtool -L $nic combined 16 && $LKP_SRC/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + fi +} + +https_long_requests() +{ + if [ "$core_nums" == 4 ] || [ "$core_nums" == 8 ]; then + requests_num=5000000 + threads_num=100 + concurrent_num=100 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=5000000 + threads_num=600 + concurrent_num=600 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=10000000 + threads_num=600 + concurrent_num=600 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + fi +} + +https_short_requests() +{ + if [ "$core_nums" == 4 ] || [ "$core_nums" == 8 ]; then + requests_num=500000 + threads_num=100 + concurrent_num=100 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=500000 + threads_num=600 + concurrent_num=600 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=1000000 + threads_num=1200 + concurrent_num=1200 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + fi +} + +main() +{ + local request_url="$protocol://$request_ip:$request_port/$request_index" + + ip addr add $nic_ip/24 dev "$nic" + ip link set "$nic" up + + if [ "$protocol" == 'http' ]; then + if [ "$alive_type" == 'long' ]; then + http_long_requests + else + http_short_requests + fi + else + if [ "$alive_type" == 'long' ]; then + https_long_requests + else + https_short_requests + fi + fi +} + +main -- 2.23.0

2 2

[PATCH compass-ci] service/monitoring: print json logs
by Wu Zhende 05 Jan '21

05 Jan '21

Replace "puts" with JSONLogger. It's easier to query and locate faults. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/monitoring.cr | 12 +++++++++++- src/monitoring/amqp.cr | 6 ++++-- src/monitoring/filter.cr | 4 +++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/monitoring.cr b/src/monitoring.cr index b1b534b..0f88be4 100644 --- a/src/monitoring.cr +++ b/src/monitoring.cr @@ -1,6 +1,16 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +require "./lib/json_logger" require "monitoring/monitoring" require "monitoring/constants" module Monitoring - Kemal.run(MONITOR_PORT) + log = JSONLogger.new + + begin + Kemal.run(MONITOR_PORT) + rescue e + log.error(e) + end end diff --git a/src/monitoring/amqp.cr b/src/monitoring/amqp.cr index b0341ce..27ad3d2 100644 --- a/src/monitoring/amqp.cr +++ b/src/monitoring/amqp.cr @@ -3,10 +3,12 @@ require "amqp-client" require "./monitoring" require "./filter" require "./constants" +require "../lib/json_logger" class MessageQueueClient def initialize(host = MQ_HOST, port = MQ_PORT) @client = AMQP::Client.new("amqp://#{host}:#{port}") + @log = JSONLogger.new end private def start @@ -24,7 +26,7 @@ class MessageQueueClient begin filter.filter_msg(msg.body_io) rescue e - puts "filter message error: #{e}" + @log.warn("filter message failed: #{e}") end end end @@ -37,7 +39,7 @@ class MessageQueueClient filter_msg(conn, filter, exchange_name, queue_name) end rescue e - puts "monitoring_message_queue error: #{e}" + @log.warn("monitoring_message_queue failed: #{e}") sleep 5 end end diff --git a/src/monitoring/filter.cr b/src/monitoring/filter.cr index 79af3ab..2bd1d1a 100644 --- a/src/monitoring/filter.cr +++ b/src/monitoring/filter.cr @@ -2,6 +2,7 @@ require "set" require "json" require "./parse_serial_logs" +require "../lib/json_logger" class Filter def initialize @@ -9,6 +10,7 @@ class Filter # like {query => [socket1, socket2]} @hash = Hash(JSON::Any, Array(HTTP::WebSocket)).new @sp = SerialParser.new + @log = JSONLogger.new end def add_filter_rule(query : JSON::Any, socket : HTTP::WebSocket) @@ -42,7 +44,7 @@ class Filter @hash[query].each do |socket| socket.send msg.to_json rescue e - puts "send msg failed: #{e}" + @log.warn("send msg failed: #{e}") remove_filter_rule(query, socket) end end -- 2.23.0

1 0

[PATCH v2 compass-ci 1/7] send-internet-mail/Dockerfile: add install gem resources
by Luan Shengde 05 Jan '21

05 Jan '21

[why] the service need to install redis to store email counts to redis queue the service need to install rufus-scheduler to reset the queue as a timing task Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/send-internet-mail/Dockerfile b/container/send-internet-mail/Dockerfile index d151bda..03e3327 100644 --- a/container/send-internet-mail/Dockerfile +++ b/container/send-internet-mail/Dockerfile @@ -6,4 +6,4 @@ MAINTAINER shdluan(a)163.com ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && \ - apt-get install -y ruby-mail ruby-json ruby-sinatra + apt-get install -y ruby-mail ruby-json ruby-sinatra ruby-redis ruby-rufus-scheduler -- 2.23.0

2 2

[PATCH v2 compass-ci 2/7] send-internet-mail: optimize
by Luan Shengde 05 Jan '21

05 Jan '21

add init redis host/port add invoke code files Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/send-internet-mail.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/container/send-internet-mail/send-internet-mail.rb b/container/send-internet-mail/send-internet-mail.rb index 8fdaa0d..90571f8 100755 --- a/container/send-internet-mail/send-internet-mail.rb +++ b/container/send-internet-mail/send-internet-mail.rb @@ -6,7 +6,13 @@ require 'mail' require 'sinatra' -require "#{ENV['CCI_SRC']}/lib/mail-post" +REDIS_HOST = %x(/sbin/ip route |awk '/default/ {print $3}').chomp +REDIS_PORT = ENV['REDIS_PORT'] + +require_relative 'email_init' +require_relative 'email_limit_queue' +require_relative 'email_mapping' +require_relative '../../lib/mail-post' smtp = { address: 'smtp.qq.com', -- 2.23.0

2 2

[PATCH v3 compass-ci] providers/qemu: adapt for x86_64 and riscv64
by Liu Yinsi 05 Jan '21

05 Jan '21

before: support kvm for aarch64 after: support kvm for aarch64, x86_64, riscv64 [how] put public options like $kernel, $initrd, $smp together, list different options like $machine, $bios, $nic according to different qemu command, then merge all options. test: /srv/result/host-info/2021-01-04/vm-2p8g/debian-sid-aarch64/z9.280302 /srv/result/host-info/2021-01-04/vm-2p8g/debian-sid-riscv64/z9.280305 /srv/result/host-info/2021-01-04/vm-2p8g/debian-sid-x86_64/z9.280303 Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 106 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 93 insertions(+), 13 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index c926d3d..8749568 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -18,12 +18,6 @@ check_logfile() } } -qemu_command() -{ - qemu=qemu-system-aarch64 - command -v $qemu >/dev/null || qemu=qemu-kvm -} - write_logfile() { ipxe_script=ipxe_script @@ -72,6 +66,50 @@ get_initrd() cat $initrds > $initrd } +get_bios() +{ + bios=/usr/share/qemu-efi-aarch64/QEMU_EFI.fd + [ -f "$bios" ] || bios=/usr/share/ovmf/OVMF.fd +} + +get_helper() +{ + helper=/usr/libexec/qemu-bridge-helper + [ -f "$helper" ] || helper=/usr/lib/qemu/qemu-bridge-helper +} + +get_nic() +{ + nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" +} + +get_device() +{ + device="virtio-net-device,netdev=net0,mac=${mac}" +} + +get_netdev() +{ + netdev="bridge,br=br0,id=net0,helper=${helper}" +} + +get_qemu() +{ + qemu=( + qemu-system-aarch64 + qemu-kvm + qemu-system-x86_64 + ) + + for qemu in "${qemu[@]}" + do + [ -n "$(command -v ${qemu})" ] && break + done + + # debian has both qemu-system-x86_64 and qemu-system-riscv64 command + [[ $kernel =~ 'riscv64' ]] && qemu=qemu-system-riscv64 +} + print_message() { echo $SCHED_PORT @@ -83,36 +121,78 @@ print_message() sleep 5 } -run_qemu() +public_option() { kvm=( $qemu - -machine virt-4.0,accel=kvm,gic-version=3 -kernel $kernel -initrd $initrd -smp $nr_cpu -m $memory - -cpu Kunpeng-920 - -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd - -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} -k en-us -no-reboot -nographic -serial file:${log_file} -monitor null ) +} - "${kvm[@]}" --append "${append}" +run_qemu() +{ + case "$qemu" in + qemu-system-aarch64) + arch_option=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) + ;; + qemu-kvm) + [ "$(arch)" == "aarch64" ] && arch_option=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) + [ "$(arch)" == "x86_64" ] && arch_option=( + -bios $bios + -nic $nic + ) + ;; + qemu-system-x86_64) + arch_option=( + -bios $bios + -nic $nic + ) + ;; + qemu-system-riscv64) + arch_option=( + -machine virt + -device virtio-net-device,netdev=net0,mac=${mac} + -netdev bridge,br=br0,id=net0,helper=${helper} + ) + ;; + esac + + "${kvm[@]}" "${arch_option[@]}" --append "${append}" } check_logfile write_logfile -qemu_command parse_ipxe_script check_option_value + get_initrd +get_bios +get_helper +get_nic +get_device +get_netdev +get_qemu print_message +public_option run_qemu -- 2.23.0

2 2

[PATCH compass-ci] provicers/docker: copy rootfs/addon/usr/* to container
by Xiao Shenwei 05 Jan '21

05 Jan '21

[why] under LKP_SRC/addon/usr we have some customized tools such as: /usr/bin/my_curl, these resource should be used by container Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- providers/docker/bin/entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/providers/docker/bin/entrypoint.sh b/providers/docker/bin/entrypoint.sh index feb5edf..4388e19 100755 --- a/providers/docker/bin/entrypoint.sh +++ b/providers/docker/bin/entrypoint.sh @@ -4,4 +4,5 @@ mkdir -p /root/.ssh cp /lkp/lkp/src/rootfs/addon/root/.ssh/id_rsa /root/.ssh/ +cp -r /lkp/lkp/src/rootfs/addon/usr/* /usr sh /lkp/lkp/src/rootfs/addon/etc/init.d/lkp-bootstrap -- 2.23.0

1 0

[PATCH compass-ci] providers/docker: read only mount of host resources
by Xiao Shenwei 05 Jan '21

05 Jan '21

Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- providers/docker/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/docker/run.sh b/providers/docker/run.sh index 127cf5b..62a1f9f 100755 --- a/providers/docker/run.sh +++ b/providers/docker/run.sh @@ -27,7 +27,7 @@ cmd=( -e CCI_SRC=/c/compass-ci -v ${load_path}/lkp:/lkp -v ${DIR}/bin:/root/bin:ro - -v $CCI_SRC:/c/compass-ci + -v $CCI_SRC:/c/compass-ci:ro -v /srv/git:/srv/git:ro -v /srv/result:/srv/result:ro -v ${busybox_path}:/usr/local/bin/busybox -- 2.23.0

1 0

[PATCH v2 compass-ci] 3-code/git: modify the git download path
by Hu Xuejiao 05 Jan '21

05 Jan '21

[why] if git path is not specified, lkp-tests will be downloaded in /c/compass-ci Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sparrow/3-code/git | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sparrow/3-code/git b/sparrow/3-code/git index 48741ec..85db0de 100755 --- a/sparrow/3-code/git +++ b/sparrow/3-code/git @@ -9,10 +9,10 @@ umask 002 cd /c || exit -# git clone https://gitee.com/openeuler/compass-ci.git +# git clone compass-ci,lkp-tests,upstream-repos # modify and manual run for now: [ -d "compass-ci/.git" ] || { - git clone https://gitee.com/wu_fengguang/compass-ci.git + git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci cd /c/compass-ci || exit [ -n "$compass_commit_id" ] && git reset --hard "$compass_commit_id" @@ -21,14 +21,14 @@ cd /c || exit } [ -d "lkp-tests/.git" ] || { - git clone https://gitee.com/wu_fengguang/lkp-tests.git + git clone https://gitee.com/wu_fengguang/lkp-tests.git /c/lkp-tests cd /c/lkp-tests || exit [ -n "$lkp_commit_id" ] && git reset --hard "$lkp_commit_id" } [ -d "upstream-repos/.git" ] || { - git clone https://gitee.com/wu_fengguang/upstream-repos.git + git clone https://gitee.com/wu_fengguang/upstream-repos.git /c/upstream-repos cd /c/upstream-repos || exit [ -n "$upstream_commit_id" ] && git reset --hard "$upstream_commit_id" -- 2.23.0

1 0

[PATCH v5 compass-ci 2/2] lib/es_query.rb: parse es_result to a Hash result
by Lu Kaiyi 05 Jan '21

05 Jan '21

[why] make es result more easy access. [example] input: es_result = [{"key"=>"build-pkg", "doc_count"=>354526, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>5708}, {"key"=>"finished", "doc_count"=>1033}, {"key"=>"incomplete", "doc_count"=>204}, {"key"=>"submit", "doc_count"=>136}, ... output: result_hash = {"build-pkg"=> {"failed"=>5708, "finished"=>1033, "incomplete"=>204, "submit"=>136, "OOM"=>11, "post_run"=>2}, "cci-depends"=> {"finished"=>1785, "failed"=>675, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 91aa6ab..207aaae 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -128,7 +128,7 @@ class ESQuery result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] return nil if result.empty? - result + parse_fields(result) end end @@ -216,3 +216,47 @@ def build_aggs_from_fields(fields) aggs_hash['aggs']["all_#{field}"].merge!(sub_aggs) if sub_aggs aggs_hash end + +# input: +# es_result = +# [{"key"=>"build-pkg", +# "doc_count"=>354526, +# "all_job_state"=> +# {"doc_count_error_upper_bound"=>0, +# "sum_other_doc_count"=>0, +# "buckets"=> +# [{"key"=>"failed", "doc_count"=>5708}, +# {"key"=>"finished", "doc_count"=>1033}, +# {"key"=>"incomplete", "doc_count"=>204}, +# {"key"=>"submit", "doc_count"=>136}, +# ... +# +# output: +# result_hash = +# {"build-pkg"=> +# {"failed"=>5708, +# "finished"=>1033, +# "incomplete"=>204, +# "submit"=>136, +# "OOM"=>11, +# "post_run"=>2}, +# "cci-depends"=> +# {"finished"=>1785, +# "failed"=>675, +# ... +def parse_fields(es_result) + result_hash ||= {} + es_result.each do |result| + key = result['key'] + sub_field = result.keys.detect { |field| field.start_with?('all_') } + + if sub_field + all_field = result[sub_field]['buckets'] + result_hash[key] = parse_fields(all_field) + else + result_hash[key] = result['doc_count'] + end + end + + result_hash +end -- 2.23.0

1 0

[PATCH v2 compass-ci 7/7] send-internet-mail/start: add key REDIS_PORT
by Luan Shengde 05 Jan '21

05 Jan '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/send-internet-mail/start b/container/send-internet-mail/start index 02c6ab6..136435d 100755 --- a/container/send-internet-mail/start +++ b/container/send-internet-mail/start @@ -13,6 +13,7 @@ names = Set.new %w[ ROBOT_EMAIL_PASSWORD SEND_MAIL_PORT SENT_MAILDIR + REDIS_PORT ] defaults = relevant_defaults(names) -- 2.23.0

1 0

[PATCH v2 compass-ci 6/7] lib/mail-post: add email check
by Luan Shengde 05 Jan '21

05 Jan '21

add check email count only email addresses that email count within the limited value can be sent mail add check email mapping if has set email mapping for an email address. the email address will be converted to the mapping value Signed-off-by: Luan Shengde <shdluan(a)163.com> --- lib/mail-post.rb | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) mode change 100644 => 100755 lib/mail-post.rb diff --git a/lib/mail-post.rb b/lib/mail-post.rb old mode 100644 new mode 100755 index 9d3e601..803a134 --- a/lib/mail-post.rb +++ b/lib/mail-post.rb @@ -57,14 +57,32 @@ post '/send_mail_encode' do send_mail(mail_info) end +def check_email_limit(mail_info) + email_limit = EmailRateLimit.new(mail_info) + email_limit.check_email_counts +end + +def check_email_mapping(mail_info) + email_mapping = EmailAddrMapping.new(mail_info) + email_mapping.check_email_mapping +end + def check_send_mail(mail_info) raise 'no/empty subject.' if mail_info['subject'].nil? || mail_info['subject'].empty? raise 'no/empty email_to address.' if mail_info['to'].nil? || mail_info['to'].empty? raise 'no/empty email content.' if mail_info['body'].nil? || mail_info['body'].empty? + + return mail_info unless ENV['SEND_MAIL_PORT'].to_s == '49000' + + mail_info = check_email_mapping(mail_info.clone) + mail_info = check_email_limit(mail_info.clone) + return mail_info end def send_mail(mail_info) - check_send_mail(mail_info) + mail_info = check_send_mail(mail_info) + return if mail_info['to'].empty? + mail = Mail.new do from ENV['ROBOT_EMAIL_ADDRESS'] subject mail_info['subject'] @@ -79,7 +97,6 @@ def send_mail(mail_info) end def check_to_store_email(mail) - return if ENV['SEND_MAIL_PORT'].to_s != '49000' return if ENV['HOST_SERVER'] != 'z9' -- 2.23.0

1 0

[PATCH v2 compass-ci 5/7] send-internet-mail: add email rate limit queue
by Luan Shengde 05 Jan '21

05 Jan '21

add email_in_limit to store email address that send count less than limit value add email_out_limit to store email address that send count beyond limit value the email queues email_in_limit/email_out_limit will be reset everyday at 00:00, every time send mail to an email address, the value for the email address in email_in_limit will +1, until the value up to the limited value, the email address will be moved to email_out_limit, and the email send to the address will be throw away. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- .../send-internet-mail/email_limit_queue.rb | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100755 container/send-internet-mail/email_limit_queue.rb diff --git a/container/send-internet-mail/email_limit_queue.rb b/container/send-internet-mail/email_limit_queue.rb new file mode 100755 index 0000000..a5ca748 --- /dev/null +++ b/container/send-internet-mail/email_limit_queue.rb @@ -0,0 +1,50 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'redis' + +# to check if the emails count has beyond the limit +class EmailRateLimit + def initialize(mail_info) + @mail_info = mail_info + @redis = Redis.new('host' => REDIS_HOST, 'port' => REDIS_PORT) + end + + def check_email_counts + email_to = @mail_info['to'].clone + email_cc = @mail_info['cc'].clone + @mail_info['to'] = check_emails(email_to) + @mail_info['cc'] = check_emails(email_cc) + + return @mail_info + end + + def check_emails(mail_list) + return if mail_list.nil? || mail_list.empty? + + mail_list.clone.each do |email| + if @redis.hexists 'email_out_limit', email + mail_list -= [email] + next + elsif @redis.hexists 'email_in_limit', email + email_account = @redis.hget 'email_in_limit', email + @redis.hset 'email_in_limit', email, email_account.to_i + 1 + else + @redis.hset 'email_in_limit', email, 1 + end + + change_queue(email) + end + return mail_list + end + + def change_queue(email) + return unless (@redis.hget 'email_in_limit', email).to_i >= 10 + + @redis.hdel 'email_in_limit', email + @redis.hset 'email_out_limit', email, 10 + end +end -- 2.23.0

1 0

[PATCH v2 compass-ci 4/7] send-internet-mail: add email mapping
by Luan Shengde 05 Jan '21

05 Jan '21

example: email_addr_a => email_addr email_addr_b => email_addr when you write either email_addr_a or email_addr_b in the send address bar,the email address will be converted to email_addr Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/email_mapping.rb | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 container/send-internet-mail/email_mapping.rb diff --git a/container/send-internet-mail/email_mapping.rb b/container/send-internet-mail/email_mapping.rb new file mode 100755 index 0000000..f190896 --- /dev/null +++ b/container/send-internet-mail/email_mapping.rb @@ -0,0 +1,39 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'redis' + +# to check if the emails count has beyond the limit +class EmailAddrMapping + def initialize(mail_info) + @mail_info = mail_info + @redis = Redis.new('host' => REDIS_HOST, 'port' => REDIS_PORT) + end + + def check_email_mapping + email_to = @mail_info['to'].clone + email_cc = @mail_info['cc'].clone + email_bcc = @mail_info['bcc'].clone + @mail_info['to'] = email_mapping(email_to) + @mail_info['cc'] = email_mapping(email_cc) + @mail_info['bcc'] = email_mapping(email_bcc) + + return @mail_info + end + + def email_mapping(mail_list) + return if mail_list.nil? || mail_list.empty? + + mail_list.clone.each do |email| + next unless @redis.hexists 'email_mapping', email + + mapped_email = @redis.hget 'email_mapping', email + mail_list -= [email] + mail_list << mapped_email unless mail_list.include? mapped_email + end + return mail_list + end +end -- 2.23.0

1 0

[PATCH v2 compass-ci 3/7] send-internet-mail: add timing work
by Luan Shengde 05 Jan '21

05 Jan '21

timing work to reset email rate limit queues Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/email_init.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 container/send-internet-mail/email_init.rb diff --git a/container/send-internet-mail/email_init.rb b/container/send-internet-mail/email_init.rb new file mode 100755 index 0000000..099d89d --- /dev/null +++ b/container/send-internet-mail/email_init.rb @@ -0,0 +1,18 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'rufus-scheduler' +require 'redis' + +redis = Redis.new(host: REDIS_HOST, port: REDIS_PORT) +email_init = Rufus::Scheduler.new + +# Timing work for email counting everyday +# the email counts queues will be reset everyday +# to enable to send mail in the new day to user +email_init.cron '0 0 * * *' do + redis.del 'email_in_limit' + redis.del 'email_out_limit' +end -- 2.23.0

1 0

[PATCH lkp-tests] tests: add httpress test script
by Wei Jihui 05 Jan '21

05 Jan '21

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- tests/httpress | 137 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100755 tests/httpress diff --git a/tests/httpress b/tests/httpress new file mode 100755 index 00000000..aed71451 --- /dev/null +++ b/tests/httpress @@ -0,0 +1,137 @@ +#!/bin/sh +# - nic +# - nic_ip +# - request_ip +# - request_port +# - request_index +# - core_nums +# - protocol +# - alive_type + +. $LKP_SRC/lib/env.sh + +: "${request_index:=index.html}" +: "${core_nums:=4}" +: "${protocol:=http}" +: "${alive_type:=short}" + + +core_range=0-15 + +http_long_requests() +{ + if [ "$core_nums" == 4 ]; then + requests_num=3000000 + threads_num=100 + concurrent_num=100 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 8 ]; then + requests_num=5000000 + threads_num=300 + concurrent_num=300 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=50000000 + threads_num=1200 + concurrent_num=1200 + ethtool -L $nic combined 16 && /lkp/lkp/src/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=50000000 + threads_num=400 + concurrent_num=400 + ethtool -L $nic combined 16 && /lkp/lkp/src/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + fi +} + + +http_short_requests() +{ + if [ "$core_nums" == 4 ]; then + requests_num=3000000 + threads_num=300 + concurrent_num=300 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 8 ]; then + requests_num=5000000 + threads_num=400 + concurrent_num=400 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=10000000 + threads_num=300 + concurrent_num=300 + ethtool -L $nic combined 16 && $LKP_SRC/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=10000000 + threads_num=600 + concurrent_num=600 + ethtool -L $nic combined 16 && $LKP_SRC/bin/set_nic_irq_affinity $core_range $nic + numactl -C 16-95 httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + fi +} + +https_long_requests() +{ + if [ "$core_nums" == 4 ] || [ "$core_nums" == 8 ]; then + requests_num=5000000 + threads_num=100 + concurrent_num=100 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=5000000 + threads_num=600 + concurrent_num=600 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=10000000 + threads_num=600 + concurrent_num=600 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" -k "$request_url" + fi +} + +https_short_requests() +{ + if [ "$core_nums" == 4 ] || [ "$core_nums" == 8 ]; then + requests_num=500000 + threads_num=100 + concurrent_num=100 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 48 ]; then + requests_num=500000 + threads_num=600 + concurrent_num=600 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + elif [ "$core_nums" == 96 ]; then + requests_num=1000000 + threads_num=1200 + concurrent_num=1200 + httpress -n "$requests_num" -c "$concurrent_num" -t "$threads_num" "$request_url" + fi +} + +main() +{ + local request_url="$protocol://$request_ip:$request_port/$request_index" + + ip addr add $nic_ip/24 dev "$nic" + ip link set "$nic" up + + if [ "$protocol" == 'http' ]; then + if [ "$alive_type" == 'long' ]; then + http_long_requests + else + http_short_requests + fi + else + if [ "$alive_type" == 'long' ]; then + https_long_requests + else + https_short_requests + fi + fi +} +main -- 2.23.0

2 1

[PATCH compass-ci] 3-code/git: modify the git download path
by Hu Xuejiao 05 Jan '21

05 Jan '21

[why] if git path is not specified, lkp-tests will be downloaded in /c/compass-ci Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sparrow/3-code/git | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/sparrow/3-code/git b/sparrow/3-code/git index 48741ec..12f27c8 100755 --- a/sparrow/3-code/git +++ b/sparrow/3-code/git @@ -5,14 +5,10 @@ # fix files permissions cause container service start failed umask 002 -# download git trees - -cd /c || exit - -# git clone https://gitee.com/openeuler/compass-ci.git +# git clone compass-ci,lkp-tests,upstream-repos # modify and manual run for now: [ -d "compass-ci/.git" ] || { - git clone https://gitee.com/wu_fengguang/compass-ci.git + git clone https://gitee.com/wu_fengguang/compass-ci.git /c/compass-ci cd /c/compass-ci || exit [ -n "$compass_commit_id" ] && git reset --hard "$compass_commit_id" @@ -21,14 +17,14 @@ cd /c || exit } [ -d "lkp-tests/.git" ] || { - git clone https://gitee.com/wu_fengguang/lkp-tests.git + git clone https://gitee.com/wu_fengguang/lkp-tests.git /c/lkp-tests cd /c/lkp-tests || exit [ -n "$lkp_commit_id" ] && git reset --hard "$lkp_commit_id" } [ -d "upstream-repos/.git" ] || { - git clone https://gitee.com/wu_fengguang/upstream-repos.git + git clone https://gitee.com/wu_fengguang/upstream-repos.git /c/upstream-repos cd /c/upstream-repos || exit [ -n "$upstream_commit_id" ] && git reset --hard "$upstream_commit_id" -- 2.23.0

1 0

[PATCH v4 compass-ci] docker-rootfs: add README.md
by Wang Chenglong 05 Jan '21

05 Jan '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 container/docker-rootfs/README.md diff --git a/container/docker-rootfs/README.md b/container/docker-rootfs/README.md new file mode 100644 index 0000000..e7b5084 --- /dev/null +++ b/container/docker-rootfs/README.md @@ -0,0 +1,18 @@ +# Use the tool to create a new nfsroot for os + +## Usage: + ./run <src_docker_file_abspath> <dst_rootfs_new_abspath> + + src_docker_file_abspath: source .tar.xz file absolute path with suffix: [ tar.xz ]. + dst_rootfs_new_abspath: destination absolute path to create for rootfs. + +## Example: + ./run /tmp/openEuler-docker/openEuler-docker.aarch64.tar.xz /tmp/openeuler-rootfs/ + # Please ensure $HOME/.config/compass-ci/rootfs-passwd exists. + +## Some configuration items: + ./packages-to-install + # If you want to pre-install the software, you can write the package names in ./packages-to-install. + + $HOME/.config/compass-ci/rootfs-passwd + # Set the password for the image into this file. -- 2.23.0

1 0

[PATCH compass-ci] lib/matrix2.rb: delete useless code
by Lu Weitao 05 Jan '21

05 Jan '21

[Why] We had filtered all useless job which include nil job['suite'] by the compass-ci/lib/params_group.rb def extract_jobs_list(jobs_list) jobs_list.map! do |job| job['_source'] if job_is_useful?(job) end ... end def job_is_useful?(job) ... suite = job['_source']['suite'] return unless suite && stats.keys.any? { |stat| stat.start_with?(suite) } ... end Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/matrix2.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index d8326c5..09573ec 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -83,7 +83,7 @@ def create_matrix(job_list) suites = [] job_list.each do |job| stats = job['stats'] - suites << job['suite'] if job['suite'] + suites << job['suite'] stats.each do |key, value| next if key.include?('timestamp') -- 2.23.0

1 0

[PATCH v3 compass-ci] docker-rootfs/common: add comment for set password file.
by Wang Chenglong 05 Jan '21

05 Jan '21

[Why] add password file is easy to ignore. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- .../README.md | 0 .../{docker-rootfs => docker2rootfs}/common | 43 ++++++++++--------- .../packages-to-install | 0 .../{docker-rootfs => docker2rootfs}/run | 0 .../setup-image | 0 5 files changed, 22 insertions(+), 21 deletions(-) rename container/{docker-rootfs => docker2rootfs}/README.md (100%) rename container/{docker-rootfs => docker2rootfs}/common (77%) rename container/{docker-rootfs => docker2rootfs}/packages-to-install (100%) rename container/{docker-rootfs => docker2rootfs}/run (100%) rename container/{docker-rootfs => docker2rootfs}/setup-image (100%) diff --git a/container/docker-rootfs/README.md b/container/docker2rootfs/README.md similarity index 100% rename from container/docker-rootfs/README.md rename to container/docker2rootfs/README.md diff --git a/container/docker-rootfs/common b/container/docker2rootfs/common similarity index 77% rename from container/docker-rootfs/common rename to container/docker2rootfs/common index fe37da9..c7ae8f5 100755 --- a/container/docker-rootfs/common +++ b/container/docker2rootfs/common @@ -33,7 +33,8 @@ usage() dst_rootfs_new_abspath: destination absolute path to create for rootfs Example: - ./run /tmp/openEuler-docker/openEuler-docker.${aarch}.tar.xz /tmp/openeuler-rootfs/ + ./run /tmp/openEuler-docker/openEuler-docker.${aarch}.tar.xz /tmp/openeuler-rootfs/ + # Please ensure $HOME/.config/compass-ci/rootfs-passwd exists. " } @@ -100,26 +101,26 @@ cp_rootfs() } create_get_initrd() { - echo "Creating initrd.lkp via container/dracut-initrd..." - cd "$CCI_SRC/container/dracut-initrd" || { - echo "Failed to change into $CCI_SRC/container/dracut-initrd" - exit 6 - } - ./run "$ROOTFS_DIR/lib/modules/$ROOTFS_KERNEL" - - echo "Finding initrd.lkp under $ROOTFS_DIR/boot ..." - cd "$ROOTFS_DIR" || { - echo "Failed to change into dir \"$ROOTFS_DIR\"" - exit 7 - } - local initrd_lkp=$(find ./boot -name "initramfs.lkp*") && export ROOTFS_INITRD_LKP=$initrd_lkp - - [ -f "$ROOTFS_INITRD_LKP" ] || { - echo "Failed to generate \"$ROOTFS_INITRD_LKP\"" - exit 8 - } - - echo "initrd_lkp: $ROOTFS_INITRD_LKP" + echo "Creating initrd.lkp via container/dracut-initrd..." + cd "$CCI_SRC/container/dracut-initrd" || { + echo "Failed to change into $CCI_SRC/container/dracut-initrd" + exit 6 + } + ./run "$ROOTFS_DIR/lib/modules/$ROOTFS_KERNEL" + + echo "Finding initrd.lkp under $ROOTFS_DIR/boot ..." + cd "$ROOTFS_DIR" || { + echo "Failed to change into dir \"$ROOTFS_DIR\"" + exit 7 + } + local initrd_lkp=$(find ./boot -name "initramfs.lkp*") && export ROOTFS_INITRD_LKP=$initrd_lkp + + [ -f "$ROOTFS_INITRD_LKP" ] || { + echo "Failed to generate \"$ROOTFS_INITRD_LKP\"" + exit 8 + } + + echo "initrd_lkp: $ROOTFS_INITRD_LKP" } get_rootfs_kernel() diff --git a/container/docker-rootfs/packages-to-install b/container/docker2rootfs/packages-to-install similarity index 100% rename from container/docker-rootfs/packages-to-install rename to container/docker2rootfs/packages-to-install diff --git a/container/docker-rootfs/run b/container/docker2rootfs/run similarity index 100% rename from container/docker-rootfs/run rename to container/docker2rootfs/run diff --git a/container/docker-rootfs/setup-image b/container/docker2rootfs/setup-image similarity index 100% rename from container/docker-rootfs/setup-image rename to container/docker2rootfs/setup-image -- 2.23.0

1 1

[PATCH compass-ci] container/assign-account/answerback-email.rb: fix spell error
by Lu Kaiyi 04 Jan '21

04 Jan '21

Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- container/assign-account/answerback-email.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index a45fef2..a99306b 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -140,7 +140,7 @@ options = OptionParser.new do |opts| init_info(mail_content, email_info, my_info, my_info_es) end - opts.on('-u', '--update', 'updata configurations') do + opts.on('-u', '--update', 'update configurations') do conf_info['is_update_account'] = true end -- 2.23.0

1 0

[PATCH compass-ci 4/7] send-internet-mail: add email mapping
by Luan Shengde 04 Jan '21

04 Jan '21

if user have multi email address, can set mapping to redis queue 'email_mapping' to enable to send mail to a specified address of user with any email address of user in the send address bar example: email_addr_a => email_addr email_addr_b => email_addr when you write either email_addr_a or email_addr_b in the send address bar, the email address will be converted to email_addr Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/email_mapping.rb | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 container/send-internet-mail/email_mapping.rb diff --git a/container/send-internet-mail/email_mapping.rb b/container/send-internet-mail/email_mapping.rb new file mode 100755 index 0000000..60ff3b8 --- /dev/null +++ b/container/send-internet-mail/email_mapping.rb @@ -0,0 +1,39 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'redis' + +# to check if the emails count has beyond the limit +class EmailAddrMapping + def initialize(mail_info) + @mail_info = mail_info + @redis = Redis.new('host' => REDIS_HOST, 'port' => REDIS_PORT) + end + + def check_email_mapping + email_to = @mail_info['to'].clone + email_cc = @mail_info['cc'].clone + email_bcc = @mail_info['bcc'].clone + @mail_info['to'] = email_mapping(email_to) + @mail_info['cc'] = email_mapping(email_cc) + @mail_info['bcc'] = email_mapping(email_bcc) + + return @mail_info + end + + def email_mapping(mail_list) + return if mail_list.nil? || mail_list.empty? + + mail_list.clone.each do |email| + if @redis.hexists 'email_mapping', email + mapped_email = @redis.hget 'email_mapping', 'luanshd(a)crystal.ci' + mail_list -= [email] + mail_list << mapped_email unless mail_list.include? mapped_email + end + end + return mail_list + end +end -- 2.23.0

1 1

[PATCH compass-ci 7/7] send-internet-mail/start: add key REDIS_PORT
by Luan Shengde 04 Jan '21

04 Jan '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/send-internet-mail/start b/container/send-internet-mail/start index 02c6ab6..136435d 100755 --- a/container/send-internet-mail/start +++ b/container/send-internet-mail/start @@ -13,6 +13,7 @@ names = Set.new %w[ ROBOT_EMAIL_PASSWORD SEND_MAIL_PORT SENT_MAILDIR + REDIS_PORT ] defaults = relevant_defaults(names) -- 2.23.0

1 0

[PATCH compass-ci 6/7] lib/mail-post: add email check
by Luan Shengde 04 Jan '21

04 Jan '21

add check email count only email addresses that email count within the limited value can be sent mail add check email mapping if has set email mapping for an email address. the email address will be converted to the mapping value Signed-off-by: Luan Shengde <shdluan(a)163.com> --- lib/mail-post.rb | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) mode change 100644 => 100755 lib/mail-post.rb diff --git a/lib/mail-post.rb b/lib/mail-post.rb old mode 100644 new mode 100755 index 9d3e601..eb21ddb --- a/lib/mail-post.rb +++ b/lib/mail-post.rb @@ -57,14 +57,31 @@ post '/send_mail_encode' do send_mail(mail_info) end +def check_email_limit(mail_info) + email_limit = EmailRateLimit.new(mail_info) + email_limit.check_email_counts +end + +def check_email_mapping(mail_info) + email_mapping = EmailAddrMapping.new(mail_info) + email_mapping.check_email_mapping +end + def check_send_mail(mail_info) raise 'no/empty subject.' if mail_info['subject'].nil? || mail_info['subject'].empty? raise 'no/empty email_to address.' if mail_info['to'].nil? || mail_info['to'].empty? raise 'no/empty email content.' if mail_info['body'].nil? || mail_info['body'].empty? + + return unless ENV['SEND_MAIL_PORT'].to_s == '49000' + + mail_info = check_email_mapping(mail_info.clone) + check_email_limit(mail_info.clone) end def send_mail(mail_info) - check_send_mail(mail_info) + mail_info = check_send_mail(mail_info) + return if mail_info['to'].empty? + mail = Mail.new do from ENV['ROBOT_EMAIL_ADDRESS'] subject mail_info['subject'] @@ -79,7 +96,6 @@ def send_mail(mail_info) end def check_to_store_email(mail) - return if ENV['SEND_MAIL_PORT'].to_s != '49000' return if ENV['HOST_SERVER'] != 'z9' -- 2.23.0

1 0

[PATCH compass-ci 5/7] send-internet-mail: add email limit queue
by Luan Shengde 04 Jan '21

04 Jan '21

add email_in_limit to store email address that send count less than limit value add email_out_limit to store email address that send count beyond limit value the email queues email_in_limit/email_out_limit will be reset everyday at 00:00, every time send mail to an email address, the value for the email address in email_in_limit will +1, until the value up to the limited value, the email address will be moved to email_out_limit, and the email send to the address will be throw away. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- .../email_redis_queue_limit.rb | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100755 container/send-internet-mail/email_redis_queue_limit.rb diff --git a/container/send-internet-mail/email_redis_queue_limit.rb b/container/send-internet-mail/email_redis_queue_limit.rb new file mode 100755 index 0000000..0999983 --- /dev/null +++ b/container/send-internet-mail/email_redis_queue_limit.rb @@ -0,0 +1,48 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'redis' + +# to check if the emails count has beyond the limit +class EmailRateLimit + def initialize(mail_info) + @mail_info = mail_info + @redis = Redis.new('host' => REDIS_HOST, 'port' => REDIS_PORT) + end + + def check_email_counts + email_to = @mail_info['to'].clone + email_cc = @mail_info['cc'].clone + @mail_info['to'] = check_emails(email_to) + @mail_info['cc'] = check_emails(email_cc) + + return @mail_info + end + + def check_emails(mail_list) + return if mail_list.nil? || mail_list.empty? + + mail_list.clone.each do |email| + if @redis.hexists 'email_out_limit', email + mail_list -= [email] + next + end + + if @redis.hexists 'email_in_limit', email + email_account = @redis.hget 'email_in_limit', email + @redis.hset 'email_in_limit', email, email_account.to_i + 1 + else + @redis.hset 'email_in_limit', email, 1 + end + + if (@redis.hget 'email_in_limit', email).to_i >= 10 + @redis.hdel 'email_in_limit', email + @redis.hset 'email_out_limit', email, 10 + end + end + return mail_list + end +end -- 2.23.0

1 0

[PATCH compass-ci 3/7] send-internet-mail: add timing work
by Luan Shengde 04 Jan '21

04 Jan '21

timing work to reset email queues Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/email_init.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 container/send-internet-mail/email_init.rb diff --git a/container/send-internet-mail/email_init.rb b/container/send-internet-mail/email_init.rb new file mode 100755 index 0000000..099d89d --- /dev/null +++ b/container/send-internet-mail/email_init.rb @@ -0,0 +1,18 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'rufus-scheduler' +require 'redis' + +redis = Redis.new(host: REDIS_HOST, port: REDIS_PORT) +email_init = Rufus::Scheduler.new + +# Timing work for email counting everyday +# the email counts queues will be reset everyday +# to enable to send mail in the new day to user +email_init.cron '0 0 * * *' do + redis.del 'email_in_limit' + redis.del 'email_out_limit' +end -- 2.23.0

1 0

[PATCH compass-ci 2/7] send-internet-mail: optimize
by Luan Shengde 04 Jan '21

04 Jan '21

add init redis host/port add invoke code files Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/send-internet-mail.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/container/send-internet-mail/send-internet-mail.rb b/container/send-internet-mail/send-internet-mail.rb index 8fdaa0d..2cdfb93 100755 --- a/container/send-internet-mail/send-internet-mail.rb +++ b/container/send-internet-mail/send-internet-mail.rb @@ -6,7 +6,13 @@ require 'mail' require 'sinatra' -require "#{ENV['CCI_SRC']}/lib/mail-post" +REDIS_HOST = %x(/sbin/ip route |awk '/default/ {print $3}').chomp +REDIS_PORT = ENV['REDIS_PORT'] + +require_relative 'email_init' +require_relative 'email_redis_queue_limit' +require_relative 'email_mapping' +require_relative '../../lib/mail-post' smtp = { address: 'smtp.qq.com', -- 2.23.0

1 0

[PATCH compass-ci 1/7] send-internet-mail/Dockerfile: add install gem resources
by Luan Shengde 04 Jan '21

04 Jan '21

[why] the service need to install redis to store email counts to redis queue the service need to install rufus-scheduler to reset the queue as a timing task Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/send-internet-mail/Dockerfile b/container/send-internet-mail/Dockerfile index d151bda..03e3327 100644 --- a/container/send-internet-mail/Dockerfile +++ b/container/send-internet-mail/Dockerfile @@ -6,4 +6,4 @@ MAINTAINER shdluan(a)163.com ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && \ - apt-get install -y ruby-mail ruby-json ruby-sinatra + apt-get install -y ruby-mail ruby-json ruby-sinatra ruby-redis ruby-rufus-scheduler -- 2.23.0

1 0

[PATCH compass-ci] providers/qemu: adapt for x86_64 and riscv64
by Liu Yinsi 04 Jan '21

04 Jan '21

add x86_64 and riscv64 qemu command and options. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index 0c98065..6a1807f 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -25,6 +25,7 @@ qemu_command() { qemu=( qemu-system-aarch64 + qemu-system-x86_64 qemu-kvm ) @@ -74,6 +75,11 @@ check_option_value() } [ -n "$initrds" ] || exit + + [ -n "$(echo $kernel | grep "riscv64")" ] && qemu=qemu-system-riscv64 + + ls $bios >/dev/null 2>&1 || bios=/usr/share/ovmf/OVMF.fd + ls $helper >/dev/null 2>&1 || helper=/usr/lib/qemu/qemu-bridge-helper } get_initrd() @@ -122,6 +128,17 @@ individual_option() -bios $bios -nic $nic ) + + x86_64=( + -bios $bios + -nic $nic + ) + + riscv64=( + -machine virt + -device virtio-net-device,netdev=net0,mac=${mac} + -netdev bridge,br=br0,id=net0,helper=${helper} + ) } run_qemu() @@ -132,6 +149,13 @@ run_qemu() ;; qemu-kvm) [ "$(arch)" == "aarch64" ] && kvm=("${kvm[@]}" "${aarch64[@]}") + [ "$(arch)" == "x86_64" ] && kvm=("${kvm[@]}" "${x86_64[@]}") + ;; + qemu-system-x86_64) + kvm=("${kvm[@]}" "${x86_64[@]}") + ;; + qemu-system-riscv64) + kvm=("${kvm[@]}" "${riscv64[@]}") ;; esac -- 2.23.0

2 2

[PATCH v3 compass-ci] container/srv-http: import h5ai
by Lu Weitao 04 Jan '21

04 Jan '21

[Why] Optimize all pages style to make it more modern Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/srv-http/Dockerfile | 16 +++++++++++++--- container/srv-http/build | 7 +++++++ .../srv-http/root/etc/nginx/conf.d/default.conf | 5 ++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/container/srv-http/Dockerfile b/container/srv-http/Dockerfile index 8f9a36d..987e40e 100644 --- a/container/srv-http/Dockerfile +++ b/container/srv-http/Dockerfile @@ -1,15 +1,25 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM nginx:alpine +FROM alpine:3.9 + ADD root / -RUN apk add --no-cache php7 php7-fpm php7-session php7-json php7-exif php7-imagick php7-gd php7-fileinfo +RUN apk add --no-cache nginx php7 php7-fpm php7-session php7-json php7-exif php7-imagick php7-gd php7-fileinfo nodejs npm \ + && mkdir /run/nginx/ \ + && mkdir /h5ai RUN sed -i '/\[global\]/a daemonize = no' /etc/php7/php-fpm.conf RUN sed -i "s/user = nobody/user = nginx/g" /etc/php7/php-fpm.d/www.conf RUN sed -i "s/group = nobody/group = nginx/g" /etc/php7/php-fpm.d/www.conf -RUN chown -R nginx:nginx /srv/_h5ai +WORKDIR /h5ai +COPY h5ai /h5ai + +# build h5ai package +RUN npm install \ + && npm run build \ + && cp -r ./build/_h5ai /srv \ + && chown -R nginx:nginx /srv/_h5ai ENTRYPOINT ["/sbin/entrypoint.sh"] diff --git a/container/srv-http/build b/container/srv-http/build index 3a500ea..054e826 100755 --- a/container/srv-http/build +++ b/container/srv-http/build @@ -2,4 +2,11 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +: ${H5AI_SRC:=/c/h5ai} + +cp -r $H5AI_SRC ./ + docker build -t srv-http . + +# clean h5ai folder +rm -rf ./h5ai diff --git a/container/srv-http/root/etc/nginx/conf.d/default.conf b/container/srv-http/root/etc/nginx/conf.d/default.conf index 676418a..646eee3 100644 --- a/container/srv-http/root/etc/nginx/conf.d/default.conf +++ b/container/srv-http/root/etc/nginx/conf.d/default.conf @@ -3,8 +3,8 @@ server { server_name "srv-http"; server_tokens off; - root /usr/share/nginx/html; - index index.html; + root /srv; + index index.html index.php /_h5ai/public/index.php; charset utf-8,gbk; @@ -24,7 +24,6 @@ server { autoindex on; autoindex_localtime on; allow all; - root /srv; location ~* /[-_a-z]+$ { add_header Content-Type text/plain; } -- 2.23.0

1 0

[PATCH compass-ci] providers/qemu: refactor kvm.sh
by Liu Yinsi 04 Jan '21

04 Jan '21

before: just support aarch64 after: easy to support more arch(aarch64, x86_64, riscv64). Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 51 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index c926d3d..0c98065 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -7,6 +7,9 @@ : ${nr_cpu:=1} : ${memory:=1G} +bios=/usr/share/qemu-efi-aarch64/QEMU_EFI.fd +helper=/usr/libexec/qemu-bridge-helper + check_logfile() { log_file=/srv/cci/serial/logs/${hostname} @@ -20,8 +23,15 @@ check_logfile() qemu_command() { - qemu=qemu-system-aarch64 - command -v $qemu >/dev/null || qemu=qemu-kvm + qemu=( + qemu-system-aarch64 + qemu-kvm + ) + + for qemu in "${qemu[@]}" + do + [ -n "$(command -v ${qemu})" ] && break + done } write_logfile() @@ -72,6 +82,11 @@ get_initrd() cat $initrds > $initrd } +get_nic() +{ + nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" +} + print_message() { echo $SCHED_PORT @@ -83,24 +98,42 @@ print_message() sleep 5 } -run_qemu() +public_option() { kvm=( $qemu - -machine virt-4.0,accel=kvm,gic-version=3 -kernel $kernel -initrd $initrd -smp $nr_cpu -m $memory - -cpu Kunpeng-920 - -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd - -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} -k en-us -no-reboot -nographic -serial file:${log_file} -monitor null ) +} + +individual_option() +{ + aarch64=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) +} + +run_qemu() +{ + case $qemu in + qemu-system-aarch64) + kvm=("${kvm[@]}" "${aarch64[@]}") + ;; + qemu-kvm) + [ "$(arch)" == "aarch64" ] && kvm=("${kvm[@]}" "${aarch64[@]}") + ;; + esac "${kvm[@]}" --append "${append}" } @@ -112,7 +145,11 @@ qemu_command parse_ipxe_script check_option_value get_initrd +get_nic print_message +public_option +individual_option + run_qemu -- 2.23.0

2 2

[PATCH v2 compass-ci 2/2] providers/qemu: adapt for x86_64 and riscv64
by Liu Yinsi 04 Jan '21

04 Jan '21

add x86_64 and riscv64 qemu command and options. test: /srv/result/host-info/2021-01-04/vm-2p8g/debian-sid-aarch64/z9.280125 /srv/result/host-info/2021-01-04/vm-2p8g/debian-sid-x86_64/z9.280114 /srv/result/host-info/2021-01-04/vm-2p8g/debian-sid-riscv64/z9.280135 Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index 0c98065..1018d06 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -25,6 +25,7 @@ qemu_command() { qemu=( qemu-system-aarch64 + qemu-system-x86_64 qemu-kvm ) @@ -74,6 +75,12 @@ check_option_value() } [ -n "$initrds" ] || exit + + # debian has both qemu-system-x86_64 and qemu-system-riscv64 command + [[ $kernel =~ 'riscv64' ]] && qemu=qemu-system-riscv64 + + ls $bios >/dev/null 2>&1 || bios=/usr/share/ovmf/OVMF.fd + ls $helper >/dev/null 2>&1 || helper=/usr/lib/qemu/qemu-bridge-helper } get_initrd() @@ -122,6 +129,17 @@ individual_option() -bios $bios -nic $nic ) + + x86_64=( + -bios $bios + -nic $nic + ) + + riscv64=( + -machine virt + -device virtio-net-device,netdev=net0,mac=${mac} + -netdev bridge,br=br0,id=net0,helper=${helper} + ) } run_qemu() @@ -132,6 +150,13 @@ run_qemu() ;; qemu-kvm) [ "$(arch)" == "aarch64" ] && kvm=("${kvm[@]}" "${aarch64[@]}") + [ "$(arch)" == "x86_64" ] && kvm=("${kvm[@]}" "${x86_64[@]}") + ;; + qemu-system-x86_64) + kvm=("${kvm[@]}" "${x86_64[@]}") + ;; + qemu-system-riscv64) + kvm=("${kvm[@]}" "${riscv64[@]}") ;; esac -- 2.23.0

3 4

[PATCH v2 lkp-tests] lib/bootstrap.sh: fix error when os is not integrated into lkp-tests
by Yu Chuan 04 Jan '21

04 Jan '21

[Why] If the current os has not been integrated into lkp-tests, run method `detect_system`, then var `$_system_name_lowercase` will be 'unknown'. So we should adapt this. [Error] + local DISTRO=unknown ++ echo ntpdate ++ /home/yuchuan/lkp-tests/sbin/adapt-packages unknown Traceback (most recent call last): 3: from /home/yuchuan/lkp-tests/sbin/adapt-packages:11:in `<main>' 2: from /usr/share/ruby/psych.rb:497:in `load_file' 1: from /usr/share/ruby/psych.rb:497:in `open' /usr/share/ruby/psych.rb:497:in `initialize': No such file or directory @ rb_sysopen - /home/yuchuan/lkp-tests/distro/adaptation/unknown (Errno::ENOENT) [How] If the current os has not been integrated into lkp-tests, then return. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 7d15bb370fa9..cbf867ce322f 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -240,6 +240,10 @@ run_ntpdate() has_cmd ntpdate || { detect_system local DISTRO=$_system_name_lowercase + [ "$DISTRO" == "unknown" ] && { + echo "[WARNING] current os cannot be recognized by lkp-test." + return + } . $LKP_SRC/distro/installer/$DISTRO $(echo ntpdate | $LKP_SRC/sbin/adapt-packages $DISTRO) } -- 2.23.0

1 1

[PATCH lkp-tests 2/2] hosts/vm-2p32g: add vm-2p32g file
by Lin Jiaxin 04 Jan '21

04 Jan '21

linux compile need big memory Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- hosts/vm-2p32g | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 hosts/vm-2p32g diff --git a/hosts/vm-2p32g b/hosts/vm-2p32g new file mode 100644 index 00000000..14455f20 --- /dev/null +++ b/hosts/vm-2p32g @@ -0,0 +1,5 @@ +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 32G -- 2.23.0

1 0

[PATCH lkp-tests 1/2] jobs/build-linux.yaml: optimize the code to make it more reasonable
by Lin Jiaxin 04 Jan '21

04 Jan '21

Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- jobs/build-linux.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/build-linux.yaml b/jobs/build-linux.yaml index 1ced05ad..c1d31dca 100644 --- a/jobs/build-linux.yaml +++ b/jobs/build-linux.yaml @@ -2,6 +2,6 @@ suite: build-pkg category: functional build-pkg: config: <%= - today = Time.new - Array.new(100) { |i| 'aarch64-randconfig-' + today.strftime('%Y-%m-%d') + '-' + format('%02d', i) } + today = Time.new.strftime('%Y-%m-%d') + Array.new(100) { |i| "aarch64-randconfig-#{today}-" + format('%02d', i) } %> -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] providers/qemu: refactor kvm.sh
by Liu Yinsi 04 Jan '21

04 Jan '21

before: just support aarch64 after: easy to support more arch(aarch64, x86_64, riscv64). Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 51 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index c926d3d..0c98065 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -7,6 +7,9 @@ : ${nr_cpu:=1} : ${memory:=1G} +bios=/usr/share/qemu-efi-aarch64/QEMU_EFI.fd +helper=/usr/libexec/qemu-bridge-helper + check_logfile() { log_file=/srv/cci/serial/logs/${hostname} @@ -20,8 +23,15 @@ check_logfile() qemu_command() { - qemu=qemu-system-aarch64 - command -v $qemu >/dev/null || qemu=qemu-kvm + qemu=( + qemu-system-aarch64 + qemu-kvm + ) + + for qemu in "${qemu[@]}" + do + [ -n "$(command -v ${qemu})" ] && break + done } write_logfile() @@ -72,6 +82,11 @@ get_initrd() cat $initrds > $initrd } +get_nic() +{ + nic="tap,model=virtio-net-pci,helper=$helper,br=br0,mac=${mac}" +} + print_message() { echo $SCHED_PORT @@ -83,24 +98,42 @@ print_message() sleep 5 } -run_qemu() +public_option() { kvm=( $qemu - -machine virt-4.0,accel=kvm,gic-version=3 -kernel $kernel -initrd $initrd -smp $nr_cpu -m $memory - -cpu Kunpeng-920 - -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd - -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} -k en-us -no-reboot -nographic -serial file:${log_file} -monitor null ) +} + +individual_option() +{ + aarch64=( + -machine virt-4.0,accel=kvm,gic-version=3 + -cpu Kunpeng-920 + -bios $bios + -nic $nic + ) +} + +run_qemu() +{ + case $qemu in + qemu-system-aarch64) + kvm=("${kvm[@]}" "${aarch64[@]}") + ;; + qemu-kvm) + [ "$(arch)" == "aarch64" ] && kvm=("${kvm[@]}" "${aarch64[@]}") + ;; + esac "${kvm[@]}" --append "${append}" } @@ -112,7 +145,11 @@ qemu_command parse_ipxe_script check_option_value get_initrd +get_nic print_message +public_option +individual_option + run_qemu -- 2.23.0

1 0

[PATCH lkp-tests] jobs/build-linux.yaml: submit 100 randconfig job
by Lin Jiaxin 04 Jan '21

04 Jan '21

Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- jobs/build-linux.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jobs/build-linux.yaml b/jobs/build-linux.yaml index 080a84b3..1ced05ad 100644 --- a/jobs/build-linux.yaml +++ b/jobs/build-linux.yaml @@ -1,4 +1,7 @@ suite: build-pkg category: functional build-pkg: -config: <%= 'aarch64-randconfig-'+SecureRandom.random_number(1000).to_s %> +config: <%= + today = Time.new + Array.new(100) { |i| 'aarch64-randconfig-' + today.strftime('%Y-%m-%d') + '-' + format('%02d', i) } + %> -- 2.23.0

2 1

[PATCH lkp-tests] lib/bootstrap.sh: fix error when os is not integrated into lkp-tests
by Yu Chuan 04 Jan '21

04 Jan '21

[Why] If the current os has not been integrated into lkp-tests, run method `detect_system`, then var `$_system_name_lowercase` will be 'unknown'. So we should adapt this. [Error] + local DISTRO=unknown ++ echo ntpdate ++ /home/yuchuan/lkp-tests/sbin/adapt-packages unknown Traceback (most recent call last): 3: from /home/yuchuan/lkp-tests/sbin/adapt-packages:11:in `<main>' 2: from /usr/share/ruby/psych.rb:497:in `load_file' 1: from /usr/share/ruby/psych.rb:497:in `open' /usr/share/ruby/psych.rb:497:in `initialize': No such file or directory @ rb_sysopen - /home/yuchuan/lkp-tests/distro/adaptation/unknown (Errno::ENOENT) [How] If the current os has not been integrated into lkp-tests, then return. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 7d15bb370fa9..ab6573af1a7a 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -240,6 +240,7 @@ run_ntpdate() has_cmd ntpdate || { detect_system local DISTRO=$_system_name_lowercase + [ "$DISTRO" == "unknown" ] && return . $LKP_SRC/distro/installer/$DISTRO $(echo ntpdate | $LKP_SRC/sbin/adapt-packages $DISTRO) } -- 2.23.0

2 4

[PATCH v2 compass-ci] container/srv-http: import h5ai
by Lu Weitao 04 Jan '21

04 Jan '21

[Why] Optimize all pages style to make it more modern Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/srv-http/Dockerfile | 16 +++++++++++++--- container/srv-http/build | 7 +++++++ .../srv-http/root/etc/nginx/conf.d/default.conf | 5 ++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/container/srv-http/Dockerfile b/container/srv-http/Dockerfile index 8f9a36d..987e40e 100644 --- a/container/srv-http/Dockerfile +++ b/container/srv-http/Dockerfile @@ -1,15 +1,25 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM nginx:alpine +FROM alpine:3.9 + ADD root / -RUN apk add --no-cache php7 php7-fpm php7-session php7-json php7-exif php7-imagick php7-gd php7-fileinfo +RUN apk add --no-cache nginx php7 php7-fpm php7-session php7-json php7-exif php7-imagick php7-gd php7-fileinfo nodejs npm \ + && mkdir /run/nginx/ \ + && mkdir /h5ai RUN sed -i '/\[global\]/a daemonize = no' /etc/php7/php-fpm.conf RUN sed -i "s/user = nobody/user = nginx/g" /etc/php7/php-fpm.d/www.conf RUN sed -i "s/group = nobody/group = nginx/g" /etc/php7/php-fpm.d/www.conf -RUN chown -R nginx:nginx /srv/_h5ai +WORKDIR /h5ai +COPY h5ai /h5ai + +# build h5ai package +RUN npm install \ + && npm run build \ + && cp -r ./build/_h5ai /srv \ + && chown -R nginx:nginx /srv/_h5ai ENTRYPOINT ["/sbin/entrypoint.sh"] diff --git a/container/srv-http/build b/container/srv-http/build index 3a500ea..7b0ea74 100755 --- a/container/srv-http/build +++ b/container/srv-http/build @@ -2,4 +2,11 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +[[ $H5AI_SRC ]] || H5AI_SRC=/c/h5ai + +cp -r $H5AI_SRC ./ + docker build -t srv-http . + +# clean h5ai folder +rm -rf ./h5ai diff --git a/container/srv-http/root/etc/nginx/conf.d/default.conf b/container/srv-http/root/etc/nginx/conf.d/default.conf index 676418a..646eee3 100644 --- a/container/srv-http/root/etc/nginx/conf.d/default.conf +++ b/container/srv-http/root/etc/nginx/conf.d/default.conf @@ -3,8 +3,8 @@ server { server_name "srv-http"; server_tokens off; - root /usr/share/nginx/html; - index index.html; + root /srv; + index index.html index.php /_h5ai/public/index.php; charset utf-8,gbk; @@ -24,7 +24,6 @@ server { autoindex on; autoindex_localtime on; allow all; - root /srv; location ~* /[-_a-z]+$ { add_header Content-Type text/plain; } -- 2.23.0

2 2

[PATCH v4 compass-ci 2/2] lib/es_query.rb: parse es_result to a Hash result
by Lu Kaiyi 04 Jan '21

04 Jan '21

[why] make es result more easy access. [example] input: result_hash = {} es_result = [{"key"=>"build-pkg", "doc_count"=>354526, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>5708}, {"key"=>"finished", "doc_count"=>1033}, {"key"=>"incomplete", "doc_count"=>204}, {"key"=>"submit", "doc_count"=>136}, ... output: result_hash = {"build-pkg"=> {"failed"=>5708, "finished"=>1033, "incomplete"=>204, "submit"=>136, "OOM"=>11, "post_run"=>2}, "cci-depends"=> {"finished"=>1785, "failed"=>675, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 29d50e9..63d32ce 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -129,7 +129,7 @@ class ESQuery result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] return nil if result.empty? - result + parse_fields(result) end end @@ -217,3 +217,47 @@ def build_aggs_from_fields(fields, aggs_hash) build_aggs_from_fields(fields, aggs_hash['aggs']["all_#{field}"]) end + +# input: +# es_result = +# [{"key"=>"build-pkg", +# "doc_count"=>354526, +# "all_job_state"=> +# {"doc_count_error_upper_bound"=>0, +# "sum_other_doc_count"=>0, +# "buckets"=> +# [{"key"=>"failed", "doc_count"=>5708}, +# {"key"=>"finished", "doc_count"=>1033}, +# {"key"=>"incomplete", "doc_count"=>204}, +# {"key"=>"submit", "doc_count"=>136}, +# ... +# +# output: +# result_hash = +# {"build-pkg"=> +# {"failed"=>5708, +# "finished"=>1033, +# "incomplete"=>204, +# "submit"=>136, +# "OOM"=>11, +# "post_run"=>2}, +# "cci-depends"=> +# {"finished"=>1785, +# "failed"=>675, +# ... +def parse_fields(es_result) + result_hash ||= {} + es_result.each do |result| + key = result['key'] + sub_field = result.keys.detect { |field| field.start_with?('all_') } + + if sub_field + all_field = result[sub_field]['buckets'] + result_hash[key] = parse_fields(all_field) + else + result_hash[key] = result['doc_count'] + end + end + + result_hash +end -- 2.23.0

1 0

[PATCH v4 compass-ci 1/2] lib/es_query.rb: get multi-field aggregation count result
by Lu Kaiyi 04 Jan '21

04 Jan '21

[why] get multi-field aggregation count for overview of report page. [input]: fields, query_items (optional for query_items, default no scope limitation) [example] input: fields = ['suite'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186214}, {"key"=>"cci-depends", "doc_count"=>1230}, {"key"=>"iperf", "doc_count"=>1103}, {"key"=>"spinlock", "doc_count"=>1000}, {"key"=>"cci-makepkg", "doc_count"=>949}, {"key"=>"iperf-walk-os-test", "doc_count"=>474}, {"key"=>"bisect", "doc_count"=>381}, ... input: fields = ['suite', 'job_state'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186227, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3830}, {"key"=>"finished", "doc_count"=>803}, {"key"=>"incomplete", "doc_count"=>196}, {"key"=>"submit", "doc_count"=>136}, ... input: fields = ['os', 'os_version', 'job_state'] query_items = {} output: [{"key"=>"openeuler", "doc_count"=>193287, "all_os_version"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"20.03", "doc_count"=>191789, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3909}, {"key"=>"finished", "doc_count"=>2401}, {"key"=>"incomplete", "doc_count"=>300}, {"key"=>"submit", "doc_count"=>136}, {"key"=>"OOM", "doc_count"=>23}, {"key"=>"post_run", "doc_count"=>1}]}}, {"key"=>"20.03-SP1", "doc_count"=>904, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>641}, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/lib/es_query.rb b/lib/es_query.rb index 88a420f..29d50e9 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -87,6 +87,50 @@ class ESQuery result end + + # input: + # fields, query_items + # + # fields => fields: Array(keyword) + # eg: + # ['suite', 'job_state'] + # + # query_items => query_items: Hash(keyword, value) + # eg: + # { + # 'os' => 'openeuler', + # ... + # } + # (optional for query_items, default no scope limitation) + # output: + # [{"key"=>"build-pkg", + # "doc_count"=>186175, + # "all_job_state"=> + # {"doc_count_error_upper_bound"=>0, + # "sum_other_doc_count"=>0, + # "buckets"=> + # [{"key"=>"failed", "doc_count"=>3830}, + # {"key"=>"finished", "doc_count"=>803}, + # {"key"=>"incomplete", "doc_count"=>196}, + # ... + def query_fields(fields, query_items = {}) + field1 = fields.first + aggs_hash = {} + build_aggs_from_fields(fields, aggs_hash) + query = { + query: { + bool: { + must: build_multi_field_subquery_body(query_items) + } + }, + aggs: aggs_hash['aggs'], + size: 0 + } + result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] + return nil if result.empty? + + result + end end # Range Query Example: @@ -146,3 +190,30 @@ def assign_desc_body(keyword) }] } end + +# input: +# fields = ['os', 'os_version', 'job_state'] +# aggs_hash = {} +# output: +# aggs_hash = { +# "aggs"=> +# {"all_os"=> +# {"terms"=>{:field=>"os", :size=>1000}, +# "aggs"=> +# {"all_os_version"=> +# {"terms"=>{:field=>"os_version", :size=>1000}, +# "aggs"=> +# {"all_job_state"=>{"terms"=>{:field=>"job_state", :size=>1000}}}}}}} +# } +def build_aggs_from_fields(fields, aggs_hash) + return if fields.empty? + + field = fields.shift + aggs_hash['aggs'] = { + "all_#{field}" => { + 'terms' => { field: field, size: 1000 } + } + } + + build_aggs_from_fields(fields, aggs_hash['aggs']["all_#{field}"]) +end -- 2.23.0

1 0

[PATCH v2 compass-ci] docker-rootfs/common: add comment for set password file.
by Wang Chenglong 04 Jan '21

04 Jan '21

[Why] add password file is easy to ignore. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/container/docker-rootfs/common b/container/docker-rootfs/common index fe37da9..2ee16c8 100755 --- a/container/docker-rootfs/common +++ b/container/docker-rootfs/common @@ -33,7 +33,8 @@ usage() dst_rootfs_new_abspath: destination absolute path to create for rootfs Example: - ./run /tmp/openEuler-docker/openEuler-docker.${aarch}.tar.xz /tmp/openeuler-rootfs/ + ./run /tmp/openEuler-docker/openEuler-docker.${aarch}.tar.xz /tmp/openeuler-rootfs/ + # please ensure $HOME/.config/compass-ci/rootfs-passwd exists " } -- 2.23.0

3 4

[PATCH v3 compass-ci] docker-rootfs: add README.md
by Wang Chenglong 04 Jan '21

04 Jan '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 container/docker-rootfs/README.md diff --git a/container/docker-rootfs/README.md b/container/docker-rootfs/README.md new file mode 100644 index 0000000..1b6f42f --- /dev/null +++ b/container/docker-rootfs/README.md @@ -0,0 +1,18 @@ +## Use the tool to create a new nfsroot for os + +## Usage: + ./run <src_docker_file_abspath> <dst_rootfs_new_abspath> + + src_docker_file_abspath: source .tar.xz file absolute path with suffix: [ tar.xz ]. + dst_rootfs_new_abspath: destination absolute path to create for rootfs. + +## Example: + ./run /tmp/openEuler-docker/openEuler-docker.aarch64.tar.xz /tmp/openeuler-rootfs/ + # please ensure $HOME/.config/compass-ci/rootfs-passwd exists + +## Some configuration items: + ./packages-to-install + # If you want to pre-install the software, you can write the package names in ./packages-to-install. + + $HOME/.config/compass-ci/rootfs-passwd + # Set the password for the image into this file. -- 2.23.0

2 2

[PATCH v4 compass-ci] docker-rootfs: add README.md
by Wang Chenglong 04 Jan '21

04 Jan '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker-rootfs/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 container/docker-rootfs/README.md diff --git a/container/docker-rootfs/README.md b/container/docker-rootfs/README.md new file mode 100644 index 0000000..e7b5084 --- /dev/null +++ b/container/docker-rootfs/README.md @@ -0,0 +1,18 @@ +# Use the tool to create a new nfsroot for os + +## Usage: + ./run <src_docker_file_abspath> <dst_rootfs_new_abspath> + + src_docker_file_abspath: source .tar.xz file absolute path with suffix: [ tar.xz ]. + dst_rootfs_new_abspath: destination absolute path to create for rootfs. + +## Example: + ./run /tmp/openEuler-docker/openEuler-docker.aarch64.tar.xz /tmp/openeuler-rootfs/ + # Please ensure $HOME/.config/compass-ci/rootfs-passwd exists. + +## Some configuration items: + ./packages-to-install + # If you want to pre-install the software, you can write the package names in ./packages-to-install. + + $HOME/.config/compass-ci/rootfs-passwd + # Set the password for the image into this file. -- 2.23.0

1 0

[PATCH compass-ci 2/2] qcow2rootfs: fix rootfs-dir permission error
by Yu Chuan 03 Jan '21

03 Jan '21

[Error Msg] '01/03/21_04:25:49' Compressing file to /root/extract/rootfs-dir/qcow2-out.tar.gz Please wait until command returned... <r.gz compress:gzip numericowner:true xattrs:true selinux:true acls:true libguestfs: error: /root/extract/rootfs-dir/qcow2-out.tar.gz: Permission denied libguestfs: error: file receive cancelled by daemon Signed-off-by: Yu Chuan <13186087857(a)163.com> --- container/qcow2rootfs/bin/main | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/qcow2rootfs/bin/main b/container/qcow2rootfs/bin/main index b6ef7238f24a..c7527b794b89 100755 --- a/container/qcow2rootfs/bin/main +++ b/container/qcow2rootfs/bin/main @@ -34,7 +34,7 @@ unzip_qcow2_xz() { extract_rootfs() { echo "[WARNNING] Temporarily change owner of qcow2 file" chown "$EXTRACT_USER":"$EXTRACT_USER" "$QCOW2_FILE" - chmod 775 "$ROOTFS_DIR" + chmod 777 "$ROOTFS_DIR" echo "[$(date +'%D %T')] Converting .qcow2 to .tar.gz, please wait several minutes..." su - "$EXTRACT_USER" -c " -- 2.23.0

1 0

[PATCH compass-ci 1/2] qcow2rootfs: generate pkgs record file for result rootfs
by Yu Chuan 03 Jan '21

03 Jan '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- container/qcow2rootfs/bin/cci-generate-pkgs | 15 +++++++++++++++ container/qcow2rootfs/bin/main | 6 ++++++ 2 files changed, 21 insertions(+) create mode 100755 container/qcow2rootfs/bin/cci-generate-pkgs diff --git a/container/qcow2rootfs/bin/cci-generate-pkgs b/container/qcow2rootfs/bin/cci-generate-pkgs new file mode 100755 index 000000000000..e6a974cd0f41 --- /dev/null +++ b/container/qcow2rootfs/bin/cci-generate-pkgs @@ -0,0 +1,15 @@ +#!/bin/bash + +RESULT_FILE=/etc/pkgs-$(date "+%Y-%m-%d-%H-%M-%S").list + +command -v rpm >/dev/null && { + rpm -qa > $RESULT_FILE +} + +command -v dpkg >/dev/null && { + dpkg -l > $RESULT_FILE +} + +chmod 644 $RESULT_FILE + +exit 0 diff --git a/container/qcow2rootfs/bin/main b/container/qcow2rootfs/bin/main index b70ec7766649..b6ef7238f24a 100755 --- a/container/qcow2rootfs/bin/main +++ b/container/qcow2rootfs/bin/main @@ -74,6 +74,12 @@ pre_config_rootfs() { if [ -f ${ROOTFS_DIR}/etc/selinux/config ]; then sed -i -r 's/SELINUX=enforcing/SELINUX=disabled/g' "$ROOTFS_DIR/etc/selinux/config" fi + + # generate /etc/pkgs-$(date "+%Y-%m-%d-%H-%M-%S").list file for result rootfs + echo "generate /etc/pkgs-$(date "+%Y-%m-%d-%H-%M-%S").list" + cp $EXTRACT_ROOT/bin/cci-generate-pkgs $ROOTFS_DIR + chroot $ROOTFS_DIR /cci-generate-pkgs + rm /$ROOTFS_DIR/cci-generate-pkgs } clean_environment() { -- 2.23.0

1 0

[PATCH v2 lkp-tests] lib/bootstrap.sh: 'run-ipconfig' step should be before the check step
by Yu Chuan 03 Jan '21

03 Jan '21

[Why] The `run-ipconfig` step is the IP configuration step for initramfs, which should be executed immediately after checking the initramfs-tools file for existence. [Error Msg] + [ ! -e /usr/share/initramfs-tools/scripts/functions ] + get_net_devices + local net_devices + local i + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + [ p0s1/ != /sys/class/net/enp0s1/ ] + basename /sys/class/net/enp0s1/ + net_devices= enp0s1 + [ != /sys/class/net/lo/ ] + continue + echo enp0s1 + local net_devices= enp0s1 + [ -z ] + warn_no_eth0 + [ -f /proc/config.gz ] + return + echo ls /sys/class/net + ls /sys/class/net + is_virt + [ -n ] + [ -n ] + has_cmd virt-what + command -v virt-what + grep -q -w hypervisor /proc/cpuinfo + return 0 + export NO_NETWORK=1 + echo export NO_NETWORK=1 due to no net devices + return Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 7d15bb370fa9..770d550724f1 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -160,6 +160,9 @@ setup_network() return } + $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig + network_ok && return + local net_devices=$(get_net_devices) if [ -z "$net_devices" ]; then @@ -181,9 +184,6 @@ setup_network() fi fi - $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig - network_ok && return - local err_msg='IP-Config: Auto-configuration of network failed' dmesg | grep -q -F "$err_msg" || { # Include $err_msg in the error message so that it matches -- 2.23.0

1 0

[PATCH lkp-tests] lib/bootstrap.sh: 'run-ipconfig' step should be before the check step
by Yu Chuan 03 Jan '21

03 Jan '21

[Before] test job: result/borrow/2020-12-31/vm-2p8g/debian-sid-aarch64/3600/z9.275666 key log: + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + cat /sys/class/net/enp0s1//operstate + [ down = up ] [After] test job: result/borrow/2020-12-31/vm-2p8g/debian-sid-aarch64/3600/z9.275673 key log: + [ /sys/class/net/enp0s1/ != /sys/class/net/enp0s1/ ] + cat /sys/class/net/enp0s1//operstate + [ up = up ] Signed-off-by: Yu Chuan <13186087857(a)163.com> --- lib/bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index 7d15bb370fa9..770d550724f1 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -160,6 +160,9 @@ setup_network() return } + $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig + network_ok && return + local net_devices=$(get_net_devices) if [ -z "$net_devices" ]; then @@ -181,9 +184,6 @@ setup_network() fi fi - $LKP_DEBUG_PREFIX $LKP_SRC/bin/run-ipconfig - network_ok && return - local err_msg='IP-Config: Auto-configuration of network failed' dmesg | grep -q -F "$err_msg" || { # Include $err_msg in the error message so that it matches -- 2.23.0

2 2

git count
by Wu Fengguang 31 Dec '20

31 Dec '20

Cao Xueliang is our code star again, thanks xueliang! 22 Cao Xueliang 21 Luan Shengde 16 Wu Zhende 13 Wei Jihui 13 Li Yuanchao 11 Hu Xuejiao 11 Ren Wen 10 Lu Weitao 9 Liu Shaofei 7 Lin Jiaxin 7 Lu Kaiyi 6 Liu Yinsi 6 Cui Yili 6 Wang Yong 5 Li Ping 4 Xiao Shenwei 3 Wu Fengguang 3 Zhang Yu 3 Yu Chuan 2 Xu Xijian 2 Wang Chenglong 1 Zhang Yale 0 Bai Jing 0 Tong Qunfeng 0 Du Kaitian 0 Sun Yukui 0 Zhang Dewan 0 Shi Zhichao

1 0

[PATCH v2 compass-ci 2/2] mail-robot: optimize assign-account-email
by Luan Shengde 31 Dec '20

31 Dec '20

set different greetings for mail-robot and bisect Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/assign-account-email.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/container/mail-robot/lib/assign-account-email.rb b/container/mail-robot/lib/assign-account-email.rb index ee3b574..b71b3fe 100755 --- a/container/mail-robot/lib/assign-account-email.rb +++ b/container/mail-robot/lib/assign-account-email.rb @@ -4,13 +4,19 @@ # frozen_string_literal: true def build_apply_account_email(my_info) + email_greeting = if my_info['bisect'] + 'We have automatically created the following information for you.' + else + 'Thank you for joining us.' + end + email_msg = <<~EMAIL_MESSAGE To: #{my_info['my_email']} Subject: [compass-ci] Account Ready Dear #{my_info['my_name']}, - Thank you for joining us. + #{email_greeting} You can use the following info to submit jobs: -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] lib/assign_account_client: optimize
by Luan Shengde 31 Dec '20

31 Dec '20

add new key for bisect way to apply account use rest-client instead of linux command 'curl' Signed-off-by: Luan Shengde <shdluan(a)163.com> --- lib/assign_account_client.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/assign_account_client.rb b/lib/assign_account_client.rb index d7e0623..4831545 100755 --- a/lib/assign_account_client.rb +++ b/lib/assign_account_client.rb @@ -7,6 +7,7 @@ require 'json' require 'mail' require 'set' require 'optparse' +require 'rest-client' require_relative '../container/defconfig' require_relative 'es_client' require_relative '../container/mail-robot/lib/assign-account-email' @@ -52,7 +53,8 @@ class AutoAssignAccount apply_info['is_update_account'] = true unless apply_info['my_login_name'].nil? apply_info['lab'] = LAB - account_info_str = %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d '#{apply_info.to_json}') + assign_account_url = "#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account" + account_info_str = RestClient.post assign_account_url, apply_info.to_json JSON.parse account_info_str end @@ -69,6 +71,7 @@ class AutoAssignAccount end def send_mail + @my_info['bisect'] = true message = build_apply_account_email(@my_info) %x(curl -XPOST "#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text" -d "#{message}") end -- 2.23.0

1 0

[PATCH compass-ci 2/2] mail-robot: optimize assign-account-email
by Luan Shengde 31 Dec '20

31 Dec '20

set different greetings for mail-robot and bisect Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/lib/assign-account-email.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/container/mail-robot/lib/assign-account-email.rb b/container/mail-robot/lib/assign-account-email.rb index ee3b574..b71b3fe 100755 --- a/container/mail-robot/lib/assign-account-email.rb +++ b/container/mail-robot/lib/assign-account-email.rb @@ -4,13 +4,19 @@ # frozen_string_literal: true def build_apply_account_email(my_info) + email_greeting = if my_info['bisect'] + 'We have automatically created the following information for you.' + else + 'Thank you for joining us.' + end + email_msg = <<~EMAIL_MESSAGE To: #{my_info['my_email']} Subject: [compass-ci] Account Ready Dear #{my_info['my_name']}, - Thank you for joining us. + #{email_greeting} You can use the following info to submit jobs: -- 2.23.0

1 0

[PATCH compass-ci 1/2] lib/assign_account_client: optimize
by Luan Shengde 31 Dec '20

31 Dec '20

add new key for bisect way to apply account use rest-client instead of linux command 'curl' Signed-off-by: Luan Shengde <shdluan(a)163.com> --- lib/assign_account_client.rb | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/assign_account_client.rb b/lib/assign_account_client.rb index d7e0623..15d3f35 100755 --- a/lib/assign_account_client.rb +++ b/lib/assign_account_client.rb @@ -7,6 +7,7 @@ require 'json' require 'mail' require 'set' require 'optparse' +require 'rest-client' require_relative '../container/defconfig' require_relative 'es_client' require_relative '../container/mail-robot/lib/assign-account-email' @@ -52,7 +53,8 @@ class AutoAssignAccount apply_info['is_update_account'] = true unless apply_info['my_login_name'].nil? apply_info['lab'] = LAB - account_info_str = %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d '#{apply_info.to_json}') + assign_account_url = "#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account" + account_info_str = RestClient.post assign_account_url, apply_info.to_json JSON.parse account_info_str end @@ -69,6 +71,7 @@ class AutoAssignAccount end def send_mail + @my_info['bisect'] = true message = build_apply_account_email(@my_info) %x(curl -XPOST "#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text" -d "#{message}") end @@ -81,3 +84,12 @@ class AutoAssignAccount send_mail end end + +my_info = { + 'my_email' => 'shdluan(a)163.com', + 'my_name' => 'lsd', + 'my_commit_url' => '' +} + +test = AutoAssignAccount.new(my_info) +test.send_account -- 2.23.0

1 0

[PATCH compass-ci] lib/mail_bisect_result: send bisect report to author email
by Cao Xueliang 31 Dec '20

31 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/mail_bisect_result.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/mail_bisect_result.rb b/lib/mail_bisect_result.rb index f0f0194..209a28d 100644 --- a/lib/mail_bisect_result.rb +++ b/lib/mail_bisect_result.rb @@ -18,8 +18,9 @@ class MailBisectResult @pkgbuild_repo = bisect_info['pkgbuild_repo'] @first_bad_commit_result_root = bisect_info['first_bad_commit_result_root'] @git_commit = GitCommit.new(@repo, @commit_id) + @to = @git_commit.author_email # now send mail to review - @to = 'caoxl(a)crystal.ci, caoxl78320(a)163.com, huming15(a)163.com' + @bcc = 'caoxl(a)crystal.ci, caoxl78320(a)163.com, huming15(a)163.com, wfg(a)mail.ustc.edu.cn' end def create_send_email @@ -37,6 +38,7 @@ class MailBisectResult data = <<~BODY To: #{@to} + Bcc: #{@bcc} Subject: #{subject} Hi #{(a)git_commit.author_name}, -- 2.23.0

1 0

[PATCH compass-ci] delimiter: use docker to run build-pkg job
by Cao Xueliang 31 Dec '20

31 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter/delimiter.rb | 2 +- src/delimiter/utils.rb | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/delimiter/delimiter.rb b/src/delimiter/delimiter.rb index bf36db3..1afffbc 100644 --- a/src/delimiter/delimiter.rb +++ b/src/delimiter/delimiter.rb @@ -25,7 +25,7 @@ class Delimiter next end - %x(#{LKP_SRC}/sbin/submit bad_job_id=#{task['job_id']} error_id=#{task['error_id'].inspect} bisect.yaml) + %x(#{LKP_SRC}/sbin/submit bad_job_id=#{task['job_id']} error_id=#{task['error_id'].inspect} bisect.yaml queue='dc-bisect') rescue StandardError => e puts e sleep(60) diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index 49d48ee..de5a302 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -135,10 +135,16 @@ module Utils job['my_email'] = account_info['my_email'] job['my_token'] = account_info['my_token'] job['bad_job_id'] = job_id - job['testbox'] = job['config'] ? 'vm-2p32g' : 'vm-2p16g' + job['testbox'] = job['config'] ? 'dc-16g' : 'dc-8g' + job['os'] = 'openeuler' + job['os_arch'] = 'aarch64' + job['os_mount'] = 'container' + job['os_version'] = '20.03-pre' + job['docker_image'] = 'openeuler:20.03-pre' - job.delete('error_ids') job.delete('start_time') + job.delete('tbox_group') + job.delete('error_ids') job.delete('job_state') job.delete('end_time') job.delete('loadavg') -- 2.23.0

1 0

[PATCH v4 compass-ci] doc/tutorial.md: fix abnormal display in markdown
by Lu Kaiyi 31 Dec '20

31 Dec '20

the middle part in * and * is italics for markdown document, if want show '*' in this condition, add ` in both sides of path. Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- doc/tutorial.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorial.md b/doc/tutorial.md index e6f8491..bf72630 100644 --- a/doc/tutorial.md +++ b/doc/tutorial.md @@ -258,7 +258,7 @@ job yaml 是测试描述和执行的基本单元，以[YAML](http://yaml.org/YAM 4) 备注目前提供 archlinux 下已有 PKGBUILD 项目构建测试，暂不支持自主导入 PKGBUILD 文件。 - PKGBUILD 存放路径： /srv/git/archlinux/*/*/PKGBUILD。 + PKGBUILD 存放路径： `/srv/git/archlinux/*/*/PKGBUILD`。 ### depends -- 2.23.0

1 0

[PATCH v2 compass-ci] doc: add runtime field in borrow-*.yml and update borrow-machine.*.md
by Zhang Yale 31 Dec '20

31 Dec '20

User can specify the borrow time in "runtime" field Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- doc/manual/borrow-machine.en.md | 7 ++++--- doc/manual/borrow-machine.zh.md | 9 +++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/doc/manual/borrow-machine.en.md b/doc/manual/borrow-machine.en.md index bed9fcc..b598bec 100644 --- a/doc/manual/borrow-machine.en.md +++ b/doc/manual/borrow-machine.en.md @@ -203,6 +203,7 @@ Ensure that you have performed the following operations according to the [apply- suite: borrow testcase: borrow + runtime: 1h ssh_pub_key: <%= begin File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp @@ -212,10 +213,10 @@ Ensure that you have performed the following operations according to the [apply- %> sshd: # sleep at the bottom + sleep: + hi684@account-vm ~/lkp-tests/jobs% grep runtime: borrow-1h.yaml sleep: 1h - hi684@account-vm ~/lkp-tests/jobs% grep sleep: borrow-1h.yaml - sleep: 1h - # Use the VIM editor to change the value of the sleep field. + # Use the VIM editor to change the value of the runtime field. hi684@account-vm ~/lkp-tests/jobs% vim borrow-1h.yaml # After changing the value, submit the request again. hi684@account-vm ~/lkp-tests/jobs% submit -c -m testbox=vm-2p8g borrow-1h.yaml diff --git a/doc/manual/borrow-machine.zh.md b/doc/manual/borrow-machine.zh.md index 3f500e2..5288377 100644 --- a/doc/manual/borrow-machine.zh.md +++ b/doc/manual/borrow-machine.zh.md @@ -206,7 +206,8 @@ hi684@account-vm ~/lkp-tests/jobs% cat borrow-1h.yaml suite: borrow testcase: borrow - + + runtime: 1h ssh_pub_key: <%= begin File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp @@ -216,10 +217,10 @@ %> sshd: # sleep at the bottom + sleep: + hi684@account-vm ~/lkp-tests/jobs% grep runtime: borrow-1h.yaml sleep: 1h - hi684@account-vm ~/lkp-tests/jobs% grep sleep: borrow-1h.yaml - sleep: 1h - # 使用vim来修改你的sleep字段的值 + # 使用vim来修改你的runtime字段的值 hi684@account-vm ~/lkp-tests/jobs% vim borrow-1h.yaml # 修改完毕后重新submit即可 hi684@account-vm ~/lkp-tests/jobs% submit -c -m testbox=vm-2p8g borrow-1h.yaml -- 2.23.0

2 1

[PATCH v3 compass-ci] sparrow/4-docker/buildall: build dist image when it doesn't exist
by Xu Xijian 31 Dec '20

31 Dec '20

If the dist docker image doesn't exist for some reason, run build script to create it. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- sparrow/4-docker/buildall | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sparrow/4-docker/buildall b/sparrow/4-docker/buildall index 32801d1..ca4406b 100755 --- a/sparrow/4-docker/buildall +++ b/sparrow/4-docker/buildall @@ -32,7 +32,8 @@ do_one() ( cd $container [ "$container_name" == 'ssh-r' ] && exit - if [ "$action" != "reboot" ]; then + docker images | grep -wq "$container_name" + if [ "$?" == 0 ] && [ "$action" != "reboot" ]; then [ -x build ] && ./build [ -x install ] && ./install fi -- 2.23.0

1 0

[PATCH v2 compass-ci] sparrow/4-docker/buildall: build dist image when it doesn't exist
by Xu Xijian 31 Dec '20

31 Dec '20

If the dist docker image doesn't exist for some reason, run build script to create it. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- sparrow/4-docker/buildall | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sparrow/4-docker/buildall b/sparrow/4-docker/buildall index 32801d1..9e66ac9 100755 --- a/sparrow/4-docker/buildall +++ b/sparrow/4-docker/buildall @@ -32,7 +32,8 @@ do_one() ( cd $container [ "$container_name" == 'ssh-r' ] && exit - if [ "$action" != "reboot" ]; then + docker images | grep -w -q "$container_name" + if [ "$?" == 0 ] && [ "$action" != "reboot" ]; then [ -x build ] && ./build [ -x install ] && ./install fi -- 2.23.0

1 1

[PATCH lkp-tests] tests/build-pkg: fix to generate the directory of *.cgz file
by Liu Shaofei 31 Dec '20

31 Dec '20

[why] Since the file upload path saved by ES is inconsistent with the file path generated by the build-pkg script, the *.cgz file upload failed. [error] ==> /tmp/stdout <== renamed '/initrd/build-pkg/nfs/openeuler/aarch64/20.03-SP1/kubelet/HEAD.cgz.tmp' -> '/initrd/build-pkg/nfs/openeuler/aarch64/20.03-SP1/kubelet/HEAD.cgz' ==> Leaving fakeroot environment. ==> Finished making: kubernetes 1.18.5-1 (Wed Dec 30 16:50:17 CST 2020) create soft link: latest.cgz -> HEAD.cgz ==> /tmp/stderr <== curl: (22) The requested URL returned error: 403 Forbidden Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- tests/build-pkg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/build-pkg b/tests/build-pkg index 801649cf4..e21397ee6 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -88,6 +88,9 @@ request_pkg() create_cgz_name() { + pkgname="${pkgname}" + [ -n "${pkgbase}" ] && pkgname="${pkgbase}" + cgz_path="$PKG_MNT/${pack_to}/${pkgname}" cgz_name="${cgz_path}/${upstream_commit}.cgz" pkg_args="-A --check --skippgpcheck" -- 2.23.0

3 5

[PATCH compass-ci] doc: add runtime field in borrow-*.yml and update borrow-machine.*.md
by Zhang Yale 31 Dec '20

31 Dec '20

User can specify the borrow time in "runtime" field Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- doc/manual/borrow-machine.en.md | 7 ++++--- doc/manual/borrow-machine.zh.md | 9 +++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/doc/manual/borrow-machine.en.md b/doc/manual/borrow-machine.en.md index bed9fcc..b598bec 100644 --- a/doc/manual/borrow-machine.en.md +++ b/doc/manual/borrow-machine.en.md @@ -203,6 +203,7 @@ Ensure that you have performed the following operations according to the [apply- suite: borrow testcase: borrow + runtime: 1h ssh_pub_key: <%= begin File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp @@ -212,10 +213,10 @@ Ensure that you have performed the following operations according to the [apply- %> sshd: # sleep at the bottom + sleep: + hi684@account-vm ~/lkp-tests/jobs% grep runtime: borrow-1h.yaml sleep: 1h - hi684@account-vm ~/lkp-tests/jobs% grep sleep: borrow-1h.yaml - sleep: 1h - # Use the VIM editor to change the value of the sleep field. + # Use the VIM editor to change the value of the runtime field. hi684@account-vm ~/lkp-tests/jobs% vim borrow-1h.yaml # After changing the value, submit the request again. hi684@account-vm ~/lkp-tests/jobs% submit -c -m testbox=vm-2p8g borrow-1h.yaml diff --git a/doc/manual/borrow-machine.zh.md b/doc/manual/borrow-machine.zh.md index 3f500e2..5288377 100644 --- a/doc/manual/borrow-machine.zh.md +++ b/doc/manual/borrow-machine.zh.md @@ -206,7 +206,8 @@ hi684@account-vm ~/lkp-tests/jobs% cat borrow-1h.yaml suite: borrow testcase: borrow - + + runtime: 1h ssh_pub_key: <%= begin File.read("#{ENV['HOME']}/.ssh/id_rsa.pub").chomp @@ -216,10 +217,10 @@ %> sshd: # sleep at the bottom + sleep: + hi684@account-vm ~/lkp-tests/jobs% grep runtime: borrow-1h.yaml sleep: 1h - hi684@account-vm ~/lkp-tests/jobs% grep sleep: borrow-1h.yaml - sleep: 1h - # ��vim��sleep�� + # ��vim��runtime�� hi684@account-vm ~/lkp-tests/jobs% vim borrow-1h.yaml # ��submit�� hi684@account-vm ~/lkp-tests/jobs% submit -c -m testbox=vm-2p8g borrow-1h.yaml -- 2.23.0

1 0

[PATCH v3 lkp-tests] optimize the deploy-cci script
by Hu Xuejiao 31 Dec '20

31 Dec '20

Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- tests/deploy-cci | 52 ++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/tests/deploy-cci b/tests/deploy-cci index 55835544..8e6acbac 100755 --- a/tests/deploy-cci +++ b/tests/deploy-cci @@ -2,30 +2,38 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -# redirect remote-repository address to "/srv/git/gitee.com" -if [ -n "$GIT_SERVER" ]; then - cat >> /etc/gitconfig <<-EOF - [url "git://$GIT_SERVER/gitee.com"] - insteadOf=https://gitee.com - EOF -fi +export_env() +{ + export initrd_http_host=$INITRD_HTTP_HOST + export initrd_http_port=$INITRD_HTTP_PORT + export gitcache_host=$GITCACHE_HOST + export gitcache_port=$GITCACHE_PORT +} -export initrd_http_host=$INITRD_HTTP_HOST -export initrd_http_port=$INITRD_HTTP_PORT -export gitcache_host=$GITCACHE_HOST -export gitcache_port=$GITCACHE_PORT +deploy() +{ + # redirect remote-repository address to "/srv/git/gitee.com" + if [ -n "$GIT_SERVER" ]; then + cat >> /etc/gitconfig <<-EOF + [url "git://$GIT_SERVER/gitee.com"] + insteadOf=https://gitee.com + EOF + fi -mkdir -p /c -cd /c || exit -git clone https://gitee.com/wu_fengguang/compass-ci.git + mkdir -p /c && cd /c + git clone https://gitee.com/wu_fengguang/compass-ci.git -cd /c/compass-ci || exit -[ -n "$compass_commitid" ] && git reset --hard "$compass_commitid" + cd /c/compass-ci || exit + [ -n "$compass_commit_id" ] && git reset --hard "$compass_commit_id" -# Need mount tmpfs to fix compass-ci/container/os-nfs/start error -# "exportfs: /exports/result does not support NFS export" -mount -t tmpfs none /srv + # Need mount tmpfs to fix compass-ci/container/os-nfs/start error + # "exportfs: /exports/result does not support NFS export" + mount -t tmpfs none /srv -# test the script for one-click deployment -cd /c/compass-ci/sparrow || exit -./install-tiny + # test the script for one-click deployment + cd /c/compass-ci/sparrow || exit + ./install-tiny +} + +export_env +deploy -- 2.23.0

1 0

[PATCH v3 lkp-tests 2/2] pkg: add httpress
by Wei Jihui 31 Dec '20

31 Dec '20

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- pkg/httpress/PKGBUILD | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 pkg/httpress/PKGBUILD diff --git a/pkg/httpress/PKGBUILD b/pkg/httpress/PKGBUILD new file mode 100644 index 00000000..8891d6c7 --- /dev/null +++ b/pkg/httpress/PKGBUILD @@ -0,0 +1,17 @@ +pkgname=httpress +pkgver=1 +pkgrel=1 +url='http://ftp.us2.freshrpms.net/linux/RPM/epel/7/aarch64/Packages/h/httpress-1…' +arch=('aarch64') +license=('BSD') +source=("https://download.fedoraproject.org/pub/epel/7/SRPMS/Packages/h/httpress-1.1…") +md5sums=('SKIP') + +package() +{ + # The rpm from source maybe not be available + # httpress-1.1.0-13.fc31.aarch64.rpm has been validated to be available + rpmpkg_path="${pkgdir}/opt/rpms/" + mkdir -p "${rpmpkg_path}" + cp "${srcdir}/"*.rpm "${rpmpkg_path}" +} -- 2.23.0

1 0

[PATCH v3 lkp-tests 1/2] distro/depends: add httpress
by Wei Jihui 31 Dec '20

31 Dec '20

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- distro/depends/httpress | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 distro/depends/httpress diff --git a/distro/depends/httpress b/distro/depends/httpress new file mode 100644 index 00000000..73fc57c6 --- /dev/null +++ b/distro/depends/httpress @@ -0,0 +1,2 @@ +numactl +ethtool -- 2.23.0

1 0

[PATCH compass-ci] lib/es_query.rb: parse es_result to a Hash result
by Lu Kaiyi 31 Dec '20

31 Dec '20

[why] make es result more easy access. [example] input: result_hash = {} es_result = [{"key"=>"build-pkg", "doc_count"=>354526, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>5708}, {"key"=>"finished", "doc_count"=>1033}, {"key"=>"incomplete", "doc_count"=>204}, {"key"=>"submit", "doc_count"=>136}, ... output: result_hash = {"build-pkg"=> {"failed"=>5708, "finished"=>1033, "incomplete"=>204, "submit"=>136, "OOM"=>11, "post_run"=>2}, "cci-depends"=> {"finished"=>1785, "failed"=>675, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 29d50e9..dcd735b 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -129,7 +129,9 @@ class ESQuery result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] return nil if result.empty? - result + result_hash = {} + parse_fields(result, result_hash) + result_hash end end @@ -217,3 +219,46 @@ def build_aggs_from_fields(fields, aggs_hash) build_aggs_from_fields(fields, aggs_hash['aggs']["all_#{field}"]) end + +# input: +# result_hash = {} +# es_result = +# [{"key"=>"build-pkg", +# "doc_count"=>354526, +# "all_job_state"=> +# {"doc_count_error_upper_bound"=>0, +# "sum_other_doc_count"=>0, +# "buckets"=> +# [{"key"=>"failed", "doc_count"=>5708}, +# {"key"=>"finished", "doc_count"=>1033}, +# {"key"=>"incomplete", "doc_count"=>204}, +# {"key"=>"submit", "doc_count"=>136}, +# ... +# +# output: +# result_hash = +# {"build-pkg"=> +# {"failed"=>5708, +# "finished"=>1033, +# "incomplete"=>204, +# "submit"=>136, +# "OOM"=>11, +# "post_run"=>2}, +# "cci-depends"=> +# {"finished"=>1785, +# "failed"=>675, +# ... +def parse_fields(es_result, result_hash) + es_result.each do |result| + key = result['key'] + sub_field = result.keys.detect { |field| field.start_with?('all_') } + + if sub_field + result_hash[key] = {} + all_field = result[sub_field]['buckets'] + parse_fields(all_field, result_hash[key]) + else + result_hash[key] = result['doc_count'] + end + end +end -- 2.23.0

2 1

[PATCH compass-ci] matrix2.rb: filter groups to make compare faster
by Li Yuanchao 31 Dec '20

31 Dec '20

When do compare, after group jobs, there are many one_size group, which can not find another job to make a group. Delete such groups to save time. But, if there is no size > 1 group, 3 one_size groups will be kept to avoid 'No Data'. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/matrix2.rb | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/lib/matrix2.rb b/lib/matrix2.rb index 14e9558..0e80858 100644 --- a/lib/matrix2.rb +++ b/lib/matrix2.rb @@ -123,18 +123,37 @@ end def combine_group_query_data(job_list, dims) suites_hash = {} groups = auto_group(job_list, dims) + have_multi_member = multi_member?(groups) + one_size_count = 0 groups.each do |group_key, value| - suite_list = [] - value.each do |dimension_key, jobs| - groups[group_key][dimension_key], suites = create_matrix(jobs) - suite_list.concat(suites) + if value.size < 2 + one_size_count += 1 + if have_multi_member || one_size_count > 3 + groups.delete(group_key) + next + end end - suites_hash[group_key] = suite_list + get_groups_matrix(groups, group_key, value, suites_hash) end - return groups, suites_hash end +def get_groups_matrix(groups, group_key, value, suites_hash) + suite_list = [] + value.each do |dimension_key, jobs| + groups[group_key][dimension_key], suites = create_matrix(jobs) + suite_list.concat(suites) + end + suites_hash[group_key] = suite_list +end + +def multi_member?(groups) + groups.each do |_key, value| + return true if value.size > 1 + end + return false +end + # input: # 1. query results(job_list) from es_query that will be auto group by auto_group_by_template() # 2. params from user's template include: -- 2.23.0

2 3

[PATCH v4 compass-ci] providers/qemu: refactor kvm script
by Liu Yinsi 31 Dec '20

31 Dec '20

encapsulating into functions. test: /srv/result/host-info/2020-12-30/vm-2p8g/openeuler-20.09-aarch64/z9.275381 Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 160 +++++++++++++++++++++++++----------------- 1 file changed, 97 insertions(+), 63 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index fd921dc..0845099 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -7,78 +7,112 @@ : ${nr_cpu:=1} : ${memory:=1G} -log_file=/srv/cci/serial/logs/${hostname} -if [ ! -f "$log_file" ]; then - touch $log_file - # fluentd refresh time is 1s - # let fluentd to monitor this file first - sleep 2 -fi +check_logfile() +{ + log_file=/srv/cci/serial/logs/${hostname} + [ ! -f "$log_file" ] || { + touch $log_file + # fluentd refresh time is 1s + # let fluentd to monitor this file first + sleep 2 + } +} + +qemu_command() +{ + qemu=qemu-system-aarch64 + command -v $qemu >/dev/null || qemu=qemu-kvm +} -qemu=qemu-system-aarch64 -command -v $qemu >/dev/null || qemu=qemu-kvm +write_logfile() +{ + ipxe_script=ipxe_script + curl http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/boot.ipxe/mac/${mac} > $ipxe_script + cat $ipxe_script >> ${log_file} +} -ipxe_script=ipxe_script -curl http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/boot.ipxe/mac/${mac} > $ipxe_script -cat $ipxe_script >> ${log_file} +parse_ipxe_script() +{ + append= + initrds= + while read a b c + do + case "$a" in + '#') + ;; + initrd) + file=$(basename "$b") + wget --timestamping -a ${log_file} --progress=bar:force $b + initrds+="$file " + ;; + kernel) + kernel=$(basename "$b") + wget --timestamping -a ${log_file} --progress=bar:force $b + append=$(echo "$c" | sed -r "s/ initrd=[^ ]+//g") + ;; + *) + ;; + esac + done < $ipxe_script +} -append= -initrds= -while read a b c -do - case "$a" in - '#') - ;; - initrd) - file=$(basename "$b") - wget --timestamping -a ${log_file} --progress=bar:force $b - initrds+="$file " - ;; - kernel) - kernel=$(basename "$b") - wget --timestamping -a ${log_file} --progress=bar:force $b - append=$(echo "$c" | sed -r "s/ initrd=[^ ]+//g") - ;; - *) - ;; - esac -done < $ipxe_script +check_option_value() +{ + [ -s "$kernel" ] || { + echo "can't get kernel or kernel size is 0" + exit + } + + [ -n "$initrds" ] || exit +} -[ -s "$kernel" ] || { - echo "can't get kernel or kernel size is 0" - exit +get_initrd() +{ + initrd=initrd + cat $initrds > $initrd } -[ -n "$initrds" ] || { - exit +print_message() +{ + echo $SCHED_PORT + echo kernel: $kernel + echo initrds: $initrds + echo append: $append + echo less $log_file + + sleep 5 } -initrd=initrd -cat $initrds > $initrd +run_qemu() +{ + kvm=( + $qemu + -machine virt-4.0,accel=kvm,gic-version=3 + -kernel $kernel + -initrd $initrd + -smp $nr_cpu + -m $memory + -cpu Kunpeng-920 + -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd + -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} + -k en-us + -no-reboot + -nographic + -serial file:${log_file} + -monitor null + ) + + "${kvm[@]}" --append "${append}" +} -echo $SCHED_PORT -echo kernel: $kernel -echo initrds: $initrds -echo append: $append -echo less $log_file +check_logfile +write_logfile -sleep 5 +qemu_command +parse_ipxe_script +check_option_value +get_initrd -kvm=( - $qemu - -machine virt-4.0,accel=kvm,gic-version=3 - -kernel $kernel - -initrd $initrd - -smp $nr_cpu - -m $memory - -cpu Kunpeng-920 - -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd - -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} - -k en-us - -no-reboot - -nographic - -serial file:${log_file} - -monitor null -) +print_message -"${kvm[@]}" --append "${append}" +run_qemu -- 2.23.0

2 2

[PATCH v2 lkp-tests] tests/build-pkg: fix to generate the directory of *.cgz file
by Liu Shaofei 31 Dec '20

31 Dec '20

[why] Since the file upload path saved by ES is inconsistent with the file path generated by the build-pkg script, the *.cgz file upload failed. [error] ==> /tmp/stdout <== renamed '/initrd/build-pkg/nfs/openeuler/aarch64/20.03-SP1/kubelet/HEAD.cgz.tmp' -> '/initrd/build-pkg/nfs/openeuler/aarch64/20.03-SP1/kubelet/HEAD.cgz' ==> Leaving fakeroot environment. ==> Finished making: kubernetes 1.18.5-1 (Wed Dec 30 16:50:17 CST 2020) create soft link: latest.cgz -> HEAD.cgz ==> /tmp/stderr <== curl: (22) The requested URL returned error: 403 Forbidden Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- tests/build-pkg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/build-pkg b/tests/build-pkg index 801649cf4..436d43840 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -88,6 +88,8 @@ request_pkg() create_cgz_name() { + [ -n "${pkgbase}" ] && pkgname="${pkgbase}" + cgz_path="$PKG_MNT/${pack_to}/${pkgname}" cgz_name="${cgz_path}/${upstream_commit}.cgz" pkg_args="-A --check --skippgpcheck" -- 2.23.0

1 0

[PATCH lkp-tests] api-avx2neon/PKGBUILD: add multi arch build
by Wang Yong 31 Dec '20

31 Dec '20

build packages through $(arch) Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- pkg/api-avx2neon/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api-avx2neon/PKGBUILD b/pkg/api-avx2neon/PKGBUILD index 3ccebca86..fac047add 100644 --- a/pkg/api-avx2neon/PKGBUILD +++ b/pkg/api-avx2neon/PKGBUILD @@ -9,7 +9,7 @@ md5sums=('SKIP') build() { cd "$srcdir/AvxToNeon/tests" - make + make ARCH=$(arch) } package() { -- 2.23.0

1 0

[PATCH lkp-tests 2/2] pkg: add httpress
by Wei Jihui 31 Dec '20

31 Dec '20

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- pkg/httpress/PKGBUILD | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 pkg/httpress/PKGBUILD diff --git a/pkg/httpress/PKGBUILD b/pkg/httpress/PKGBUILD new file mode 100644 index 00000000..b906303d --- /dev/null +++ b/pkg/httpress/PKGBUILD @@ -0,0 +1,17 @@ +pkgname=httpress +pkgver=1 +pkgrel=1 +url='' +arch=('aarch64') +license=('BSD') +source=("https://download.fedoraproject.org/pub/epel/7/SRPMS/Packages/h/httpress-1.1…") +md5sums=('SKIP') + +package() +{ + # The rpm which from source maybe unavailable + # httpress-1.1.0-13.fc31.aarch64.rpm is validated to be available + rpmpkg_path="${pkgdir}/opt/rpms/" + mkdir -p "${rpmpkg_path}" + cp "${srcdir}/"*.rpm "${rpmpkg_path}" +} -- 2.23.0

2 1

[PATCH v2 lkp-tests 2/2] pkg: add httpress
by Wei Jihui 31 Dec '20

31 Dec '20

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- pkg/httpress/PKGBUILD | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 pkg/httpress/PKGBUILD diff --git a/pkg/httpress/PKGBUILD b/pkg/httpress/PKGBUILD new file mode 100644 index 00000000..44af00ea --- /dev/null +++ b/pkg/httpress/PKGBUILD @@ -0,0 +1,17 @@ +pkgname=httpress +pkgver=1 +pkgrel=1 +url='' +arch=('aarch64') +license=('BSD') +source=("https://download.fedoraproject.org/pub/epel/7/SRPMS/Packages/h/httpress-1.1…") +md5sums=('SKIP') + +package() +{ + # The rpm from source maybe bot be available + # httpress-1.1.0-13.fc31.aarch64.rpm has been validated to be available + rpmpkg_path="${pkgdir}/opt/rpms/" + mkdir -p "${rpmpkg_path}" + cp "${srcdir}/"*.rpm "${rpmpkg_path}" +} -- 2.23.0

2 1

[PATCH v2 lkp-tests 1/2] distro/depends: add httpress
by Wei Jihui 31 Dec '20

31 Dec '20

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- distro/depends/httpress | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 distro/depends/httpress diff --git a/distro/depends/httpress b/distro/depends/httpress new file mode 100644 index 00000000..73fc57c6 --- /dev/null +++ b/distro/depends/httpress @@ -0,0 +1,2 @@ +numactl +ethtool -- 2.23.0

1 0

[PATCH v2 lkp-tests] optimize the deploy-cci script
by Hu Xuejiao 31 Dec '20

31 Dec '20

Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- tests/deploy-cci | 52 ++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/tests/deploy-cci b/tests/deploy-cci index 55835544..8e6acbac 100755 --- a/tests/deploy-cci +++ b/tests/deploy-cci @@ -2,30 +2,38 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -# redirect remote-repository address to "/srv/git/gitee.com" -if [ -n "$GIT_SERVER" ]; then - cat >> /etc/gitconfig <<-EOF - [url "git://$GIT_SERVER/gitee.com"] - insteadOf=https://gitee.com - EOF -fi +export_env() +{ + export initrd_http_host=$INITRD_HTTP_HOST + export initrd_http_port=$INITRD_HTTP_PORT + export gitcache_host=$GITCACHE_HOST + export gitcache_port=$GITCACHE_PORT +} -export initrd_http_host=$INITRD_HTTP_HOST -export initrd_http_port=$INITRD_HTTP_PORT -export gitcache_host=$GITCACHE_HOST -export gitcache_port=$GITCACHE_PORT +deploy() +{ + # redirect remote-repository address to "/srv/git/gitee.com" + if [ -n "$GIT_SERVER" ]; then + cat >> /etc/gitconfig <<-EOF + [url "git://$GIT_SERVER/gitee.com"] + insteadOf=https://gitee.com + EOF + fi -mkdir -p /c -cd /c || exit -git clone https://gitee.com/wu_fengguang/compass-ci.git + mkdir -p /c && cd /c + git clone https://gitee.com/wu_fengguang/compass-ci.git -cd /c/compass-ci || exit -[ -n "$compass_commitid" ] && git reset --hard "$compass_commitid" + cd /c/compass-ci || exit + [ -n "$compass_commit_id" ] && git reset --hard "$compass_commit_id" -# Need mount tmpfs to fix compass-ci/container/os-nfs/start error -# "exportfs: /exports/result does not support NFS export" -mount -t tmpfs none /srv + # Need mount tmpfs to fix compass-ci/container/os-nfs/start error + # "exportfs: /exports/result does not support NFS export" + mount -t tmpfs none /srv -# test the script for one-click deployment -cd /c/compass-ci/sparrow || exit -./install-tiny + # test the script for one-click deployment + cd /c/compass-ci/sparrow || exit + ./install-tiny +} + +export_env +deploy -- 2.23.0

2 1

[PATCH lkp-tests 1/2] distro/depends: add httpress
by Wei Jihui 30 Dec '20

30 Dec '20

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- distro/depends/httpress | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 distro/depends/httpress diff --git a/distro/depends/httpress b/distro/depends/httpress new file mode 100644 index 00000000..73fc57c6 --- /dev/null +++ b/distro/depends/httpress @@ -0,0 +1,2 @@ +numactl +ethtool -- 2.23.0

1 0

[PATCH compass-ci] 3-code/dev-env: add variables
by Hu Xuejiao 30 Dec '20

30 Dec '20

[why] $GIT_SERVER will be defined in job.yaml $LKP_SERVER is ip in test-machine for upload_files Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sparrow/3-code/dev-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 750738f..ac38f56 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -23,6 +23,8 @@ mkdir -p /etc/compass-ci/defaults cat > /etc/compass-ci/defaults/$server_name.yaml <<EOF SCHED_HOST: $sched_host SCHED_PORT: $sched_port +LKP_SERVER: $server_ip +GIT_SERVER: $GIT_SERVER TASKQUEUE_HOST: $sched_host OS_HTTP_HOST: $os_http_host INITRD_HTTP_HOST: $initrd_http_host -- 2.23.0

1 0

[PATCH v4 compass-ci] sbin/run_job: Automatic task execution
by Hu Xuejiao 30 Dec '20

30 Dec '20

[purpose] Verify that a local job can be consumed properly Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sbin/run_job | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100755 sbin/run_job diff --git a/sbin/run_job b/sbin/run_job new file mode 100755 index 0000000..c84df0d --- /dev/null +++ b/sbin/run_job @@ -0,0 +1,74 @@ +#!/bin/bash + +cci_makepkg_file="$LKP_SRC/jobs/cci-makepkg.yaml" +grep "atomic" $cci_makepkg_file || sed -i 's|benchmark:|& atomic|g' $cci_makepkg_file + +submit_one_job() +{ + cmd=( + submit -m $yaml + os=${single_arg[0]} + os_arch=${single_arg[1]} + os_version=${single_arg[2]} + os_mount=${single_arg[3]} + docker_images=${single_arg[4]} + testbox=$testbox + queue="${tbox_group}.${HOSTNAME}" + ) + output=$("${cmd[@]}") + echo "$output" | grep -o -E 'job_state=finished[^\"]*' +} + +submit_job() +{ + yaml_args=("$cci_makepkg_file" "host-info.yaml") + for yaml in "${yaml_args[@]}" + do + submit_one_job & + done +} + +run_qemu() +{ + local nr_vm=4 + local testbox="vm-2p8g" + local tbox_group="vm-2p8g" + local hostname="${tbox_group}.${HOSTNAME}" + local queues="${tbox_group}.${HOSTNAME}" + + $CCI_SRC/providers/multi-qemu -n "$hostname" -c "$nr_vm" -q "$queues" + + qemu_args=( + "openeuler aarch64 20.03 cifs" + "centos aarch64 7.6.1810 initramfs" + ) + for qemu_arg in "${qemu_args[@]}" + do + local single_arg=($(echo $qemu_arg)) + submit_job + done +} + +run_docker() +{ + local nr_container=2 + local tbox_group="dc-8g" + local testbox="dc-8g" + local hostname="${tbox_group}.${HOSTNAME}" + local queues="${tbox_group}.${HOSTNAME}" + + $CCI_SRC/providers/multi-docker -n $hostname -c $nr_container -q $queues + + docker_args=( + "centos aarch64 7 container" + ) + for docker_arg in "${docker_args[@]}" + do + local single_arg=($(echo $docker_arg)) + submit_job + done +} + +run_qemu +run_docker +wait -- 2.23.0

1 0

[PATCH v2 compass-ci] lib/mail_bisect_result: optimize bisect report
by Cao Xueliang 30 Dec '20

30 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- doc/bisect_email.en.md | 28 ++++++++++++++++++++++++++++ lib/constants.rb | 2 ++ lib/mail_bisect_result.rb | 28 ++++++++++++++++++---------- 3 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 doc/bisect_email.en.md diff --git a/doc/bisect_email.en.md b/doc/bisect_email.en.md new file mode 100644 index 0000000..fee7ee1 --- /dev/null +++ b/doc/bisect_email.en.md @@ -0,0 +1,28 @@ +# Who are we? + +Compass-CI runs a number of tests for open source projects for free. +We aim to improve OSS quality in a number of hardware and OS environment. +We are part of the openEuler community. + +# Why do you get the email? + +Whenever you git push, our test robot will pull and test the new code. +When new errors are found, Compass-CI will auto run git bisect to find out the first bad commit, +and send email report to the commit author. + +# About the email + +The first bad commit job result directory: +you can read the below file to understand the content of each file. +- https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/browse-res… + +Notice: +You may need to read the output file of the first bad commit job result directory, +it is the build log about the first bad commit. + +# how to confirm bug fix + +- git push your fix patch to your project. +- download the job.yaml from the first bad commit job directory. +- submit -m job.yaml upstream_commit=$your_fix_commit. +- check result from the new result directory which is in output above command and key is result_root. diff --git a/lib/constants.rb b/lib/constants.rb index f0938f2..c3c7543 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -13,3 +13,5 @@ MAIL_PORT = config['SEND_MAIL_PORT'] || 49000 SRV_HTTP_HOST = config['SRV_HTTP_HOST'] || ENV['SRV_HTTP_HOST'] || '172.17.0.1' SRV_HTTP_PORT = config['SRV_HTTP_PORT'] || ENV['SRV_HTTP_PORT'] || 11300 + +SRV_HTTP_DOMAIN = config['SRV_HTTP_DOMAIN'] || ENV['SRV_HTTP_DOMAIN'] || 'api.compass-ci.openeuler.org' diff --git a/lib/mail_bisect_result.rb b/lib/mail_bisect_result.rb index 5f88c36..f0f0194 100644 --- a/lib/mail_bisect_result.rb +++ b/lib/mail_bisect_result.rb @@ -3,6 +3,8 @@ require 'json' require 'base64' require_relative 'git' +require_relative 'es_query' +require_relative 'constants' require_relative 'mail_client' require_relative 'assign_account_client' @@ -27,10 +29,11 @@ class MailBisectResult def compose_mail subject = "[Compass-CI][#{(a)repo.split('/')[1]}] #{@commit_id[0..9]} #{(a)bisect_error[0].split("\n")[0]}" - prefix_srv = "http://#{ENV['SRV_HTTP_HOST']}:#{ENV['SRV_HTTP_PORT']}" - bisect_job_url = ENV['result_root'] ? "bisect job info: #{prefix_srv}#{ENV['result_root']}\n" : '' - pkgbuild_repo_url = "PKGBUILD info: #{prefix_srv}/#{@pkgbuild_repo}\n" - first_bad_commit_job_url = "first bad commit job info: #{prefix_srv}#{@first_bad_commit_result_root}\n" + prefix_srv = "http://#{SRV_HTTP_DOMAIN}:#{SRV_HTTP_PORT}" + bisect_job_url = ENV['result_root'] ? "bisect job result directory:\n#{prefix_srv}#{ENV['result_root']}\n" : '' + bisect_report_doc = "bisect email doc:\nhttps://gitee.com/wu_fengguang/compass-ci/blob/master/doc/bisect_emai…" + pkgbuild_repo_url = "PKGBUILD:\n#{prefix_srv}/#{@pkgbuild_repo}\n" + first_bad_commit_job_url = "first bad commit job result directory:\n#{prefix_srv}#{@first_bad_commit_result_root}\n" data = <<~BODY To: #{@to} @@ -38,17 +41,19 @@ class MailBisectResult Hi #{(a)git_commit.author_name}, - url: #{(a)git_commit.url} - commit: #{@commit_id} ("#{(a)git_commit.subject}") - compiler: gcc (GCC) 7.3.0 + We found some error/warning(s) and the first bad commit in the below project: + git url: #{(a)git_commit.url} + git commit: #{@commit_id} ("#{(a)git_commit.subject}") - all errors/warnings (new ones prefixed by >>): + All error/warning(s) (new ones prefixed by >>): #{@all_errors} - reference information: + Reference information: + compiler: gcc (GCC) 7.3.0 #{pkgbuild_repo_url} - #{bisect_job_url} #{first_bad_commit_job_url} + #{bisect_job_url} + #{bisect_report_doc} Regards, Compass CI team BODY @@ -67,6 +72,9 @@ class MailBisectResult 'my_commit_url' => "#{@git_commit.url}/commit/#{@commit_id}" } + account_info = ESQuery.new(index: 'accounts').query_by_id(@to) + return if account_info + apply_account = AutoAssignAccount.new(user_info) apply_account.send_account end -- 2.23.0

1 0

[PATCH compass-ci] lib/mail_bisect_result: optimize bisect report
by Cao Xueliang 30 Dec '20

30 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/mail_bisect_result.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/mail_bisect_result.rb b/lib/mail_bisect_result.rb index 5f88c36..4883072 100644 --- a/lib/mail_bisect_result.rb +++ b/lib/mail_bisect_result.rb @@ -3,6 +3,7 @@ require 'json' require 'base64' require_relative 'git' +require_relative 'es_query' require_relative 'mail_client' require_relative 'assign_account_client' @@ -28,9 +29,9 @@ class MailBisectResult def compose_mail subject = "[Compass-CI][#{(a)repo.split('/')[1]}] #{@commit_id[0..9]} #{(a)bisect_error[0].split("\n")[0]}" prefix_srv = "http://#{ENV['SRV_HTTP_HOST']}:#{ENV['SRV_HTTP_PORT']}" - bisect_job_url = ENV['result_root'] ? "bisect job info: #{prefix_srv}#{ENV['result_root']}\n" : '' - pkgbuild_repo_url = "PKGBUILD info: #{prefix_srv}/#{@pkgbuild_repo}\n" - first_bad_commit_job_url = "first bad commit job info: #{prefix_srv}#{@first_bad_commit_result_root}\n" + bisect_job_url = ENV['result_root'] ? "bisect job result directory: #{prefix_srv}#{ENV['result_root']}\n" : '' + pkgbuild_repo_url = "PKGBUILD file directory: #{prefix_srv}/#{@pkgbuild_repo}\n" + first_bad_commit_job_url = "first bad commit job result directory: #{prefix_srv}#{@first_bad_commit_result_root}\n" data = <<~BODY To: #{@to} @@ -46,9 +47,9 @@ class MailBisectResult #{@all_errors} reference information: + #{first_bad_commit_job_url} #{pkgbuild_repo_url} #{bisect_job_url} - #{first_bad_commit_job_url} Regards, Compass CI team BODY @@ -67,6 +68,9 @@ class MailBisectResult 'my_commit_url' => "#{@git_commit.url}/commit/#{@commit_id}" } + account_info = ESQuery.new(index: 'accounts').query_by_id(@to) + return if account_info + apply_account = AutoAssignAccount.new(user_info) apply_account.send_account end -- 2.23.0

1 1

[PATCH v1 compass-ci] lib/mail_bisect_result: optimize bisect report
by Cao Xueliang 30 Dec '20

30 Dec '20

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/constants.rb | 2 ++ lib/mail_bisect_result.rb | 28 ++++++++++++++++++---------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/lib/constants.rb b/lib/constants.rb index f0938f2..c3c7543 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -13,3 +13,5 @@ MAIL_PORT = config['SEND_MAIL_PORT'] || 49000 SRV_HTTP_HOST = config['SRV_HTTP_HOST'] || ENV['SRV_HTTP_HOST'] || '172.17.0.1' SRV_HTTP_PORT = config['SRV_HTTP_PORT'] || ENV['SRV_HTTP_PORT'] || 11300 + +SRV_HTTP_DOMAIN = config['SRV_HTTP_DOMAIN'] || ENV['SRV_HTTP_DOMAIN'] || 'api.compass-ci.openeuler.org' diff --git a/lib/mail_bisect_result.rb b/lib/mail_bisect_result.rb index 5f88c36..bc3c74a 100644 --- a/lib/mail_bisect_result.rb +++ b/lib/mail_bisect_result.rb @@ -3,6 +3,8 @@ require 'json' require 'base64' require_relative 'git' +require_relative 'es_query' +require_relative 'constants' require_relative 'mail_client' require_relative 'assign_account_client' @@ -27,10 +29,11 @@ class MailBisectResult def compose_mail subject = "[Compass-CI][#{(a)repo.split('/')[1]}] #{@commit_id[0..9]} #{(a)bisect_error[0].split("\n")[0]}" - prefix_srv = "http://#{ENV['SRV_HTTP_HOST']}:#{ENV['SRV_HTTP_PORT']}" - bisect_job_url = ENV['result_root'] ? "bisect job info: #{prefix_srv}#{ENV['result_root']}\n" : '' - pkgbuild_repo_url = "PKGBUILD info: #{prefix_srv}/#{@pkgbuild_repo}\n" - first_bad_commit_job_url = "first bad commit job info: #{prefix_srv}#{@first_bad_commit_result_root}\n" + prefix_srv = "http://#{SRV_HTTP_DOMAIN}:#{SRV_HTTP_PORT}" + bisect_job_url = ENV['result_root'] ? "bisect job result directory:\n#{prefix_srv}#{ENV['result_root']}\n" : '' + bisect_report_doc = "bisect report:\nhttps://gitee.com/wu_fengguang/compass-ci/blob/master/doc/bisect_r…" + pkgbuild_repo_url = "PKGBUILD:\n#{prefix_srv}/#{@pkgbuild_repo}\n" + first_bad_commit_job_url = "first bad commit job result directory:\n#{prefix_srv}#{@first_bad_commit_result_root}\n" data = <<~BODY To: #{@to} @@ -38,17 +41,19 @@ class MailBisectResult Hi #{(a)git_commit.author_name}, - url: #{(a)git_commit.url} - commit: #{@commit_id} ("#{(a)git_commit.subject}") - compiler: gcc (GCC) 7.3.0 + We found some error/warning(s) and the first bad commit in the below project: + git url: #{(a)git_commit.url} + git commit: #{@commit_id} ("#{(a)git_commit.subject}") - all errors/warnings (new ones prefixed by >>): + All error/warning(s) (new ones prefixed by >>): #{@all_errors} - reference information: + Reference information: + compiler: gcc (GCC) 7.3.0 #{pkgbuild_repo_url} - #{bisect_job_url} #{first_bad_commit_job_url} + #{bisect_job_url} + #{bisect_report_doc} Regards, Compass CI team BODY @@ -67,6 +72,9 @@ class MailBisectResult 'my_commit_url' => "#{@git_commit.url}/commit/#{@commit_id}" } + account_info = ESQuery.new(index: 'accounts').query_by_id(@to) + return if account_info + apply_account = AutoAssignAccount.new(user_info) apply_account.send_account end -- 2.23.0

1 0

[PATCH lkp-tests] tests/build-pkg: add ${config} judge in common job
by Wang Yong 30 Dec '20

30 Dec '20

if not exist ${config} in general build-pkg job, ${cci_dir} should not be judeged and do not need upload file Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- tests/build-pkg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/build-pkg b/tests/build-pkg index 801649cf4..d356b2f14 100755 --- a/tests/build-pkg +++ b/tests/build-pkg @@ -49,7 +49,9 @@ get_config() upload_config() { local upload_dir="${cci_dir}/${pkgname}" - [ -d "${upload_dir}" ] || return + + [ -n "${config}" ] || return + [ -d "${upload_dir}" ] || return upload_to_target_dir "${upload_dir}" } -- 2.23.0

1 0

[PATCH compass-ci] container/srv-http: import h5ai into container
by Lu Weitao 30 Dec '20

30 Dec '20

[Why] Optimize result page style to make it more modern Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/srv-http/Dockerfile | 16 +++++++++++++--- container/srv-http/build | 7 +++++++ .../srv-http/root/etc/nginx/conf.d/default.conf | 5 ++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/container/srv-http/Dockerfile b/container/srv-http/Dockerfile index 8f9a36d..987e40e 100644 --- a/container/srv-http/Dockerfile +++ b/container/srv-http/Dockerfile @@ -1,15 +1,25 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM nginx:alpine +FROM alpine:3.9 + ADD root / -RUN apk add --no-cache php7 php7-fpm php7-session php7-json php7-exif php7-imagick php7-gd php7-fileinfo +RUN apk add --no-cache nginx php7 php7-fpm php7-session php7-json php7-exif php7-imagick php7-gd php7-fileinfo nodejs npm \ + && mkdir /run/nginx/ \ + && mkdir /h5ai RUN sed -i '/\[global\]/a daemonize = no' /etc/php7/php-fpm.conf RUN sed -i "s/user = nobody/user = nginx/g" /etc/php7/php-fpm.d/www.conf RUN sed -i "s/group = nobody/group = nginx/g" /etc/php7/php-fpm.d/www.conf -RUN chown -R nginx:nginx /srv/_h5ai +WORKDIR /h5ai +COPY h5ai /h5ai + +# build h5ai package +RUN npm install \ + && npm run build \ + && cp -r ./build/_h5ai /srv \ + && chown -R nginx:nginx /srv/_h5ai ENTRYPOINT ["/sbin/entrypoint.sh"] diff --git a/container/srv-http/build b/container/srv-http/build index 3a500ea..bdda319 100755 --- a/container/srv-http/build +++ b/container/srv-http/build @@ -2,4 +2,11 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +[[ $H5AI_SRC ]] || H5AI_SRC=/c/h5ai + +cp -r $H5AI_SRC ./ + docker build -t srv-http . + +# clean h5ai foler +rm -rf ./h5ai diff --git a/container/srv-http/root/etc/nginx/conf.d/default.conf b/container/srv-http/root/etc/nginx/conf.d/default.conf index 676418a..646eee3 100644 --- a/container/srv-http/root/etc/nginx/conf.d/default.conf +++ b/container/srv-http/root/etc/nginx/conf.d/default.conf @@ -3,8 +3,8 @@ server { server_name "srv-http"; server_tokens off; - root /usr/share/nginx/html; - index index.html; + root /srv; + index index.html index.php /_h5ai/public/index.php; charset utf-8,gbk; @@ -24,7 +24,6 @@ server { autoindex on; autoindex_localtime on; allow all; - root /srv; location ~* /[-_a-z]+$ { add_header Content-Type text/plain; } -- 2.23.0

1 1

Re: [PATCH v3 compass-ci] providers/qemu: refactor kvm script
by Ren Wen 30 Dec '20

30 Dec '20

>>+ if [ ! -f "$log_file" ]; then >>>+ touch $log_file >>>+ # fluentd refresh time is 1s >>>+ # let fluentd to monitor this file first >>>+ sleep 2 >>>+ fi >> >>how about: [ -f "$log_file" ] || { ... } ? > >there are 2 line codes, if just 1 line code use like you say is good. {} is a way to group a list of commands to be executed as a unit, you can also use like this: [ ] || { expression1 expression2 } Thanks, RenWen > >Thanks, >Yinsi > > >> >>-------- >>Thanks >>Yu Chuan >> >>>+} >>> >>>-qemu=qemu-system-aarch64 >>>-command -v $qemu >/dev/null || qemu=qemu-kvm >>>+qemu_command() >>>+{ >>>+ qemu=qemu-system-aarch64 >>>+ command -v $qemu >/dev/null || qemu=qemu-kvm >>>+} >>> >>>-ipxe_script=ipxe_script >>>-curl http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/boot.ipxe/mac/${mac} > $ipxe_script >>>-cat $ipxe_script >> ${log_file} >>>+write_logfile() >>>+{ >>>+ ipxe_script=ipxe_script >>>+ curl http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/boot.ipxe/mac/${mac} > $ipxe_script >>>+ cat $ipxe_script >> ${log_file} >>>+} >>> >>>-append= >>>-initrds= >>>-while read a b c >>>-do >>>- case "$a" in >>>- '#') >>>- ;; >>>- initrd) >>>- file=$(basename "$b") >>>- wget --timestamping -a ${log_file} --progress=bar:force $b >>>- initrds+="$file " >>>- ;; >>>- kernel) >>>- kernel=$(basename "$b") >>>- wget --timestamping -a ${log_file} --progress=bar:force $b >>>- append=$(echo "$c" | sed -r "s/ initrd=[^ ]+//g") >>>- ;; >>>- *) >>>- ;; >>>- esac >>>-done < $ipxe_script >>>+option_value() >>>+{ >>>+ append= >>>+ initrds= >>>+ while read a b c >>>+ do >>>+ case "$a" in >>>+ '#') >>>+ ;; >>>+ initrd) >>>+ file=$(basename "$b") >>>+ wget --timestamping -a ${log_file} --progress=bar:force $b >>>+ initrds+="$file " >>>+ ;; >>>+ kernel) >>>+ kernel=$(basename "$b") >>>+ wget --timestamping -a ${log_file} --progress=bar:force $b >>>+ append=$(echo "$c" | sed -r "s/ initrd=[^ ]+//g") >>>+ ;; >>>+ *) >>>+ ;; >>>+ esac >>>+ done < $ipxe_script >>>+} >>> >>>-[ -s "$kernel" ] || { >>>- echo "can't get kernel or kernel size is 0" >>>- exit >>>+check_option_value() >>>+{ >>>+ [ -s "$kernel" ] || { >>>+ echo "can't get kernel or kernel size is 0" >>>+ exit >>>+ } >>>+ >>>+ [ -n "$initrds" ] || exit >>>+ >>>+ initrd=initrd >>>+ cat $initrds > $initrd >>> } >>> >>>-[ -n "$initrds" ] || { >>>- exit >>>+print_message() >>>+{ >>>+ echo $SCHED_PORT >>>+ echo kernel: $kernel >>>+ echo initrds: $initrds >>>+ echo append: $append >>>+ echo less $log_file >>>+ >>>+ sleep 5 >>> } >>> >>>-initrd=initrd >>>-cat $initrds > $initrd >>>+run_qemu() >>>+{ >>>+ kvm=( >>>+ $qemu >>>+ -machine virt-4.0,accel=kvm,gic-version=3 >>>+ -kernel $kernel >>>+ -initrd $initrd >>>+ -smp $nr_cpu >>>+ -m $memory >>>+ -cpu Kunpeng-920 >>>+ -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd >>>+ -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} >>>+ -k en-us >>>+ -no-reboot >>>+ -nographic >>>+ -serial file:${log_file} >>>+ -monitor null >>>+ ) >>>+ >>>+ "${kvm[@]}" --append "${append}" >>>+} >>> >>>-echo $SCHED_PORT >>>-echo kernel: $kernel >>>-echo initrds: $initrds >>>-echo append: $append >>>-echo less $log_file >>>+check_logfile >>>+write_logfile >>> >>>-sleep 5 >>>+qemu_command >>>+option_value >>>+check_option_value >>> >>>-kvm=( >>>- $qemu >>>- -machine virt-4.0,accel=kvm,gic-version=3 >>>- -kernel $kernel >>>- -initrd $initrd >>>- -smp $nr_cpu >>>- -m $memory >>>- -cpu Kunpeng-920 >>>- -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd >>>- -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} >>>- -k en-us >>>- -no-reboot >>>- -nographic >>>- -serial file:${log_file} >>>- -monitor null >>>-) >>>+print_message >>> >>>-"${kvm[@]}" --append "${append}" >>>+run_qemu >>>-- >>>2.23.0 >>>

2 1

[PATCH v3 compass-ci] lib/es_query.rb: get multi-field aggregation count result
by Lu Kaiyi 30 Dec '20

30 Dec '20

[why] get multi-field aggregation count for overview of report page. [input]: fields, query_items (optional for query_items, default no scope limitation) [example] input: fields = ['suite'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186214}, {"key"=>"cci-depends", "doc_count"=>1230}, {"key"=>"iperf", "doc_count"=>1103}, {"key"=>"spinlock", "doc_count"=>1000}, {"key"=>"cci-makepkg", "doc_count"=>949}, {"key"=>"iperf-walk-os-test", "doc_count"=>474}, {"key"=>"bisect", "doc_count"=>381}, ... input: fields = ['suite', 'job_state'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186227, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3830}, {"key"=>"finished", "doc_count"=>803}, {"key"=>"incomplete", "doc_count"=>196}, {"key"=>"submit", "doc_count"=>136}, ... input: fields = ['os', 'os_version', 'job_state'] query_items = {} output: [{"key"=>"openeuler", "doc_count"=>193287, "all_os_version"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"20.03", "doc_count"=>191789, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3909}, {"key"=>"finished", "doc_count"=>2401}, {"key"=>"incomplete", "doc_count"=>300}, {"key"=>"submit", "doc_count"=>136}, {"key"=>"OOM", "doc_count"=>23}, {"key"=>"post_run", "doc_count"=>1}]}}, {"key"=>"20.03-SP1", "doc_count"=>904, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>641}, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/lib/es_query.rb b/lib/es_query.rb index 88a420f..29d50e9 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -87,6 +87,50 @@ class ESQuery result end + + # input: + # fields, query_items + # + # fields => fields: Array(keyword) + # eg: + # ['suite', 'job_state'] + # + # query_items => query_items: Hash(keyword, value) + # eg: + # { + # 'os' => 'openeuler', + # ... + # } + # (optional for query_items, default no scope limitation) + # output: + # [{"key"=>"build-pkg", + # "doc_count"=>186175, + # "all_job_state"=> + # {"doc_count_error_upper_bound"=>0, + # "sum_other_doc_count"=>0, + # "buckets"=> + # [{"key"=>"failed", "doc_count"=>3830}, + # {"key"=>"finished", "doc_count"=>803}, + # {"key"=>"incomplete", "doc_count"=>196}, + # ... + def query_fields(fields, query_items = {}) + field1 = fields.first + aggs_hash = {} + build_aggs_from_fields(fields, aggs_hash) + query = { + query: { + bool: { + must: build_multi_field_subquery_body(query_items) + } + }, + aggs: aggs_hash['aggs'], + size: 0 + } + result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] + return nil if result.empty? + + result + end end # Range Query Example: @@ -146,3 +190,30 @@ def assign_desc_body(keyword) }] } end + +# input: +# fields = ['os', 'os_version', 'job_state'] +# aggs_hash = {} +# output: +# aggs_hash = { +# "aggs"=> +# {"all_os"=> +# {"terms"=>{:field=>"os", :size=>1000}, +# "aggs"=> +# {"all_os_version"=> +# {"terms"=>{:field=>"os_version", :size=>1000}, +# "aggs"=> +# {"all_job_state"=>{"terms"=>{:field=>"job_state", :size=>1000}}}}}}} +# } +def build_aggs_from_fields(fields, aggs_hash) + return if fields.empty? + + field = fields.shift + aggs_hash['aggs'] = { + "all_#{field}" => { + 'terms' => { field: field, size: 1000 } + } + } + + build_aggs_from_fields(fields, aggs_hash['aggs']["all_#{field}"]) +end -- 2.23.0

1 0

[PATCH v3 compass-ci] providers/qemu: refactor kvm script
by Liu Yinsi 30 Dec '20

30 Dec '20

encapsulating into functions. test: /srv/result/host-info/2020-12-30/vm-2p8g/openeuler-20.09-aarch64/z9.275242 Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- providers/qemu/kvm.sh | 156 +++++++++++++++++++++++++----------------- 1 file changed, 93 insertions(+), 63 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index fd921dc..c48c0c7 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -7,78 +7,108 @@ : ${nr_cpu:=1} : ${memory:=1G} -log_file=/srv/cci/serial/logs/${hostname} -if [ ! -f "$log_file" ]; then - touch $log_file - # fluentd refresh time is 1s - # let fluentd to monitor this file first - sleep 2 -fi +check_logfile() +{ + log_file=/srv/cci/serial/logs/${hostname} + if [ ! -f "$log_file" ]; then + touch $log_file + # fluentd refresh time is 1s + # let fluentd to monitor this file first + sleep 2 + fi +} -qemu=qemu-system-aarch64 -command -v $qemu >/dev/null || qemu=qemu-kvm +qemu_command() +{ + qemu=qemu-system-aarch64 + command -v $qemu >/dev/null || qemu=qemu-kvm +} -ipxe_script=ipxe_script -curl http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/boot.ipxe/mac/${mac} > $ipxe_script -cat $ipxe_script >> ${log_file} +write_logfile() +{ + ipxe_script=ipxe_script + curl http://${SCHED_HOST:-172.17.0.1}:${SCHED_PORT:-3000}/boot.ipxe/mac/${mac} > $ipxe_script + cat $ipxe_script >> ${log_file} +} -append= -initrds= -while read a b c -do - case "$a" in - '#') - ;; - initrd) - file=$(basename "$b") - wget --timestamping -a ${log_file} --progress=bar:force $b - initrds+="$file " - ;; - kernel) - kernel=$(basename "$b") - wget --timestamping -a ${log_file} --progress=bar:force $b - append=$(echo "$c" | sed -r "s/ initrd=[^ ]+//g") - ;; - *) - ;; - esac -done < $ipxe_script +option_value() +{ + append= + initrds= + while read a b c + do + case "$a" in + '#') + ;; + initrd) + file=$(basename "$b") + wget --timestamping -a ${log_file} --progress=bar:force $b + initrds+="$file " + ;; + kernel) + kernel=$(basename "$b") + wget --timestamping -a ${log_file} --progress=bar:force $b + append=$(echo "$c" | sed -r "s/ initrd=[^ ]+//g") + ;; + *) + ;; + esac + done < $ipxe_script +} -[ -s "$kernel" ] || { - echo "can't get kernel or kernel size is 0" - exit +check_option_value() +{ + [ -s "$kernel" ] || { + echo "can't get kernel or kernel size is 0" + exit + } + + [ -n "$initrds" ] || exit + + initrd=initrd + cat $initrds > $initrd } -[ -n "$initrds" ] || { - exit +print_message() +{ + echo $SCHED_PORT + echo kernel: $kernel + echo initrds: $initrds + echo append: $append + echo less $log_file + + sleep 5 } -initrd=initrd -cat $initrds > $initrd +run_qemu() +{ + kvm=( + $qemu + -machine virt-4.0,accel=kvm,gic-version=3 + -kernel $kernel + -initrd $initrd + -smp $nr_cpu + -m $memory + -cpu Kunpeng-920 + -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd + -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} + -k en-us + -no-reboot + -nographic + -serial file:${log_file} + -monitor null + ) + + "${kvm[@]}" --append "${append}" +} -echo $SCHED_PORT -echo kernel: $kernel -echo initrds: $initrds -echo append: $append -echo less $log_file +check_logfile +write_logfile -sleep 5 +qemu_command +option_value +check_option_value -kvm=( - $qemu - -machine virt-4.0,accel=kvm,gic-version=3 - -kernel $kernel - -initrd $initrd - -smp $nr_cpu - -m $memory - -cpu Kunpeng-920 - -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd - -nic tap,model=virtio-net-pci,helper=/usr/libexec/qemu-bridge-helper,br=br0,mac=${mac} - -k en-us - -no-reboot - -nographic - -serial file:${log_file} - -monitor null -) +print_message -"${kvm[@]}" --append "${append}" +run_qemu -- 2.23.0

3 4

[PATCH v2 compass-ci] lib/es_query.rb: get multi-field aggregation count result
by Lu Kaiyi 30 Dec '20

30 Dec '20

[why] get multi-field aggregation count for overview of report page. [input]: fields, query_items (optional for query_items, default no scope limitation) [example1] input: fields = ['suite'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186214}, {"key"=>"cci-depends", "doc_count"=>1230}, {"key"=>"iperf", "doc_count"=>1103}, {"key"=>"spinlock", "doc_count"=>1000}, {"key"=>"cci-makepkg", "doc_count"=>949}, {"key"=>"iperf-walk-os-test", "doc_count"=>474}, {"key"=>"bisect", "doc_count"=>381}, ... [example2] input: fields = ['suite', 'job_state'] query_items = { 'os' => 'openeuler' } output: [{"key"=>"build-pkg", "doc_count"=>186227, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3830}, {"key"=>"finished", "doc_count"=>803}, {"key"=>"incomplete", "doc_count"=>196}, {"key"=>"submit", "doc_count"=>136}, ... [example3] input: fields = ['os', 'os_version', 'job_state'] query_items = {} output: [{"key"=>"openeuler", "doc_count"=>193287, "all_os_version"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"20.03", "doc_count"=>191789, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>3909}, {"key"=>"finished", "doc_count"=>2401}, {"key"=>"incomplete", "doc_count"=>300}, {"key"=>"submit", "doc_count"=>136}, {"key"=>"OOM", "doc_count"=>23}, {"key"=>"post_run", "doc_count"=>1}]}}, {"key"=>"20.03-SP1", "doc_count"=>904, "all_job_state"=> {"doc_count_error_upper_bound"=>0, "sum_other_doc_count"=>0, "buckets"=> [{"key"=>"failed", "doc_count"=>641}, ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_query.rb | 52 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/lib/es_query.rb b/lib/es_query.rb index 88a420f..a9b651e 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -87,6 +87,45 @@ class ESQuery result end + + # select field1, field2, count(*) from index where query_items group by field1, field2 + # example: + # fields = ['suite', 'job_state'] + # query_items = { + # 'os' => 'openeuler' + # } + # input: + # fields, query_items + # (optional for query_items, default no scope limitation) + # output: + # [{"key"=>"build-pkg", + # "doc_count"=>186175, + # "all_job_state"=> + # {"doc_count_error_upper_bound"=>0, + # "sum_other_doc_count"=>0, + # "buckets"=> + # [{"key"=>"failed", "doc_count"=>3830}, + # {"key"=>"finished", "doc_count"=>803}, + # {"key"=>"incomplete", "doc_count"=>196}, + # ... + def query_fields(fields, query_items = {}) + field1 = fields.first + aggs_hash = {} + build_aggs_from_fields(fields, aggs_hash) + query = { + query: { + bool: { + must: build_multi_field_subquery_body(query_items) + } + }, + aggs: aggs_hash['aggs'], + size: 0 + } + result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets'] + return nil if result.empty? + + result + end end # Range Query Example: @@ -146,3 +185,16 @@ def assign_desc_body(keyword) }] } end + +def build_aggs_from_fields(fields, aggs_hash) + return if fields.empty? + + field = fields.shift + aggs_hash['aggs'] = { + "all_#{field}" => { + 'terms' => { field: field, size: 1000 } + } + } + + build_aggs_from_fields(fields, aggs_hash['aggs']["all_#{field}"]) +end -- 2.23.0

2 1

[PATCH lkp-tests] distro/common: fix the scenario when the directory not exist
by Liu Shaofei 30 Dec '20

30 Dec '20

Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- distro/common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/distro/common b/distro/common index 166ff8550..1fd7df4e8 100755 --- a/distro/common +++ b/distro/common @@ -87,8 +87,9 @@ upload_to_target_dir() . $LKP_SRC/lib/upload.sh local dir="$1" - local file="$(ls -F ${dir} | grep -v "@")" + [ -d "${dir}" ] || return 0 + local file="$(ls -F ${dir} | grep -v "@")" [ -z "${file}" ] && return 0 upload_one_curl "${dir}/${file}" "${dir}" >/dev/null -- 2.23.0

1 0

[PATCH v2 lkp-tests] lib/monitor: print ssh info
by Wu Zhende 30 Dec '20

30 Dec '20

After the user logs out, can copy this printed information to log in again. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/monitor.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/monitor.rb b/lib/monitor.rb index 283cd788d..12f5280f3 100755 --- a/lib/monitor.rb +++ b/lib/monitor.rb @@ -87,8 +87,11 @@ class Monitor def ssh_connect(ssh_host, ssh_port, web_socket) web_socket.close - exec "ssh-keygen -R #{ssh_host}; - ssh root@#{ssh_host} -p #{ssh_port} -o StrictHostKeyChecking=no" + cmd = "ssh root@#{ssh_host} -p #{ssh_port} -o StrictHostKeyChecking=no" + puts cmd + + cmd = "ssh-keygen -R #{ssh_host};" + cmd + exec cmd end def stop(data, web_socket, code = 1000, reason = 'normal') -- 2.23.0

1 0

[PATCH compass-ci 3/3] monitoring/filter.cr: fix send message error
by Wu Zhende 30 Dec '20

30 Dec '20

filter message error: Error writing to socket: Broken pipe Exception: Error writing to socket: Broken pipe (Errno) from /usr/lib/crystal/core/socket.cr:82:13 in 'unbuffered_write' from /usr/lib/crystal/core/io/buffered.cr:192:5 in 'flush' from /usr/lib/crystal/core/io/buffered.cr:200:5 in 'close' from /usr/lib/crystal/core/http/server/handlers/websocket_handler.cr:44:9 in 'call' from /usr/share/crystal/app/lib/kemal/src/kemal/websocket.cr:11:7 in 'call' from /usr/share/crystal/app/lib/kemal/src/kemal/websocket_handler.cr:14:7 in 'call' from /usr/lib/crystal/core/http/server/handler.cr:26:7 in 'call_next' from /usr/share/crystal/app/lib/kemal/src/kemal/static_file_handler.cr:64:9 in 'call' from /usr/lib/crystal/core/http/server/handler.cr:26:7 in 'call_next' from /usr/share/crystal/app/lib/kemal/src/kemal/exception_handler.cr:8:7 in 'call' from /usr/lib/crystal/core/http/server/handler.cr:26:7 in 'call_next' from /usr/share/crystal/app/lib/kemal/src/kemal/log_handler.cr:8:37 in 'call' from /usr/lib/crystal/core/http/server/handler.cr:26:7 in 'call_next' from /usr/share/crystal/app/lib/kemal/src/kemal/init_handler.cr:12:7 in 'call' from /usr/lib/crystal/core/http/server/request_processor.cr:48:11 in 'process' from /usr/lib/crystal/core/http/server/request_processor.cr:22:3 in 'process' from /usr/lib/crystal/core/http/server.cr:497:5 in 'handle_client' from /usr/lib/crystal/core/http/server.cr:463:13 in '->' from /usr/lib/crystal/core/fiber.cr:255:3 in 'run' from /usr/lib/crystal/core/fiber.cr:48:34 in '->' from ??? Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/monitoring/filter.cr | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/monitoring/filter.cr b/src/monitoring/filter.cr index 5d3bc8b..79af3ab 100644 --- a/src/monitoring/filter.cr +++ b/src/monitoring/filter.cr @@ -41,6 +41,9 @@ class Filter @hash[query].each do |socket| socket.send msg.to_json + rescue e + puts "send msg failed: #{e}" + remove_filter_rule(query, socket) end end -- 2.23.0

1 0

[PATCH compass-ci 2/3] monitoring/monitoring.cr: change rabbitmq queue
by Wu Zhende 30 Dec '20

30 Dec '20

Get logs from different rabbitmq queues. And use multi-process to improve service processing capabilities. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/monitoring/monitoring.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/monitoring/monitoring.cr b/src/monitoring/monitoring.cr index d09c363..8d52d46 100644 --- a/src/monitoring/monitoring.cr +++ b/src/monitoring/monitoring.cr @@ -9,7 +9,8 @@ module Monitoring message_queue_client = MessageQueueClient.new - spawn message_queue_client.monitoring_message_queue(filter, "logging-test", "logging-test") + spawn message_queue_client.monitoring_message_queue(filter, "serial-logging", "serial-logging") + spawn message_queue_client.monitoring_message_queue(filter, "docker-logging", "docker-logging") ws "/filter" do |socket| query = JSON::Any.new("") -- 2.23.0

1 0