March 2021 - Compass-ci - mailweb.openeuler.org

[PATCH lkp-tests] tests/ansible_test: ansible test case
by Wang Chenglong 11 Mar '21

11 Mar '21

Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- tests/ansible_test | 84 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 tests/ansible_test diff --git a/tests/ansible_test b/tests/ansible_test new file mode 100755 index 000000000..36164bcae --- /dev/null +++ b/tests/ansible_test @@ -0,0 +1,84 @@ +#!/bin/bash +# - ansible_repo + +. $LKP_SRC/lib/debug.sh + +check_vars() +{ + ansible_host=localhost + echo $ansible_host + + [ -n "$ansible_repo" ] || die "ansible_repo is empty" + benchmark=${ansible_repo##*/} + + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible +} + +install_ansible() +{ + if [ "$os" == "openeuler" ]; then + yum -y reinstall git + yum -y install python3 + pip3 install ansible + elif [ "$os" == "centos" ]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + yum -y reinstall git + yum -y install ansible + elif [ "$os" == "debian" ]; then + apt-get install -y ansible + fi + + command -v ansible > /dev/null && { + ansible localhost -m ping + [ $? -eq 0 ] && { + echo "Ansible is successfully deployed!" + } + } +} + +download_ansible_role() +{ + + ansible_roles_path=$ansible_path/roles + [ -d $ansible_roles_path ] || mkdir -p $ansible_roles_path + cd $ansible_roles_path + git clone git://github.com/$ansible_repo.git || exit 1 + # git clone git://$GIT_SERVER/$ansible_repo.git || exit + # ansible-galaxy install $username.$rolename + if [ $? -eq 0 ]; then + echo "$benchmark is successfully installed" + else + echo "Please check whether your ansible_repo is correct!" + fi +} + +ansible_playbook_run() +{ + cat <<-EOF > "$ansible_path"/site.yml + - hosts: $ansible_host + roles: + - role: $benchmark + EOF + [ $? -eq 0 ] && { + echo "Playbook has been generated!" + } + + ansible-playbook "$ansible_path"/site.yml + + if [ $? -eq 0 ]; then + echo "Your playbook is running successfully!" + else + echo "Your playbook failed to run!" && exit 1 + fi +} + +main() +{ + check_vars + install_ansible + download_ansible_role + ansible_playbook_run +} + +main -- 2.23.0

3 4

[PATCH v3 lkp-tests] add new type error
by Bai Jing 11 Mar '21

11 Mar '21

[input] Error: Unknown repo: 'powertools' [output] unknown-repo.powertools: 1 Signed-off-by: Bai Jing <799286817(a)qq.com> --- stats/openeuler_docker | 8 +++++++- tests/openeuler_docker | 43 +++++++++++++++++++++++------------------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/stats/openeuler_docker b/stats/openeuler_docker index f85c9ca63..4c65c6bb3 100755 --- a/stats/openeuler_docker +++ b/stats/openeuler_docker @@ -6,12 +6,18 @@ sys_set = Set[] while (line = STDIN.gets) case line.chomp! - # Error: Unable to find a match: docker-registry + # Error: Unable to find a match: docker-registry mock xx when /Error: Unable to find a match: (.+)/ $1.split.each do |repo| repo_set << repo end + # Error: Unknown repo: 'powertools' + when /Error: Unknown repo: (.+)/ + repo = $1.delete!("'") + puts "unknown-repo.#{repo}: 1" + puts "unknown-repo.#{repo}.message: #{line}" + # /bin/sh: passwd: command not found when /\/bin\/sh: (.+): command not found/ puts "sh.command-not-found.#{$1}: 1" diff --git a/tests/openeuler_docker b/tests/openeuler_docker index b77b890a9..5270bde6e 100755 --- a/tests/openeuler_docker +++ b/tests/openeuler_docker @@ -13,7 +13,7 @@ cd $BENCHMARK_ROOT -centos_one_ok=false +nr_openeuler_fail=0 fit_repo() { @@ -21,7 +21,7 @@ fit_repo() git clone "$dockerfile_url" || exit 0 file_dir=$(find ./ -type f -name "*Dockerfile*" | xargs grep -HE '^FROM centos:(centos|)[78]') [ -n "$file_dir" ] || { - echo "current-repo not support centos" + echo "current-repo-not-support-centos: 1" exit 0 } } @@ -42,21 +42,23 @@ centos_docker() [ -n "$image" ] || image=${file#*/}_$i if [ -f "$file_dir/build.sh" ] && [ -x "$file_dir/build.sh" ]; then "$file_dir/build.sh" > /dev/null 2> /dev/null && { - centos_one_ok=true handle_dockerfile } else docker_build > /dev/null 2> /dev/null && { - centos_one_ok=true handle_dockerfile } fi let i++ done - [ "$centos_one_ok" == "false" ] && { - echo "centos all failed" + if [ "$nr_openeuler_fail" != 0 ]; then + echo "openeuler failed times: $nr_openeuler_fail" + echo "openeuler_valid_url: $dockerfile_url" + exit 1 + else + echo "current-repo-not-support-centos: 1" exit 0 - } + fi } handle_FROM() @@ -81,19 +83,18 @@ handle_epelrpm() } } -handle_groupadd() +add_base_commands() { - grep -qw "groupadd*" "$file" && { - sed -i '/groupadd*/i\ yum -y install shadow && \\' "$file" - } + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$file" + # } } handle_epel_repo() { - grep -qwE '.* wget .*(epel|CentOS-Base)\.repo' "$file" && { - sed 's|^|#|g' "$file" - } + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$file" } handle_dockerfile() @@ -101,15 +102,18 @@ handle_dockerfile() flag=1 handle_FROM handle_epel - handle_groupadd - docker_build "-openeuler" - [ $? -ne 0 ] || docker_run "-openeuler" + add_base_commands + if docker_build "-openeuler"; then + docker_run "-openeuler" + else + let nr_openeuler_fail++ + fi } docker_build() { docker build -f "$file" -t "${image}$1" . - [ $? -ne 0 ] && { + [ $? != 0 ] && { [ "$flag" == "1" ] && echo "${image}$1.build.fail: 1" return 1 } @@ -120,7 +124,8 @@ docker_build() docker_run() { docker run -d --name "${image}" "${image}$1" - docker_stats=$(docker inspect ${image} | grep Running | awk -F ' ' '{print $2}' | tr -d ',') + wait + docker_stats=$(docker inspect -f '{{.State.Running}}' ${image}) [ $docker_stats == false ] && { echo "${image}.run.fail: 1" return 1 -- 2.23.0

2 2

[PATCH v4 lkp-tests] add new type error
by Bai Jing 11 Mar '21

11 Mar '21

[input] Error: Unknown repo: 'powertools' [output] unknown-repo.powertools: 1 Signed-off-by: Bai Jing <799286817(a)qq.com> --- stats/openeuler_docker | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/stats/openeuler_docker b/stats/openeuler_docker index f85c9ca63..4c65c6bb3 100755 --- a/stats/openeuler_docker +++ b/stats/openeuler_docker @@ -6,12 +6,18 @@ sys_set = Set[] while (line = STDIN.gets) case line.chomp! - # Error: Unable to find a match: docker-registry + # Error: Unable to find a match: docker-registry mock xx when /Error: Unable to find a match: (.+)/ $1.split.each do |repo| repo_set << repo end + # Error: Unknown repo: 'powertools' + when /Error: Unknown repo: (.+)/ + repo = $1.delete!("'") + puts "unknown-repo.#{repo}: 1" + puts "unknown-repo.#{repo}.message: #{line}" + # /bin/sh: passwd: command not found when /\/bin\/sh: (.+): command not found/ puts "sh.command-not-found.#{$1}: 1" -- 2.23.0

1 0

[PATCH compass-ci 1/2] container/web-backend: add API /get_table_error
by Lu Weitao 11 Mar '21

11 Mar '21

[Example] curl -X GET 'localhost:30000/get_job_error?group_id=wcl_ansible-openeuler-03-10' return: { "filter":{"group_id":"wcl_ansible-openeuler-03-10"}, "table_fields":["job_id","error_id","error_message","link to result"], "data_table":[ [ "crystal.1354921", "ansible_test.error.Unable-to-start-service-httpd-Job-for-httpdservice-xxx.message", "{\"changed\": false, \"msg\": \"Unable to start service httpd: Job for httpd.service failed because a timeout was exceeded.}", "http://172.17.0.1:11300/result/ansible_test/2021-03-10/vm-2p16g--wcl1/opene… ate/crystal.1354921" ], [ "crystal.1354929", ... ] ... ] } Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend/web-backend | 19 +++++++++ src/lib/web_backend.rb | 67 +++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 74eda85..fa2a53d 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -146,3 +146,22 @@ end get '/get_repo_statistics' do new_refs_statistics(params) end + +# GET /get_job_error?suite=virttest&tbox_group=vm-2p8g +# Response +# { +# "filter":{"group_id":"wcl_ansible-openeuler-03-10"}, +# "table_fields":["job_id","error_id","error_message","link to result"], +# "data_table":[ +# [ +# "crystal.1354921", +# "ansible_test.error.Unable-to-start-service-httpd-Job-for-httpdservice-failed.message", +# "{\"changed\": false, \"msg\": \"Unable to start service httpd: Job for httpd.service failed because a timeout was exceeded.} +# "http://172.17.0.1:11300/result/ansible_test/2021-03-10/vm-2p16g--wcl1/opene…" +# ], +# ... +# ] +# } +get '/get_job_error' do + get_job_error(params) +end diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 89c0f3c..f09fc3d 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -12,6 +12,7 @@ require "#{CCI_SRC}/lib/compare.rb" require "#{CCI_SRC}/lib/constants.rb" require "#{CCI_SRC}/lib/es_query.rb" require "#{CCI_SRC}/lib/matrix2.rb" +require "#{CCI_SRC}/lib/params_group.rb" require "#{CCI_SRC}/lib/compare_data_format.rb" UPSTREAM_REPOS_PATH = ENV['UPSTREAM_REPOS_PATH'] || '/c/upstream-repos' @@ -548,3 +549,69 @@ def new_refs_statistics(params) end [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] end + + +# ------------------------------------------------------------------------------------------- +# job error table like: +# job_id error_id error_message link to result +# ------------------------------------------------------------------------------------- +# crystal.630608 "stderr.xxx" "messag:xxxx" https://$host:$port/$result_root +# ... +# ------------------------------------------------------------------------------------------- + +def get_job_error(params) + begin + body = job_error_body(params) + rescue StandardError => e + warn e.message + return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'get error table error'] + end + + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] +end + +def job_error_body(params) + error_table = get_error_table(params) + { + filter: params, + table_fields: ["job_id", "error_id", "error_message", "link to result"], + data_table: error_table, + }.to_json +end + +def get_error_table(filter_items) + error_table = [] + + job_list = get_job_list(filter_items) + job_list.each do |job| + error_table += get_error_from_job(job) + end + + error_table +end + +def get_job_list(items) + es = ESQuery.new(ES_HOST, ES_PORT) + query_results = es.multi_field_query(items) + + extract_jobs_list(query_results['hits']['hits']) +end + +# get all error_id from one job +def get_error_from_job(job) + job_error = [] + job['stats'].each do |metric, value| + next unless metric.end_with?('.message', 'log') + + result_host = ENV['SRV_HTTP_RESULT_HOST'] || SRV_HTTP_RESULT_HOST + result_port = ENV['SRV_HTTP_RESULT_PORT'] || SRV_HTTP_RESULT_PORT + job_error << [ + job['id'], + metric, + value, + "http://#{result_host}:#{result_port}#{job['result_root']}" + ] + end + + job_error +end -- 2.23.0

2 2

[PATCH v2 compass-ci 3/3] src/scheduler: add local mount of kernel_common_params
by Xu Xijian 11 Mar '21

11 Mar '21

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/scheduler/kernel_params.cr | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/scheduler/kernel_params.cr b/src/scheduler/kernel_params.cr index bb5983f..8ad5029 100644 --- a/src/scheduler/kernel_params.cr +++ b/src/scheduler/kernel_params.cr @@ -4,8 +4,12 @@ class Job private def kernel_common_params common_params = "user=lkp job=/lkp/scheduled/job.yaml RESULT_ROOT=/result/job ip=dhcp" - return "#{common_params} rw" if "#{self.os_mount}" == "local" - return "#{common_params} rootovl ro" + return "#{common_params} rootovl ro" unless "#{self.os_mount}" == "local" + + os_info = "#{os}_#{os_arch}_#{os_version}" + use_root_partition = "/dev/mapper/os-#{os_info}_#{src_lv_suffix}" if @hash["src_lv_suffix"]? != nil + save_root_partition = "/dev/mapper/os-#{os_info}_#{boot_lv_suffix}" if @hash["boot_lv_suffix"]? != nil + return "#{common_params} local use_root_partition=#{use_root_partition} save_root_partition=#{save_root_partition} rw" end private def kernel_custom_params @@ -20,7 +24,7 @@ class Job "cifs" => "root=cifs://#{OS_HTTP_HOST}#{os_real_path}" + ",guest,ro,hard,vers=1.0,noacl,nouser_xattr,noserverino", "initramfs" => "rdinit=/sbin/init prompt_ramdisk=0", - "local" => "root=/dev/mapper/os-#{self.os}_#{self.os_arch}_#{self.os_version}", + "local" => "root=#{OS_HTTP_HOST}:#{os_real_path}", # root is just used to temporarily mount a root in initqueue stage when lvm is not ready "container" => "", } -- 2.23.0

2 1

[PATCH v4 compass-ci 2/2] lib/dump-stat.rb: refactor lkp-tests/sbin/dump-stat
by Lu Weitao 11 Mar '21

11 Mar '21

[Why] support handle hash from lkp-tests/stats/$script.rb Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/dump_stat.rb | 194 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 lib/dump_stat.rb diff --git a/lib/dump_stat.rb b/lib/dump_stat.rb new file mode 100644 index 0000000..1c0695e --- /dev/null +++ b/lib/dump_stat.rb @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: GPL-2.0-only + +LKP_SRC ||= ENV['LKP_SRC'] || '/c/lkp-tests' + +require "#{LKP_SRC}/lib/statistics" +require "#{LKP_SRC}/lib/bounds" +require "#{LKP_SRC}/lib/yaml" +require "#{LKP_SRC}/lib/job" +require "#{LKP_SRC}/lib/string_ext" +require "#{LKP_SRC}/lib/log" +require 'set' + +UNSTRUCTURED_MONITORS = %w[ftrace].to_set + +def warn_stat(msg, monitor) + log_warn msg + log_warn "check #{RESULT_ROOT}/#{monitor}" +end + +# dump stat which input by lkp-tests/stats/$script return +# input: +# eg-1: +# { +# "pgfree" => [275506, 280018], +# ... +# } +# eg-2: +# { +# "iperf.tcp.sender.bps" => 34804801216.197174, +# "iperf.tcp.receiver.bps" => "34804762215.18231" +# } +module DumpStat + def self.dump_stat(monitor, stat_result) + @result = {} + @invalid_records = [] + @record_index = 0 + @monitor = monitor + + stat_result.each do |key, value| + key = key.resolve_invalid_bytes + next if key[0] == '#' + next if value.empty? || value == 0 + next if monitor =~ /^(dmesg|kmsg)$/ && key =~ /^(message|pattern):/ + + if key =~ /[ \t]/ + @invalid_records.push @record_index + warn_stat "whitespace in stats name: #{key}", @monitor + return nil # for exit current stats/script dump-stat + end + next if assign_log_message(key, value) + + k = @monitor + '.' + key + @result[k] ||= [] + fill_zero(k) + if value.is_a?(String) + value = check_string_value(k, value, @monitor) + next unless value + return nil unless number?(value, @invalid_records) + + value = value.index('.') ? value.to_f : value.to_i + elsif value.is_a?(Array) + (0..value.size - 1).each do |i| + next unless value[i].is_a?(String) + + value[i] = check_string_value(k, value[i], @monitor) + next unless value[i] + return nil unless number?(value[i], @invalid_records) + + value[i] = value[i].index('.') ? value[i].to_f : value[i].to_i + valid_stats_verification(k, value[i]) + end + @result[k] = value + next + end + valid_stats_verification(k, value) + @result[k].push value + end + return nil if @result.empty? + + remove_zero_stats + delete_invalid_number(@result, @invalid_records, @monitor) + cols_verifation + return nil unless useful_result?(@result) + + save_json(@result, "#{RESULT_ROOT}/#{(a)monitor}.json", @result.size * @min_cols > 1000) + end + + # keep message | log line which key end with .message|.log + def self.assign_log_message(key, value) + if key.end_with?('.message', '.log') + k = @monitor + '.' + key + @result[k] = value + return true + end + + false + end + + def self.fill_zero(key) + size = @result[key].size + if @record_index < size + @record_index = size + elsif (@record_index - size).positive? + # fill 0 for missing values + @result[key].concat([0] * (@record_index - size)) + end + end + + def self.valid_stats_verification(key, value) + return nil if valid_stats_range? key, value + + @invalid_records.push @record_index + puts "outside valid range: #{value} in #{key} #{RESULT_ROOT}" + end + + def self.remove_zero_stats + @max_cols = 0 + @min_cols = Float::INFINITY + @min_cols_stat = '' + @max_cols_stat = '' + zero_stats = [] + @result.each do |key, val| + next if key.end_with?('.message', '.log') + + if @max_cols < val.size + @max_cols = val.size + @max_cols_stat = key + end + if @min_cols > val.size + @min_cols = val.size + @min_cols_stat = key + end + next if val[0] != 0 + next if val[-1] != 0 + next if val.sum != 0 + + zero_stats << key + end + zero_stats.each { |x| @result.delete x } + end + + def self.cols_verifation + return nil unless @min_cols < @max_cols && !UNSTRUCTURED_MONITORS.include?(@monitor) + + if @min_cols == @max_cols - 1 + @result.each { |_k, y| y.pop if y.size == @max_cols } + puts "Last record seems incomplete. Truncated #{RESULT_ROOT}/#{(a)monitor}.json" + else + warn_stat "Not a matrix: value size is different - #{@min_cols_stat}: #{@min_cols} != #{@max_cols_stat}: #{@max_cols}: #{RESULT_ROOT}/#{(a)monitor}.json", @monitor + end + end +end + +def check_string_value(key, value, monitor) + # value terminator is expected. If not, throw out an error warning. + warn_stat "no line terminator in stats value: #{value}", monitor if value.chomp!.nil? + + value.strip! + if value.empty? + warn_stat "empty stat value of #{key}", monitor + return nil + end + + return value +end + +# only number is valid +def number?(value, invalid_records) + unless value.numeric? + invalid_records.push record_index + warn_stat "invalid stats value: #{value}", monitor + return nil + end + + true +end + +def delete_invalid_number(result, invalid_records, monitor) + return nil if monitor == 'ftrace' + + invalid_records.reverse_each do |index| + result.each do |_k, value| + value.delete_at index + end + end +end + +def useful_result?(result) + return nil if result.empty? + return nil if result.values[0].size.zero? + return nil if result.values[-1].size.zero? + + true +end -- 2.23.0

1 0

[PATCH v4 compass-ci 1/2] sbin/result2stats: support call function from stats/script
by Lu Weitao 11 Mar '21

11 Mar '21

[Why] old logical: lkp-tests/stats/script < script_log > tmpfile lkp-tests/sbin/dump-stat < tmpfile > $script.json this flow will escape special characters [How] use return of function to keep original characters lkp-tests/stats/script.rb func(read(script_log)) --> hash compass-ci/lib/dump-stats.rb hash --> dump_stats() --> $script.json [Dependence] lkp-tests/stats/$script.rb, for example: file_name: lkp-tests/stats/proc-vmstats.rb function name: proc_vmstats() input: Array(String) ["time: 231231234", "nr_free_page: 89721", ...] output: Hash(String, Array(Number|String) | Number | String) { "time" => [231231234, 231231235], "nr_free_page" => [89721,88564], ... } or { "time" => 231231234, "nr_free_page" => 89721, ... } Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/stats.rb | 21 ++++++++++++++++----- lib/stats_wrapper.rb | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/lib/stats.rb b/lib/stats.rb index c01d38d..a02f074 100644 --- a/lib/stats.rb +++ b/lib/stats.rb @@ -14,11 +14,18 @@ end def available_stats vailable_stats = [] + available_stats_func = [] Dir.open(LKP_SRC + '/stats').each do |file| - vailable_stats << file unless file.start_with?('.') + next if file.start_with?('.') + + if file.end_with?('.rb') + available_stats_func << file.chomp('.rb') + else + vailable_stats << file + end end - vailable_stats + [vailable_stats, available_stats_func] end def assign_default_stats(stats_list) @@ -32,13 +39,17 @@ class Stats def initialize(result_root) @result_root = result_root @job = load_yaml(File.join(result_root, 'job.yaml')) - @available_stats = available_stats + @available_stats, @available_stats_func = available_stats end def extract_stats stats_list = assign_stats_list stats_list.each do |stat| - StatsWrapper.wrapper(*stat) + if @available_stats_func.include?(stat) + StatsWrapper.wrapper_func(*stat) + else + StatsWrapper.wrapper(*stat) + end end end @@ -48,7 +59,7 @@ class Stats stats_list << ['time', @job['suite'] + '.time'] stats_list << 'stderr' @job.each_key do |k| - if @available_stats.include?(k) + if @available_stats.include?(k) || @available_stats_func.include?(k) stats_list << k end end diff --git a/lib/stats_wrapper.rb b/lib/stats_wrapper.rb index 8a6d6a9..faa81df 100644 --- a/lib/stats_wrapper.rb +++ b/lib/stats_wrapper.rb @@ -6,6 +6,11 @@ LKP_SRC ||= ENV['LKP_SRC'] || '/c/lkp-tests' require "#{LKP_SRC}/lib/log.rb" require 'tempfile' require 'English' +require_relative './dump_stat' + +Dir[File.expand_path("#{LKP_SRC}/stats/*.rb")].uniq.each do |file| + require file +end PROGRAM_DIR = "#{LKP_SRC}/stats" @@ -35,6 +40,27 @@ module StatsWrapper delete_log_package end + def self.wrapper_func(program, program_time = nil) + @program = program + @stats_group = program_time || program + @log = "#{RESULT_ROOT}/#{@stats_group}" + + return unless File.exist?("#{(a)log}.yaml") || pretreatment + + if File.exist?("#{(a)log}.yaml") + log_lines = read_log("#{(a)log}.yaml") + stat_result = parse_simple_log_yaml(log_lines) + else + log_lines = read_log(@log) + call_func_cmd = "#{(a)program.gsub('-', '_')}(log_lines)" # eg: proc_vmstats(log_lines) + stat_result = eval(call_func_cmd) + end + return unless DumpStat.dump_stat(@stats_group, stat_result) + + check_empty_json + delete_log_package + end + def self.pretreatment return unless available_program? return unless unzip_log @@ -261,3 +287,21 @@ module StatsWrapper File.delete(@tmpfile) if File.exist?(@tmpfile) end end + +# read line of log +# return Array(String) +def read_log(log_path) + return nil unless File.exist?(log_path) + + File.readlines(log_path) +end + +def parse_simple_log_yaml(log_lines) + result = Hash.new { |hash, key| hash[key] = [] } + log_lines.each do |line| + key, value = line.split(/:?\s+/) + result[key] << value + end + + result +end -- 2.23.0

1 0

[PATCH compass-ci 2/2] container/web-backend/start: add SRV_HTTP_RESULT/PORT to env
by Lu Weitao 11 Mar '21

11 Mar '21

[Why] the API /get_job_error need the host/port Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/web-backend/start | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/container/web-backend/start b/container/web-backend/start index 941fb11..6731d27 100755 --- a/container/web-backend/start +++ b/container/web-backend/start @@ -9,6 +9,10 @@ . $CCI_SRC/container/defconfig.sh docker_rm web-backend +load_cci_defaults +[[ $SRV_HTTP_RESULT_HOST ]] || SRV_HTTP_RESULT_HOST=$(curl ip.sb) +[[ $SRV_HTTP_RESULT_PORT ]] || SRV_HTTP_RESULT_PORT=11300 + cmd=( docker run --name web-backend @@ -17,6 +21,8 @@ cmd=( -p 32767:32767 -e ES_PORT=9200 + -e SRV_HTTP_RESULT_HOST=$SRV_HTTP_RESULT_HOST + -e SRV_HTTP_RESULT_PORT=$SRV_HTTP_RESULT_PORT -e CCI_SRC=/c/compass-ci -e LKP_SRC=/c/lkp-tests -e UPSTREAM_REPOS_PATH=/c/upstream-repos -- 2.23.0

1 0

[PATCH compass-ci] container: add start depends
by Cao Xueliang 11 Mar '21

11 Mar '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/auto-submit/start-depends | 2 +- container/delimiter/start-depends | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/container/auto-submit/start-depends b/container/auto-submit/start-depends index 38fc696..b5a1e0c 100755 --- a/container/auto-submit/start-depends +++ b/container/auto-submit/start-depends @@ -1,2 +1,2 @@ rabbitmq -es +register-accounts diff --git a/container/delimiter/start-depends b/container/delimiter/start-depends index 95df1a1..880f5b1 100755 --- a/container/delimiter/start-depends +++ b/container/delimiter/start-depends @@ -1,2 +1,3 @@ taskqueue send-mail +register-accounts -- 2.23.0

2 2

[PATCH v2 compass-ci 2/3] container/dracut-initrd: set sysroot in local mount situation
by Xu Xijian 11 Mar '21

11 Mar '21

When mount local filesystem in os starting, we will create or use a local existed logical volume to replace remote filesystem. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- .../dracut-initrd/bin/set-local-sysroot.sh | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 container/dracut-initrd/bin/set-local-sysroot.sh diff --git a/container/dracut-initrd/bin/set-local-sysroot.sh b/container/dracut-initrd/bin/set-local-sysroot.sh new file mode 100644 index 0000000..dc9000b --- /dev/null +++ b/container/dracut-initrd/bin/set-local-sysroot.sh @@ -0,0 +1,83 @@ +#!/bin/sh + +analyse_kernel_cmdline_params() { + rootfs="$(getarg root=)" + # example: $nfs_server_ip:/os/${os}/${os_arch}/${os_version}-snapshots/20210310005959 + rootfs_src=$(echo $"$rootfs" | sed 's/\///') + timestamp="$(basename $root)" + os="$(echo $root_src | awk -F '/|-' '{print $2}')" + os_arch="$(echo $root_src | awk -F '/|-' '{print $3}')" + os_version="$(echo $root_src | awk -F '/|-' '{print $4}')" + os_info="${os}_${os_arch}_${os_version}" + export rootfs_src timestamp os_info +} + +sync_src_lv() { + local src_lv="$1" + + [ -e "$src_lv" ] && return + + # create logical volume + src_lv_devname="$(basename $src_lv)" + lvm lvcreate -y -L 10G --name "${src_lv_devname#os-}" os + mke2fs -t ext4 -F "$src_lv" + + # sync nfsroot to $src_lv + mkdir -p /mnt1 && mount -t nfs "$rootfs_src" /mnt1 + mkdir -p /mnt2 && mount "$src_lv" /mnt2 + cp -a /mnt1/. /mnt2/ + umount /mnt1 /mnt2 + + # change permission of "$src_lv" to readonly + lvm lvchange --permission r "$src_lv" +} + +snapshot_boot_lv() { + local src_lv="$1" + local boot_lv="$2" + + [ "$src_lv" == "$boot_lv" ] && return + + lvm lvremove --force "$boot_lv" + boot_lv_devname="$(basename $boot_lv)" + lvm lvcreate --size 10G --name ${boot_lv_devname#os-} --snapshot "$src_lv" +} + +set_sysroot() { + boot_lv="$1" + umount "$NEWROOT" + mount "$boot_lv" "$NEWROOT" +} + +if ! getargbool 0 local; then + exit 0 +fi + +analyse_kernel_cmdline_params + +sed -i "s/^locking_type = .*/locking_type = 1/" /etc/lvm/lvm.conf + +use_root_partition="$(getarg use_root_partition=)" +if [ -n "$use_root_partition" ]; then + src_lv="$use_root_partition" # /dev/mapper/os-${os_info}_${timestamp} + sync_src_lv "$src_lv" +else + src_lv="/dev/mapper/os-${os_info}_$timestamp" + [ -e "src_lv" ] || { + echo "warn dracut: FATAL: no src_lv with local mount, reboot" + reboot + } +fi + +save_root_partition="$(getarg save_root_partition=)" +if [ -n "$save_root_partition" ]; then + boot_lv="$save_root_partition" +else + boot_lv="/dev/mapper/os-${os_info}" +fi + +snapshot_boot_lv "$src_lv" "$boot_lv" + +set_sysroot "$boot_lv" + +exit 0 -- 2.23.0

1 0