- Compass-ci - mailweb.openeuler.org

[PATCH lkp-tests] sbin/submit: add parameter option -i for submit
by Wei Jihui 18 Jan '21

18 Jan '21

[usage] 1. submit -i include.yaml job.yaml this will add include.yaml content to job.yaml, and priority is higher than job.yaml 2. submit -i 1.yaml -i 2.yaml job.yaml this will add 1.yaml and 2.yaml to job.yaml Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- lib/job.rb | 12 ++++++++---- sbin/submit | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/lib/job.rb b/lib/job.rb index 631ebfc9..1650eb2f 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -215,6 +215,7 @@ class Job attr_accessor :overrides attr_accessor :defaults attr_accessor :cmdline_defaults + attr_accessor :include_yamls def initialize(job = {}, defaults = {}, overrides = {}, cmdline_defaults={}) @job = job @@ -222,6 +223,7 @@ class Job @overrides = overrides # from command line @cmdline_defaults = cmdline_defaults # from command line @available_programs = {} + @include_yamls = [] end def source_file_symkey(file) @@ -254,8 +256,8 @@ class Job @overrides.merge!(hash['override']){ |_key, a, _b| a} hash.delete('override') end - hash.merge!(@overrides) - hash.merge!(fault_reproduction) if hash.has_key?("id") + hash.merge!(@overrides) unless @overrides.empty? + hash.merge!(load_include_yamls) unless @include_yamls.empty? @jobs.concat(multi_args(hash)) # return [hash] or [h1,h2] end rescue StandardError => e @@ -286,8 +288,10 @@ class Job @jobfile = jobfile end - def fault_reproduction - wait = load_yaml("#{lkp_src}/jobs/fault_reproduction.yaml", {}) + def load_include_yamls + include_hash = {} + include_hash[comment_to_symbol('auto generated by load_include_yamls()')] = nil + include_hash.merge!(load_yaml_merge(@include_yamls, {})) end def multi_args(hash) diff --git a/sbin/submit b/sbin/submit index 39cdcf57..6eca3b6e 100755 --- a/sbin/submit +++ b/sbin/submit @@ -14,6 +14,7 @@ opt_set_key_value = {} opt_cmdline_defaults = {} opt_output_dir = nil opt_auto_define_files = false +opt_include_yamls = [] opt_monitor = false opt_monitor_query = {} opt_my_queue = false @@ -43,6 +44,10 @@ options = OptionParser.new do |opts| opt_auto_define_files = true end + opts.on('-i include.yaml', '--include include.yaml', 'include other job yamls') do |yaml| + opt_include_yamls << yaml + end + opts.on('-c', '--connect', 'auto connect to the host') do actions << 'connect' end @@ -103,6 +108,14 @@ def find_jobfile(jobfile) exit 1 end +def find_jobfiles(jobfile_list) + search_jobfile_list = [] + jobfile_list.each do |jobfile| + search_jobfile_list << find_jobfile(jobfile) + end + return search_jobfile_list +end + submit_id = %x(uuidgen).chomp puts "submit_id=#{submit_id}" @@ -111,6 +124,7 @@ ARGV.each do |jobfile| jobs = Job2sh.new jobs.cmdline_defaults = opt_cmdline_defaults jobs.overrides = opt_set_key_value + jobs.include_yamls = find_jobfiles(opt_include_yamls) jobs.load(jobfile, true) || next jobs[:expand_params] = true jobs['testbox'] = opt_set_key_value['testbox'] if opt_set_key_value['testbox'] -- 2.23.0

3 2

[PATCH v2 compass-ci 3/3] docker/docker.rb: add docker serial log
by Wu Zhende 18 Jan '21

18 Jan '21

Generate serial log file. And add the start and end flags into it. It's for generate docker's dmsg file in result root. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- providers/docker/docker.rb | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/providers/docker/docker.rb b/providers/docker/docker.rb index 8e489c6..e1ab057 100755 --- a/providers/docker/docker.rb +++ b/providers/docker/docker.rb @@ -18,6 +18,7 @@ names = Set.new %w[ defaults = relevant_defaults(names) SCHED_HOST = defaults['SCHED_HOST'] || '172.17.0.1' SCHED_PORT = defaults['SCHED_PORT'] || 3000 +LOG_DIR = '/srv/cci/serial/logs' def get_url(hostname) "http://#{SCHED_HOST}:#{SCHED_PORT}/boot.container/hostname/#{hostname}" @@ -82,12 +83,35 @@ def start_container(hostname, load_path, hash) docker_image = hash['docker_image'] system "#{ENV['CCI_SRC']}/sbin/docker-pull #{docker_image}" system( - { 'hostname' => hostname, 'docker_image' => docker_image, 'load_path' => load_path }, + { 'hostname' => hostname, + 'docker_image' => docker_image, + 'load_path' => load_path, + 'log_dir' => "#{LOG_DIR}/#{hostname}" + }, ENV['CCI_SRC'] + '/providers/docker/run.sh' ) clean_dir(load_path) end +def record_runtime_log(hostname, start_time: Time.new, is_start: true, hash: {}) + log_file = "#{LOG_DIR}/#{hostname}" + if is_start + File.open(log_file, 'w') do |f| + # fluentd refresh time is 1s + # let fluentd to monitor this file first + sleep(2) + f.puts "\n#{start_time.strftime('%Y-%m-%d %H:%M:%S')} starting DOCKER" + f.puts "\n#{hash['job']}" + end + return start_time + end + + duration = ((Time.new - start_time) / 60).round(2) + File.open(log_file, 'a') do |f| + f.puts "\nTotal DOCKER duration: #{duration} minutes" + end +end + def main(hostname, queues) set_host2queues(hostname, queues) url = get_url hostname @@ -95,10 +119,14 @@ def main(hostname, queues) hash = parse_response url return del_host2queues(hostname) if hash.nil? + start_time = record_runtime_log(hostname, hash: hash) + load_path = build_load_path(hostname) load_initrds(load_path, hash) start_container(hostname, load_path, hash) + del_host2queues(hostname) + record_runtime_log(hostname, start_time: start_time, is_start: false) end def loop_main(hostname, queues) -- 2.23.0

1 0

[PATCH v2 compass-ci 2/3] docker/run.sh: collect docker serial logs
by Wu Zhende 18 Jan '21

18 Jan '21

Save docker logs to /srv/cci/serial/logs/$hostname. So fluentd can tail this file and send logs to monitoring service. In this way can generate docker's dmsg file Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- providers/docker/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/docker/run.sh b/providers/docker/run.sh index 4d53db7..2810641 100755 --- a/providers/docker/run.sh +++ b/providers/docker/run.sh @@ -7,6 +7,7 @@ : ${docker_image:="centos:7"} : ${load_path:="${HOME}/jobs"} : ${hostname:="dc-1g-1"} +: ${log_dir:="/srv/cci/serial/logs/$hostname"} if [[ $hostname =~ ^(.*)-[0-9]+$ ]]; then tbox_group=${BASH_REMATCH[1]} @@ -38,4 +39,4 @@ cmd=( /root/sbin/entrypoint.sh ) -"${cmd[@]}" +"${cmd[@]}" 2>&1 | tee -a "$log_dir" -- 2.23.0

1 0

[PATCH v2 compass-ci 1/3] monitoring/parse_serial_logs.cr: add docker patterns
by Wu Zhende 18 Jan '21

18 Jan '21

Add the start and end flags of the Docker startup mode. Use to generate docker's dmsg file. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/monitoring/parse_serial_logs.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/monitoring/parse_serial_logs.cr b/src/monitoring/parse_serial_logs.cr index 5a54849..0589523 100644 --- a/src/monitoring/parse_serial_logs.cr +++ b/src/monitoring/parse_serial_logs.cr @@ -25,6 +25,7 @@ require "json" class SerialParser START_PATTERNS = [ "starting QEMU", + "starting DOCKER", "Start PXE over IPv4", "iPXE initialising devices", "Open Source Network Boot Firmware", @@ -34,6 +35,7 @@ class SerialParser END_PATTERNS = [ "Total QEMU duration: ", + "Total DOCKER duration: ", "No job now", "Restarting system", ] -- 2.23.0

1 0

[PATCH v7 compass-ci] lib/es_jobs.rb: compact es-jobs output results
by Lu Kaiyi 18 Jan '21

18 Jan '21

[why] Reduce the nesting of results and output yaml [example] es-jobs os=archLinux [output] kvcount.arch=aarch64: 43 kvcount.arch=x86_64: 2 kvcount.category=benchmark: 12 kvcount.category=functional: 33 kvcount.job_state=failed: 20 kvcount.job_state=finished: 25 kvcount.os=archLinux: 45 kvcount.suite=cci-depends: 28 kvcount.suite=cci-makepkg: 5 kvcount.suite=iperf: 3 kvcount.suite=iperf-walk-os-test: 9 kvcount.tbox_group=vm-2p16g--wangyong: 1 kvcount.tbox_group=vm-2p16g--wcl: 1 kvcount.tbox_group=vm-2p16g.wangyong: 34 kvcount.tbox_group=vm-2p8g: 9 sum.stats.stderr./lkp/lkp/src/monitors/perf-stat:#:main: 12 sum.stats.stderr.Can_not_find_perf_command: 12 sum.stats.stderr./lkp/lkp/src/tests/wrapper:line#:which:command_not_found: 10 sum.stats.last_state.test.iperf.exit_code.127: 12 ... raw.stats.sched_debug.cfs_rq:/.load.stddev: [524288.0, null, null, null, null, null, null, null, null, 524288.0, 524288.0, null, null, null, null, null, 516608.0, null, 1048576.0, null, null, null, null, null, null, null, 524288.0, null, null, null, null, 2104832.0, 1572864.0, null, null, 2097152.0, null, null, null, null, null, null, null, null, null] ... avg.stats.sched_debug.cfs_rq:/.load.stddev: 1048576.0 avg.stats.softirqs.CPU1.NET_RX: 2.5833333333333335 avg.stats.slabinfo.kmalloc-512.active_objs: 1372.75 ... crystal.122454.summary.success: 1 crystal.122454.summary.result: success crystal.122690.summary.any_warning: 1 crystal.122690.summary.any_stderr: 1 crystal.122690.summary.any_error: 1 crystal.122690.summary.result: error ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_jobs.rb | 126 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 119 insertions(+), 7 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..c8a5b6c 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -2,7 +2,9 @@ # frozen_string_literal: true LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' +KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo].freeze +require 'yaml' require "#{LKP_SRC}/lib/stats" require_relative './es_query' @@ -123,17 +125,127 @@ class ESJobs return 0 end - def output + def get_all_metrics(jobs) + metrics = [] + jobs.each do |job| + stats = job['stats'] + next unless stats + + metrics.concat(stats.keys) + end + metrics.uniq! + end + + def initialize_result_hash(metrics) result = { - 'stats.count' => @stats['stats.count'] + 'kvcount' => {}, + 'sum.stats' => {}, + 'raw.stats' => {}, + 'avg.stats' => {}, + 'max.stats' => {}, + 'min.stats' => {} } + metrics.each { |metric| result['raw.stats'][metric] = [] } + result + end + + def set_default_value(result, stats, metrics) + left_metrics = metrics - stats.keys + left_metrics.each { |metric| result['raw.stats'][metric] << nil } - @stats.each do |key, value| - result[key] = value if @fields.include?(key) + stats.each do |key, value| + result['raw.stats'][key] << value end + end - @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? - @result.merge!(result) - puts JSON.pretty_generate(@result) + def kvcount(result, job) + KEYWORD.each do |keyword| + next unless job[keyword] + + result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0 + result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 + end + end + + def set_jobs_stat(result, stats, job) + summary_result = '' + if stats.keys.any? { |stat| stat.match(/warn/i) } + result["#{job['id']}.summary.any_warning"] = 1 + summary_result = 'warning' + end + if stats.keys.any? { |stat| stat.match(/stderr/i) } + result["#{job['id']}.summary.any_stderr"] = 1 + summary_result = 'stderr' + end + if stats.keys.any? { |stat| stat.match(/error|nr_fail=0/i) } + result["#{job['id']}.summary.any_error"] = 1 + summary_result = 'error' + end + if stats.keys.any? { |stat| stat.match(/fail/i) } && + stats.keys.all? { |stat| !stat.match(/nr_fail=0/i) } + result["#{job['id']}.summary.any_fail"] = 1 + summary_result = 'fail' + end + + if summary_result.empty? + result["#{job['id']}.summary.success"] = 1 + result["#{job['id']}.summary.result"] = 'success' + else + result["#{job['id']}.summary.result"] = summary_result + end + end + + def stats_count(result) + result['raw.stats'].each do |key, value| + if function_stat?(key) + result['sum.stats'][key] = value.compact.size + else + result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f + result['max.stats'][key] = value.compact.max + result['min.stats'][key] = value.compact.min + end + result['raw.stats'][key] = '---' + value.inspect + '---' + end + end + + def query_jobs_state(jobs) + metrics = get_all_metrics(jobs) + result = initialize_result_hash(metrics) + jobs.each do |job| + stats = job['stats'] + next unless stats + + set_jobs_stat(result, stats, job) + set_default_value(result, stats, metrics) + kvcount(result, job) + end + + stats_count(result) + result + end + + def compact_hash(prefix, result) + result.each do |key, value| + if prefix.empty? + prefix_key = "#{key}" + else + prefix_key = "#{prefix}.#{key}" + end + + if value.is_a? Hash + compact_hash(prefix_key, value) + else + @results[prefix_key] = value + end + end + end + + def output + @result = query_jobs_state((a)jobs.values) + @result['kvcount'] = @result['kvcount'].to_a.sort.to_h + @results = {} + compact_hash('', @result) + yaml = @results.to_yaml.delete_prefix!("---\n") + puts yaml.gsub('nil', 'null').gsub('"---', '').gsub('---"', '') end end -- 2.23.0

2 2

[PATCH v2 compass-ci 2/2] my-qemu.sh: add `-h|--help`, `-d|--debug` params for my-qemu.sh
by Yu Chuan 18 Jan '21

18 Jan '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- providers/my-qemu.sh | 33 +++++++++++++++++++++++++++++++++ providers/qemu/kvm.sh | 13 +++++++------ providers/qemu/pxe.sh | 8 ++++++-- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/providers/my-qemu.sh b/providers/my-qemu.sh index 8845a346e195..7d72a8fea708 100755 --- a/providers/my-qemu.sh +++ b/providers/my-qemu.sh @@ -2,6 +2,39 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +source $CCI_SRC/lib/log.sh + +help() +{ + cat <<-'EOF' + Usage: ./my-qemu.sh [OPTION]... + Ask scheduler service whether there is a job, and start a local vm if there is a job. + Scheduler api: http://${SCHED_HOST}:${SCHED_PORT}/boot.ipxe/mac/${mac} + + Mandatory arguments to long options are mandatory for short options too. + -h, --help display this help and exit + -d, --debug open the local vm serial port in the current shell + EOF + + exit 0 +} + +while true +do + [ $# -eq 0 ] && break + case "$1" in + -h|--help) + help;; + -d|--debug) + export DEBUG=true;; + *) + die "Unknown param: $1" + help + exit 1;; + esac + shift +done + [[ $tbox_group ]] || tbox_group=vm-2p8g export hostname=$tbox_group.$USER-$$ diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index a136c8ff45cb..098e8c2edcc7 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -115,11 +115,11 @@ set_qemu() print_message() { - echo SCHED_PORT: $SCHED_PORT - echo kernel: $kernel - echo initrds: $initrds - echo append: $append - echo less $log_file + log_info SCHED_PORT: $SCHED_PORT + log_info kernel: $kernel + log_info initrds: $initrds + log_info append: $append + [ "$DEBUG" == "true" ] || log_info less $log_file sleep 5 } @@ -136,9 +136,10 @@ public_option() -k en-us -no-reboot -nographic - -serial file:${log_file} -monitor null ) + + [ "$DEBUG" == "true" ] || kvm=("${kvm[@]}" -serial file:${log_file}) } individual_option() diff --git a/providers/qemu/pxe.sh b/providers/qemu/pxe.sh index fa688331964a..ff2c5661d3e4 100755 --- a/providers/qemu/pxe.sh +++ b/providers/qemu/pxe.sh @@ -4,6 +4,8 @@ # - nr_cpu # - memory +source $CCI_SRC/lib/log.sh + : ${nr_cpu:=1} : ${memory:=1G} @@ -18,7 +20,7 @@ fi qemu=qemu-system-aarch64 command -v $qemu >/dev/null || qemu=qemu-kvm -echo less $serial_log +[ "$DEBUG" == "true" ] || log_info less $serial_log kvm=( $qemu @@ -33,7 +35,9 @@ kvm=( -k en-us -no-reboot -nographic - -serial file:${serial_log} -monitor null ) + +[ "$DEBUG" == "true" ] || kvm=("${kvm[@]}" -serial file:${serial_log}) + "${kvm[@]}" -- 2.23.0

2 1

[PATCH v6 compass-ci] lib/es_jobs.rb: compact es-jobs output results
by Lu Kaiyi 18 Jan '21

18 Jan '21

[why] Reduce the nesting of results and output yaml [example] es-jobs os=archLinux [output] kvcount.arch=aarch64: 43 kvcount.arch=x86_64: 2 kvcount.category=benchmark: 12 kvcount.category=functional: 33 kvcount.job_state=failed: 20 kvcount.job_state=finished: 25 kvcount.os=archLinux: 45 kvcount.suite=cci-depends: 28 kvcount.suite=cci-makepkg: 5 kvcount.suite=iperf: 3 kvcount.suite=iperf-walk-os-test: 9 kvcount.tbox_group=vm-2p16g--wangyong: 1 kvcount.tbox_group=vm-2p16g--wcl: 1 kvcount.tbox_group=vm-2p16g.wangyong: 34 kvcount.tbox_group=vm-2p8g: 9 sum.stats.stderr./lkp/lkp/src/monitors/perf-stat:#:main: 12 sum.stats.stderr.Can_not_find_perf_command: 12 sum.stats.stderr./lkp/lkp/src/tests/wrapper:line#:which:command_not_found: 10 sum.stats.last_state.test.iperf.exit_code.127: 12 ... raw.stats.sched_debug.cfs_rq:/.load.stddev: [524288.0, nil, nil, nil, nil, nil, nil, nil, nil, 524288.0, 524288.0, nil, nil, nil, nil, nil, 516608.0, nil, 1048576.0, nil, nil, nil, nil, nil, nil, nil, 524288.0, nil, nil, nil, nil, 2104832.0, 1572864.0, nil, nil, 2097152.0, nil, nil, nil, nil, nil, nil, nil, nil, nil] ... avg.stats.sched_debug.cfs_rq:/.load.stddev: 1048576.0 avg.stats.softirqs.CPU1.NET_RX: 2.5833333333333335 avg.stats.slabinfo.kmalloc-512.active_objs: 1372.75 ... Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_jobs.rb | 97 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 7 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..a36a241 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -2,7 +2,9 @@ # frozen_string_literal: true LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' +KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo submit_id group_id] +require 'yaml' require "#{LKP_SRC}/lib/stats" require_relative './es_query' @@ -123,17 +125,98 @@ class ESJobs return 0 end - def output + def get_all_metrics(jobs) + metrics = [] + jobs.each do |job| + stats = job['stats'] + next unless stats + + metrics.concat(stats.keys) + end + metrics.uniq! + end + + def initialize_result_hash(metrics) result = { - 'stats.count' => @stats['stats.count'] + 'kvcount' => {}, + 'sum.stats' => {}, + 'raw.stats' => {}, + 'avg.stats' => {}, + 'max.stats' => {}, + 'min.stats' => {} } + metrics.each { |metric| result['raw.stats'][metric] = [] } + result + end + + def set_default_value(result, stats, metrics) + left_metrics = metrics - stats.keys + left_metrics.each { |metric| result['raw.stats'][metric] << nil } + + stats.each do |key, value| + result['raw.stats'][key] << value + end + end + + def kvcount(result, job) + KEYWORD.each do |keyword| + next unless job[keyword] + + result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0 + result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 + end + end - @stats.each do |key, value| - result[key] = value if @fields.include?(key) + def stats_count(result) + result['raw.stats'].each do |key, value| + if function_stat?(key) + result['sum.stats'][key] = value.compact.size + else + result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f + result['max.stats'][key] = value.compact.max + result['min.stats'][key] = value.compact.min + end + result['raw.stats'][key] = '---' + value.inspect + '---' + end + end + + def query_jobs_state(jobs) + metrics = get_all_metrics(jobs) + result = initialize_result_hash(metrics) + jobs.each do |job| + stats = job['stats'] + next unless stats + + set_default_value(result, stats, metrics) + kvcount(result, job) + end + + stats_count(result) + result + end + + def compact_hash(prefix, result) + result.each do |key, value| + if prefix.empty? + prefix_key = "#{key}" + else + prefix_key = "#{prefix}.#{key}" + end + + if value.is_a? Hash + compact_hash(prefix_key, value) + else + @results[prefix_key] = value + end end + end - @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? - @result.merge!(result) - puts JSON.pretty_generate(@result) + def output + @result = query_jobs_state((a)jobs.values) + @result['kvcount'] = @result['kvcount'].to_a.sort.to_h + @results = {} + compact_hash('', @result) + yaml = @results.to_yaml.delete_prefix!("---\n") + puts yaml.gsub('"---', '').gsub('---"', '') end end -- 2.23.0

2 1

[PATCH lkp-tests] distro/depends: add ansible_test depends file
by Zhang Yale 18 Jan '21

18 Jan '21

Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- distro/depends/ansible_test | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 distro/depends/ansible_test diff --git a/distro/depends/ansible_test b/distro/depends/ansible_test new file mode 100644 index 00000000..5eaf3f3c --- /dev/null +++ b/distro/depends/ansible_test @@ -0,0 +1,5 @@ +python3-pip +gcc +python3-devel +libffi-devel +openssl-devel -- 2.23.0

1 0

[PATCH lkp-tests 3/4] tests: add ansible_tests script
by Zhang Yale 18 Jan '21

18 Jan '21

This script can be invoked to run the ansible command to deploy ansible-roles which you specified in your environment. Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- tests/ansible_test | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100755 tests/ansible_test diff --git a/tests/ansible_test b/tests/ansible_test new file mode 100755 index 00000000..09e5ca9e --- /dev/null +++ b/tests/ansible_test @@ -0,0 +1,59 @@ +#!/bin/bash +# username +# rolename +# ansible_host +# ansible_user +# ansible_python_interpreter + +[ -n "$username" ] || echo "username is empty" +[ -n "$rolename" ] || echo "rolename is empty" +[ -n "$ansible_host" ] || $ansible_host=localhost +[ -n "$ansible_user" ] || $ansible_user=root +[ -n "$ansible_python_interpreter" ] || $ansible_python_interpreter=/user/bin/python3 + +build_mutual_trust() +{ + echo -e "y\n" | ssh-keygen -t rsa -b 2048 -N '' -f /root/.ssh/id_rsa + cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys + ssh -o "StrictHostKeyChecking=no" -o "ExitOnForwardFailure=yes" -o "passwordAuthentication=no" root@$ansible__host 'chmod 600 ~/.ssh/authorized_keys' +} + +install_ansible() +{ + pip3 install ansible + ansible localhost -m ping + #ansible all -m authorized_key -a "user=root key='{{ lookup('file', '/root/.ssh/id_rsa.pub') }}' path=/root/.ssh/authorized_keys manage_dir=no" --ask-pass -c paramiko + [ $? -eq 0 ] && echo "Ansible is successfully deployed!" +} + +download_ansible_role() +{ + ansible-galaxy install $username.$rolename + [ $? -eq 0 ] && echo "Please check whether your rolename is correct!" +} + +ansible_playbook_run() +{ + ansible_path=/root/.ansible + [ -d "$ansible_path" ] || mkdir -p /root/.ansible + cat <<-EOF > "$ansible_path"/hosts + $ansible_host ansible_user=$ansible_user ansible_python_interpreter=$ansible_python_interpreter + EOF + [ $? -eq 0 ] && echo "Hostfile has been generated!" + + cat <<-EOF > "$ansible_path"/$rolename.yml + - hosts: $ansible_host + remote_user: $ansible_user + roles: + - role: $username.$rolename + EOF + [ $? -eq 0 ] && echo "Playbook has been generated!" + + ansible-playbook "$ansible_path"/$rolename.yml + [ $? -eq 0 ] && echo "Your playbook is running successfully!" +} + +#build_mutual_trust +install_ansible +download_ansible_role +ansible_playbook_run -- 2.23.0

1 1

[PATCH lkp-tests 4/4] distro/depends: add vim in lkp
by Zhang Yale 18 Jan '21

18 Jan '21

Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- distro/depends/lkp | 1 + 1 file changed, 1 insertion(+) diff --git a/distro/depends/lkp b/distro/depends/lkp index 82c67184..5a8196ac 100644 --- a/distro/depends/lkp +++ b/distro/depends/lkp @@ -6,3 +6,4 @@ gzip which rsync ntpdate +vim -- 2.23.0

1 0

[PATCH lkp-tests 1/4] pkg/turbostat: changed turbostat's source url
by Zhang Yale 18 Jan '21

18 Jan '21

turbostat can not be found in x86_64 Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- pkg/turbostat/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/turbostat/PKGBUILD b/pkg/turbostat/PKGBUILD index d7e536c7..24adcffe 100644 --- a/pkg/turbostat/PKGBUILD +++ b/pkg/turbostat/PKGBUILD @@ -4,7 +4,7 @@ pkgrel=1 url='https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git' arch=('i386' 'x86_64') license=('GPL') -source=('https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git') +source=("git://$GIT_SERVER/l/linux/linux.git") md5sums=('SKIP') build() -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] lib/compare_error_messages.rb: add the credibility check of job
by Lin Jiaxin 18 Jan '21

18 Jan '21

defect: may match the wrong result error_id="build-pkg.pixz.c:warning:'subsuf'defined-but-not-used[-Wunused-function]" filename="pixz.c" correct result: gcc -DHAVE_CONFIG_H -I. -I.. -D_FORTIFY_SOURCE=2 -pthread -Wall -Wno-unknown-pragmas -march=native -O2 -pipe -fstack-protector-strong -fno-plt -Wl,-O1, --sort-common,--as-needed,-z,relro,-z,now -MT pixz-pixz.o -MD -MP -MF .deps/pixz-pixz.Tpo -c -o pixz-pixz.o `test -f 'pixz.c' || echo './'`pixz.c ^^^^^^ wrong result: mv -f .deps/pixz.cpu.Tpo .deps/pixz.cpu.Po ^^^^^^ Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- lib/compare_error_messages.rb | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/lib/compare_error_messages.rb b/lib/compare_error_messages.rb index 0b51fc3..b3812bb 100644 --- a/lib/compare_error_messages.rb +++ b/lib/compare_error_messages.rb @@ -8,6 +8,39 @@ require_relative 'es_query' require_relative 'error_messages' require_relative "#{ENV['LKP_SRC']}/lib/common" +# previous_job_id's error_ids include error_id +# If later_job_id's output include error filename that extract from error_id, +# the later_job_id is credible. +def credible?(previous_job_id, later_job_id, error_id) + es = ESQuery.new + + previous_result_file = File.join('/srv', es.query_by_id(previous_job_id)['result_root'], 'build-pkg') + later_result_file = File.join('/srv', es.query_by_id(later_job_id)['result_root'], 'build-pkg') + + return false if filenames_check(previous_result_file, later_result_file, error_id).value?(false) + + return true +end + +def filenames_check(previous_result_file, later_result_file, error_id) + filenames = Set.new + filenames_check = Hash.new { |h, k| h[k] = false } + + error_lines = ErrorMessages.new(previous_result_file).obtain_error_messages_by_error_id(error_id, true) + error_lines.each do |error_line| + filenames << $1 if error_line =~ /(.*)(:.*){2}: (error|warning):/ + end + + File.open(later_result_file).each_line do |line| + filenames.each do |filename| + filenames_check[filename] + filenames_check[filename] = true if line.include?(filename) + end + end + + return filenames_check +end + def get_compare_result(previous_job_id, later_job_id) es = ESQuery.new -- 2.23.0

2 1

[PATCH v3 compass-ci] lib/error_messages.rb: obtain_error_messages_by_error_id function add keyword argument
by Lin Jiaxin 18 Jan '21

18 Jan '21

Determine whether to return the error line or error messages by the value of return_error_line Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- lib/error_messages.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/error_messages.rb b/lib/error_messages.rb index 3e1564b..d83d890 100644 --- a/lib/error_messages.rb +++ b/lib/error_messages.rb @@ -40,12 +40,12 @@ class ErrorMessages return @error_messages end - def obtain_error_messages_by_error_id(error_id) + def obtain_error_messages_by_error_id(error_id, return_error_line = false) error_messages_by_error_id = [] error_messages = obtain_error_messages error_messages.each do |k, v| if ('build-pkg.' + build_pkg_error_id(k)) == error_id - error_messages_by_error_id += v.to_a + error_messages_by_error_id += return_error_line ? [k] : v.to_a end end error_messages_by_error_id -- 2.23.0

2 1

[PATCH compass-ci 2/3] qemu/kvm.sh: optimize the prompt when the error occurs
by Yu Chuan 18 Jan '21

18 Jan '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- providers/qemu/kvm.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index 35678fbd44ef..f1d65272dbf9 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -52,15 +52,23 @@ parse_ipxe_script() check_option_value() { + [ -n "$kernel" ] || { + echo "[ERROR] Can not find job for current hostname: $hostname." + exit 1 + } + [ -s "$kernel" ] || { - echo "The kernel does not exist: $kernel" - exit + echo "[ERROR] Can not find kernel file or kernel file is empty: $kernel." + exit 1 } # debian has both qemu-system-x86_64 and qemu-system-riscv64 command [[ $kernel =~ 'riscv64' ]] && qemu=qemu-system-riscv64 - [ -n "$initrds" ] || exit + [ -n "$initrds" ] || { + echo "[ERROR] Can not find any initrd for current vm." + exit 1 + } } set_initrd() -- 2.23.0

3 4

[PATCH compass-ci] src/lib/web_backend.rb: fix wrong number of arguments
by Lu Weitao 18 Jan '21

18 Jan '21

[Why] wrong number of arguments (given 3, expected 4) 2021-01-18 11:49:46.000000000 +0800 web-backend: 172.17.0.1 - - [18/Jan/2021:03:49:44 UTC] "GET /compare?suite=iperf&dimension=os HTTP/1.1" 500 13 Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- src/lib/web_backend.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index ea67064..61e3334 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -186,15 +186,15 @@ end def do_get_groups_matrices(must, dimension, total, size, from) job_list = get_dimension_job_list(dimension, must, size, from) - matrices, suites_hash = combine_group_query_data(job_list, dimension) + matrices, suites_hash, latest_jobs_hash = combine_group_query_data(job_list, dimension) while matrices.empty? from += size break if from > total job_list = get_dimension_job_list(dimension, must, size, from) - matrices, suites_hash = combine_group_query_data(job_list, dimension) + matrices, suites_hash, latest_jobs_hash = combine_group_query_data(job_list, dimension) end - [matrices, suites_hash] + [matrices, suites_hash, latest_jobs_hash] end def get_groups_matrices(conditions, dimension, must, size, from) @@ -209,11 +209,12 @@ end def get_compare_body(params) dimension, conditions = get_dimension_conditions(params) must = get_es_must(params) - groups_matrices, suites_hash = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) + groups_matrices, suites_hash, latest_jobs_hash = + get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) if !groups_matrices || groups_matrices.empty? body = 'No Data.' else - body = compare_group_matrices(groups_matrices, suites_hash, { no_print: true }) + body = compare_group_matrices(groups_matrices, suites_hash, latest_jobs_hash, { no_print: true }) body = 'No Difference.' if !body || body.empty? end return body -- 2.23.0

1 0

[PATCH compass-ci 3/3] my-qemu.sh: add `-h|--help`, `-d|--debug` params for my-qemu.sh
by Yu Chuan 18 Jan '21

18 Jan '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- providers/my-qemu.sh | 32 ++++++++++++++++++++++++++++++++ providers/qemu/kvm.sh | 5 +++-- providers/qemu/pxe.sh | 6 ++++-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/providers/my-qemu.sh b/providers/my-qemu.sh index 8845a346e195..a8f09a610d3f 100755 --- a/providers/my-qemu.sh +++ b/providers/my-qemu.sh @@ -2,6 +2,38 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +help() +{ + cat <<-'EOF' + Usage: ./my-qemu.sh [OPTION]... + Ask scheduer service whether there is a job, and start a local vm if there is a job. + Scheduler api: http://${SCHED_HOST}:${SCHED_PORT}/boot.ipxe/mac/${mac} + + Mandatory arguments to long options are mandatory for short options too. + -h, --help display this help and exit + -d, --debug open the local vm serial port in the current shell + EOF + + exit 0 +} + +while true +do + [ $# -eq 0 ] && break + case "$1" in + -h|--help) + help + shift;; + -d|--debug) + export DEBUG=true + shift;; + *) + echo "[ERROR] Unknown param: $1" + help + exit 1;; + esac +done + [[ $tbox_group ]] || tbox_group=vm-2p8g export hostname=$tbox_group.$USER-$$ diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index f1d65272dbf9..a3cbe4248a3a 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -123,7 +123,7 @@ print_message() echo kernel: $kernel echo initrds: $initrds echo append: $append - echo less $log_file + [ "$DEBUG" == "true" ] || echo less $log_file sleep 5 } @@ -140,9 +140,10 @@ public_option() -k en-us -no-reboot -nographic - -serial file:${log_file} -monitor null ) + + [ "$DEBUG" == "true" ] || kvm=("${kvm[@]}" -serial file:${log_file}) } individual_option() diff --git a/providers/qemu/pxe.sh b/providers/qemu/pxe.sh index fa688331964a..2e205405c793 100755 --- a/providers/qemu/pxe.sh +++ b/providers/qemu/pxe.sh @@ -18,7 +18,7 @@ fi qemu=qemu-system-aarch64 command -v $qemu >/dev/null || qemu=qemu-kvm -echo less $serial_log +[ "$DEBUG" == "true" ] || echo less $serial_log kvm=( $qemu @@ -33,7 +33,9 @@ kvm=( -k en-us -no-reboot -nographic - -serial file:${serial_log} -monitor null ) + +[ "$DEBUG" == "true" ] || kvm=("${kvm[@]}" -serial file:${serial_log}) + "${kvm[@]}" -- 2.23.0

2 2

[PATCH compass-ci] sparrow/0-package: add packages
by Liu Yinsi 18 Jan '21

18 Jan '21

container os-cifs and os-nfs will use sudo command, so add it. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/0-package/os/centos | 1 + sparrow/0-package/os/debian | 1 + sparrow/0-package/os/openEuler | 1 + 3 files changed, 3 insertions(+) diff --git a/sparrow/0-package/os/centos b/sparrow/0-package/os/centos index b25dade..b8ef769 100755 --- a/sparrow/0-package/os/centos +++ b/sparrow/0-package/os/centos @@ -24,6 +24,7 @@ pkgs=( hostname wget curl + sudo ) yum install -y --skip-broken "${pkgs[@]}" diff --git a/sparrow/0-package/os/debian b/sparrow/0-package/os/debian index 4d13413..947cbdc 100755 --- a/sparrow/0-package/os/debian +++ b/sparrow/0-package/os/debian @@ -24,6 +24,7 @@ pkgs=( qemu-system-aarch64 qemu-system-x86 qemu-system-riscv64 + sudo ) apt-get update diff --git a/sparrow/0-package/os/openEuler b/sparrow/0-package/os/openEuler index 04996f5..61a37c4 100755 --- a/sparrow/0-package/os/openEuler +++ b/sparrow/0-package/os/openEuler @@ -23,6 +23,7 @@ pkgs=( hostname libvirt curl + sudo ) yum install -y --skip-broken "${pkgs[@]}" -- 2.23.0

1 0

[PATCH compass-ci] container/srv-http: mount /srv/cci/libvirt-xml to srv-http
by Xiao Shenwei 18 Jan '21

18 Jan '21

[why] libvirt support user define domain.xml file to start kvm the /srv/cci/libvirt-xml will save user's xml file Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- container/srv-http/start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/srv-http/start b/container/srv-http/start index d9f461b..945cde0 100755 --- a/container/srv-http/start +++ b/container/srv-http/start @@ -16,6 +16,7 @@ cmd=( -v /srv/initrd/qemu-image:/srv/initrd/qemu-image:ro -v /srv/os:/srv/os:ro -v /srv/git/archlinux:/srv/git/archlinux:ro + -v /srv/cci/libvirt-xml:/srv/cci/libvirt-xml:ro -d srv-http ) -- 2.23.0

1 0

[PATCH compass-ci 1/3] qemu/kvm.sh: fix script will hang when there is no job
by Yu Chuan 18 Jan '21

18 Jan '21

[Error Msg] ++ set_initrd ++ initrd=initrd ++ cat <<< hang here [Root Cause] Function set_initrd() have follow line: cat $initrds > $initrd When there is no job corresponding to current vm in the remote redis queue, var $initrds will be equal to "". And from `cat --help`, we can know the follow: cat Copy standard input to standard output. So the process will get stuck waiting for input. [How] Function check_option_value() is used to check whether $initrds or $kernel is empty. And $initrds and $kernel are assigned in the function parse_ipxe_script(). So move check_option_value() before set_initrd(). Signed-off-by: Yu Chuan <13186087857(a)163.com> --- providers/qemu/kvm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/qemu/kvm.sh b/providers/qemu/kvm.sh index 854e9138a7c0..35678fbd44ef 100755 --- a/providers/qemu/kvm.sh +++ b/providers/qemu/kvm.sh @@ -197,8 +197,8 @@ write_logfile parse_ipxe_script -set_options check_option_value +set_options print_message -- 2.23.0

2 6

[PATCH v4 compass-ci] docker2rootfs: add unzip vmlinux function
by Wang Chenglong 18 Jan '21

18 Jan '21

[Why] Unpacked vmlinux into vmlinuz format. Signeo-off-by: Wang Chenglong <18509160991(a)163.com> --- container/docker2rootfs/common | 33 ++++++++++++++++++++++++++++- container/docker2rootfs/run | 1 + container/docker2rootfs/setup-image | 2 +- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/container/docker2rootfs/common b/container/docker2rootfs/common index c26bbc8..e8c56ba 100755 --- a/container/docker2rootfs/common +++ b/container/docker2rootfs/common @@ -5,6 +5,8 @@ # This file is called by script run running on host # define functions and set host environment variables +. $CCI_SRC/container/qcow2rootfs/bin/common + get_host_aarch() { aarch=$(arch) @@ -133,7 +135,7 @@ get_rootfs_kernel() local vmlinuz_file local vmlinuz kernel - vmlinuz_file=$(find ./boot -name "vmlinuz-*" | grep -v rescue) && export ROOTFS_VMLINUZ_FILE=$vmlinuz_file + vmlinuz_file=$(find ./boot -name "vmlinu[z|x]-*" | grep -v rescue) && export ROOTFS_VMLINUZ_FILE=$vmlinuz_file vmlinuz=$(basename "$vmlinuz_file") && export ROOTFS_VMLINUZ=$vmlinuz kernel=${vmlinuz:8} && export ROOTFS_KERNEL=$kernel @@ -155,3 +157,32 @@ create_links_vmlinuz_initrd() echo "[INFO] Create links to initrd.lkp and vmlinuz success!" } + +unzip_vmlinuz() { + get_rootfs_kernel + + file ${ROOTFS_VMLINUZ_FILE} | grep -q gzip || return + + get_vmlinuz_start_postion ${ROOTFS_VMLINUZ_FILE} + [ $VMLINUZ_START_POSTION -eq -1 ] && return + + local unzip_str + + if [ $VMLINUZ_START_POSTION -eq 0 ]; then + unzip_str="dd if=$ROOTFS_VMLINUZ bs=1 | zcat > $ROOTFS_VMLINUZ.tmp" + else + unzip_str="dd if=$ROOTFS_VMLINUZ bs=1 skip=$VMLINUZ_START_POSTION | zcat > $ROOTFS_VMLINUZ.tmp" + fi + + echo "Unzipping vmlinuz..." + cd "$ROOTFS_DIR/boot" + $unzip_str + mv $ROOTFS_VMLINUZ.tmp $ROOTFS_VMLINUZ + chmod o+r $ROOTFS_VMLINUZ + + if [ $? -eq 0 ]; then + echo "[INFO] Unzipping vmlinuz success!" + else + echo "[WARNING] Unzipping vmlinuz failed!" + fi +} diff --git a/container/docker2rootfs/run b/container/docker2rootfs/run index 8a7b315..cdb19bf 100755 --- a/container/docker2rootfs/run +++ b/container/docker2rootfs/run @@ -31,3 +31,4 @@ start=( cp_rootfs $IMAGE_PACK $2 docker rm -f rootfs-docker create_links_vmlinuz_initrd $2 +unzip_vmlinuz diff --git a/container/docker2rootfs/setup-image b/container/docker2rootfs/setup-image index ffe4937..47bfb7e 100755 --- a/container/docker2rootfs/setup-image +++ b/container/docker2rootfs/setup-image @@ -38,7 +38,7 @@ setup_login() pack_cgz() { echo "Packing package. Please wait." - find ./ ! -path "./tmp/${IMAGE_PACK}" | cpio -o -Hnewc | gzip -9 >./tmp/$IMAGE_PACK + find ./ ! -path "./tmp/$IMAGE_PACK" | cpio -o -Hnewc | gzip -9 > "./tmp/$IMAGE_PACK" chmod 644 /tmp/${IMAGE_PACK} } -- 2.23.0

1 0

[PATCH v3 compass-ci] sparrow/4-docker: fix logical error
by Liu Yinsi 18 Jan '21

18 Jan '21

before: if execute ./buildall and docker images exist, then ./build after: if execute ./buildall or docker images not exist, then ./build Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/4-docker/buildall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sparrow/4-docker/buildall b/sparrow/4-docker/buildall index f24226c..77fe43a 100755 --- a/sparrow/4-docker/buildall +++ b/sparrow/4-docker/buildall @@ -33,7 +33,7 @@ do_one() cd "$container" [ "$container_name" == 'ssh-r' ] && exit docker images | grep -wq "$container_name" - if [ "$?" == 0 ] && [ "$action" != "reboot" ]; then + if [ "$?" -ne 0 ] || [ "$action" != "reboot" ]; then [ -x build ] && ./build [ -x install ] && ./install fi -- 2.23.0

5 7

[PATCH v2 compass-ci 1/2] lib/error_messages.rb: obtain_error_messages_by_error_id function add keyword argument
by Lin Jiaxin 18 Jan '21

18 Jan '21

Determine whether to return the error line or error messages by the value of eturn_error_line Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- lib/error_messages.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/error_messages.rb b/lib/error_messages.rb index 3e1564b..d83d890 100644 --- a/lib/error_messages.rb +++ b/lib/error_messages.rb @@ -40,12 +40,12 @@ class ErrorMessages return @error_messages end - def obtain_error_messages_by_error_id(error_id) + def obtain_error_messages_by_error_id(error_id, return_error_line = false) error_messages_by_error_id = [] error_messages = obtain_error_messages error_messages.each do |k, v| if ('build-pkg.' + build_pkg_error_id(k)) == error_id - error_messages_by_error_id += v.to_a + error_messages_by_error_id += return_error_line ? [k] : v.to_a end end error_messages_by_error_id -- 2.23.0

2 1

[PATCH lkp-tests] hosts: add new host file for libvirt
by Xiao Shenwei 18 Jan '21

18 Jan '21

[why] libvirt use a xml file to start vm <os> <kernel>xxx</kernel> <initrd>xxx</initrd> <cmdline>xxx</cmdline> </os> so, we can submit job use os_mount=initramfs to get kernel and initrd absolute path. Minimum memory limit: 16G Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- hosts/libvirt-2p16g | 5 +++++ hosts/libvirt-2p32g | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 hosts/libvirt-2p16g create mode 100644 hosts/libvirt-2p32g diff --git a/hosts/libvirt-2p16g b/hosts/libvirt-2p16g new file mode 100644 index 000000000..aecbb7948 --- /dev/null +++ b/hosts/libvirt-2p16g @@ -0,0 +1,5 @@ +provider: libvirt +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 16G diff --git a/hosts/libvirt-2p32g b/hosts/libvirt-2p32g new file mode 100644 index 000000000..aa7c8c065 --- /dev/null +++ b/hosts/libvirt-2p32g @@ -0,0 +1,5 @@ +provider: libvirt +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 32G -- 2.23.0

1 1

[PATCH v2 compass-ci] sbin/es-find support range query
by Lu Weitao 18 Jan '21

18 Jan '21

[Example] es-find -r/--range "start_time=2021-1-1 00:00:00,2021-1-1 23:59:59;..." Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- sbin/es-find | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/sbin/es-find b/sbin/es-find index 17eefe6..c144a8b 100755 --- a/sbin/es-find +++ b/sbin/es-find @@ -5,6 +5,7 @@ # Usage: # es-find key=val ... +# es-find -r "start_time=2021-1-1 00:00:00,2021-1-1 23:59:59;..." # if given multi keys, they create AND conditions # if given val1,val2,... for a key, they create OR conditions # Eg: query (testbox is xxx or yyy) and (suite is aaa) and (user is bbb). @@ -29,7 +30,43 @@ def parse_argv items end +# output: +# eg: +# { +# start_time => { +# gte: '2020-09-10 01:50:00', +# lte: '2020-09-10 01:53:00' +# }, +# ... +# } +def prase_range(range) + range_hash = {} + keyword_list = range.split(';') + keyword_list.each do |keyword| + key, value = keyword.split('=') + next unless key && value + + range_hash[key] = assign_range(value) + end + + range_hash +end + +def assign_range(value) + range_value = {} + value_list = value.split(',') + if value_list.length > 1 + range_value[:gte] = value_list[0] + range_value[:lte] = value_list[1] + else + range_value[:gte] = value + end + + range_value +end + options = { nr_jobs: 10, desc_keyword: nil } +range = nil opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: es-find [options] search_key1=val1[,val2..] ..' @@ -49,6 +86,10 @@ opt_parser = OptionParser.new do |opts| options[:desc_keyword] = d end + opts.on('-r', '--range KEYWORD', 'range by keyword: start_time|end_time|...') do |r| + range = r + end + opts.on_tail('-h', '--help', 'show this message') do puts opts exit @@ -57,6 +98,8 @@ end opt_parser.parse!(ARGV) items = parse_argv +items[:range] = prase_range(range) if range + es = ESQuery.new(ES_HOST, ES_PORT) query_result = es.multi_field_query(items, size: options[:nr_jobs], desc_keyword: options[:desc_keyword]) json_string = JSON.pretty_generate(query_result) -- 2.23.0

1 0

[PATCH v2 compass-ci 2/3] docker/run.sh: collect docker serial logs
by Wu Zhende 18 Jan '21

18 Jan '21

Save docker logs to /srv/cci/serial/logs/$hostname. So fluentd can tail this file and send logs to monitoring service. In this way can generate docker's dmsg file Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- providers/docker/run.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/docker/run.sh b/providers/docker/run.sh index 4d53db7..2810641 100755 --- a/providers/docker/run.sh +++ b/providers/docker/run.sh @@ -7,6 +7,7 @@ : ${docker_image:="centos:7"} : ${load_path:="${HOME}/jobs"} : ${hostname:="dc-1g-1"} +: ${log_dir:="/srv/cci/serial/logs/$hostname"} if [[ $hostname =~ ^(.*)-[0-9]+$ ]]; then tbox_group=${BASH_REMATCH[1]} @@ -38,4 +39,4 @@ cmd=( /root/sbin/entrypoint.sh ) -"${cmd[@]}" +"${cmd[@]}" 2>&1 | tee -a "$log_dir" -- 2.23.0

2 1

2025

2024

2023

2022

2021

2020

Compass-ci