- Compass-ci - mailweb.openeuler.org

[PATCH v2 lkp-tests] /stats add proc-vmstat.rb
by Lu Weitao 08 Mar '21

08 Mar '21

[Why] convert stats/proc-vmstat to proc_vmstat() of proc-vmstat.rb Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- stats/proc-vmstat.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 stats/proc-vmstat.rb diff --git a/stats/proc-vmstat.rb b/stats/proc-vmstat.rb new file mode 100644 index 000000000..cd252faf1 --- /dev/null +++ b/stats/proc-vmstat.rb @@ -0,0 +1,10 @@ + +def proc_vmstat(log_lines) + result = Hash.new { |hash, key| hash[key] = [] } + log_lines.each do |line| + key, value = line.split(/:?\s+/) + result[key] << value.to_i + end + + result +end -- 2.23.0

2 2

[PATCH v2 compass-ci 2/2] lib/json_logger.rb: fix that no warp cause no puts
by Li Yuanchao 05 Mar '21

05 Mar '21

The log didn't warp before, that caused sub-fluentd can not put the log out. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/json_logger.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/json_logger.rb b/lib/json_logger.rb index 78f95ca..2c33c2f 100644 --- a/lib/json_logger.rb +++ b/lib/json_logger.rb @@ -29,7 +29,7 @@ class JSONLogger < Logger msg = { 'message' => msg } unless msg.is_a?(Hash) logger_hash.merge!(msg) - logger_hash.to_json + logger_hash.to_json + "\n" } def initialize(logdev = STDOUT, formatter = FORMATTER) -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] lib/git_mirror.rb: Add new_refs_log for kibana
by Li Yuanchao 05 Mar '21

05 Mar '21

The logs can be shown as chart by kibana. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index bb0c9f9..ae7e64b 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -11,6 +11,7 @@ require 'priority_queue' require 'English' require 'elasticsearch' require_relative 'constants.rb' +require_relative 'json_logger.rb' # worker threads class GitMirror @@ -192,6 +193,7 @@ class MirrorMain feedback_info[:new_refs] = new_refs send_message(feedback_info) + new_refs_log(git_repo, new_refs[:heads].length) end def do_push(fork_key) @@ -509,4 +511,14 @@ class MirrorMain traverse_repodir(repodir, belong) end end + + def new_refs_log(git_repo, nr_new_branch) + log = JSONLogger.new + log.info({ + msg: 'new refs', + time: Time.now, + repo: git_repo, + nr_new_branch: nr_new_branch + }.to_json) + end end -- 2.23.0

1 0

[PATCH v5 lkp-tests] daemon: add mysql
by Wei Jihui 05 Mar '21

05 Mar '21

add mysql for sysbench-mysql test Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- daemon/mysql | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100755 daemon/mysql diff --git a/daemon/mysql b/daemon/mysql new file mode 100755 index 00000000..808e3eec --- /dev/null +++ b/daemon/mysql @@ -0,0 +1,91 @@ +#!/bin/sh +# - data_dev +# mysql_user +# mysql_host +# mysql_port +# mysql_db +# mysql_password +. $LKP_SRC/lib/common.sh + +: "${mysql_user:=root}" +: "${mysql_host:=$server}" +: "${mysql_port:=3306}" +: "${mysql_db:=sysbench_test}" +: "${mysql_password:=mysql_password}" + +prepare() +{ + systemctl stop firewalld >/dev/null 2>&1 + iptables -F >/dev/null 2>&1 + systemctl start tuned >/dev/null 2>&1 +} + +insmod_getcpu() +{ + if [ -d "/getcpu" ]; then + cd /getcpu && insmod getcpu-hi.ko + else + echo "this test will not use getcpu optimize to bind core." + fi +} + +create_basedir() +{ + mkdir -p /data + mkfs.xfs -f /dev/"$data_dev" + mount /dev/"$data_dev" /data + + mkdir -p /data/mysql/{data,share,tmp,run,log} + + useradd mysql + touch /data/mysql/log/mysql.log + chown -R mysql:mysql /data + chown -R mysql:mysql /data/mysql/log/mysql.log +} + +configure_mysql() +{ + if [ ! -f $mysqld ] || [ ! -f $mysql ] || [ ! -f $mysql_server ]; then + echo "mysql is not installed!" && exit 1 + fi + + "$mysqld" --user=mysql --basedir=/data/mysql --datadir=/data/mysql/data/ --initialize --port="$mysql_port" + sleep 10 + + "$mysql_server" start + sleep 10 + + "$mysql" -u "$mysql_user" -e "flush privileges; use mysql; alter user 'root'@'localhost' identified by '$mysql_password'; flush privileges;" + sed -i '/^skip-grant-tables/d' /etc/my.cnf + "$mysql_server" restart + sleep 10 +} + +create_database() +{ + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "use mysql; update user set host ='%' where user='root'; flush privileges;" + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $mysql_db;" + "$mysql_server" stop + sleep 10 +} + +start_mysql() +{ + numactl -C 0-92 -i 0-3 "$mysqld" --defaults-file=/etc/my.cnf --port="$mysql_port" & + sleep 20 +} + +main() +{ + local mysql="/usr/local/mysql/bin/mysql" + local mysqld="/usr/local/mysql/bin/mysqld" + local mysql_server="/usr/local/mysql/support-files/mysql.server" + + prepare + insmod_getcpu + create_basedir + configure_mysql + create_database + start_mysql +} +main -- 2.23.0

1 0

[PATCH v5 compass-ci] doc/manual: add document for submit container
by Luan Shengde 05 Mar '21

05 Mar '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- doc/manual/build-lkp-tests-container.en.md | 63 ++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 doc/manual/build-lkp-tests-container.en.md diff --git a/doc/manual/build-lkp-tests-container.en.md b/doc/manual/build-lkp-tests-container.en.md new file mode 100644 index 0000000..47c85c5 --- /dev/null +++ b/doc/manual/build-lkp-tests-container.en.md @@ -0,0 +1,63 @@ +# Preface + +We provide a docker container to suit various of Linux OS(es). +In this case you do not need to install the lkp-tests to your local server. +Also you can avoid installation failures from undesired dependency package(s). + +# Prepare + + Do on your local server: + + - install docker + - apply account and config default yaml + - generate ssh key(s) + +# build container + +## 1. download resource + + Download lkp-tests and compass-ci to your local server. + + Command(s): + + git clone https://gitee.com/wu_fengguang/compass-ci.git + git clone https://gitee.com/wu_fengguang/lkp-tests.git + +## 2. add environment variable(s) + + Command(s): + + echo "export LKP_SRC=$PWD/lkp-tests" >> ~/.${SHELL##*/}rc + echo "export CCI_SRC=$PWD/compass-ci" >> ~/.${SHELL##*/}rc + source ~/.${SHELL##*/}rc + +## 3. build docker image + + Command(s): + + cd compass-ci/container/submit + ./build + +## 4. add executable file + + Command(s): + + ln -s $CCI_SRC/container/submit/submit /usr/bin/submit + +# try it + + instruction: + + You can directly use the command 'submit' to submit jobs. + It is the same as you install the lkp-tests on your own server. + It will start a disposable container to submit your job. + The container will attach the directory lkp-tests to the container itself. + You can edit the job yaml(s) in lkp-tests/jobs and it will take effect when you submit jobs. + + Example: + + submit -c -m testbox=vm-2p8g borrow-1h.yaml + + About submit: + + For detailed usage for command submit, please reference to: [submit user manual](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/su… -- 2.23.0

1 0

[PATCH lkp-tests 08/11] install-iso-pre: add step: prepare_submit
by Yu Chuan 05 Mar '21

05 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- distro/depends/install-iso-pre | 2 ++ tests/install-iso-pre | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/distro/depends/install-iso-pre b/distro/depends/install-iso-pre index 6f88eb919896..4569e1e3cdc0 100644 --- a/distro/depends/install-iso-pre +++ b/distro/depends/install-iso-pre @@ -1 +1,3 @@ lvm2 +ruby +git diff --git a/tests/install-iso-pre b/tests/install-iso-pre index e27c9ea80232..d867f1441003 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -163,7 +163,23 @@ prepare_ks() prepare_submit() { - : + ${LKP_SRC}/sbin/install-dependencies.sh + + export CCI_REPOS="/c" + + mkdir -p $CCI_REPOS && cd $CCI_REPOS + git clone $LAB_Z9_URL || die "clone lab-z9 git repo failed: $LAB_Z9_URL" + + local config_yaml="/etc/compass-ci/defaults/sparrow.yaml" + mkdir -p $(dirname $config_yaml) + cat >> $config_yaml <<-EOF + SCHED_HOST: ${SCHED_HOST} + SCHED_PORT: ${SCHED_PORT} + my_name: ${my_name} + my_email: ${my_email} + my_token: ${MY_TOKEN} + lab: z9 + EOF } submit_install_iso_yaml() -- 2.23.0

1 0

[PATCH lkp-tests 01/11] feat: step 1 of iso2rootfs: install iso to testbox and reboot
by Yu Chuan 05 Mar '21

05 Mar '21

[Why] When testbox request job in pxe -> ipxe step, scheduler returns a response composed of ipxe cmd. So we add 'custom_ipxe' field of job.yaml, then we can customize the response what testbox get, then testbox can auto install iso by executing the specified ipxe cmd. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/install-iso.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 jobs/install-iso.yaml diff --git a/jobs/install-iso.yaml b/jobs/install-iso.yaml new file mode 100644 index 000000000000..bcccc6b76f85 --- /dev/null +++ b/jobs/install-iso.yaml @@ -0,0 +1,3 @@ +suite: install-iso + +custom_ipxe: -- 2.23.0

1 0

[PATCH v3 compass-ci] src/lib/job.cr: delete os_mount in common_dir for upload rpm
by Li Ping 05 Mar '21

05 Mar '21

[why] rpm has no much to do with os_mount. Signed-off-by: Li Ping <1477412247(a)qq.com> --- src/lib/job.cr | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 0b4bee4..c07033c 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -267,14 +267,19 @@ class Job pkg_style = JSON.parse("{}") if pkg_style == nil - tmp_os = pkg_style["os"]? == nil ? "#{os}" : pkg_style["os"] - tmp_os_arch = pkg_style["os_arch"]? == nil ? "#{os_arch}" : pkg_style["os_arch"] - tmp_os_version = pkg_style["os_version"]? == nil ? "#{os_version}" : pkg_style["os_version"] + tmp_os = pkg_style["os"]? || "#{os}" + tmp_os_arch = pkg_style["os_arch"]? || "#{os_arch}" + tmp_os_version = pkg_style["os_version"]? || "#{os_version}" - tmp_os_mount = pkg_style["os_mount"]? == nil ? "#{os_mount}" : pkg_style["os_mount"] - mount_type = tmp_os_mount == "cifs" ? "nfs" : tmp_os_mount.dup + mount_type = pkg_style["os_mount"]? || "#{os_mount}" + # same usage for client + mount_type = "nfs" if mount_type == "cifs" - common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + if @hash["rpmbuild-pkg"]? + common_dir = "#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + else + common_dir = "#{mount_type}/#{tmp_os}/#{tmp_os_arch}/#{tmp_os_version}" + end return common_dir end -- 2.23.0

2 2

[PATCH v2 lkp-tests 1/2] jobs/iso2rootfs.yaml: change some field to secrets field.
by Li Ping 05 Mar '21

05 Mar '21

[why] the related fields when submit test.yaml should be add to secrets field. submit_name: submit_email: submit_token: and the required fields when submit test.yaml not have much to do with the iso2rootfs be changed to level-1 parameter. Signed-off-by: Li Ping <1477412247(a)qq.com> --- jobs/iso2rootfs.yaml | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 55c4345fa..5df2cdf61 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -51,23 +51,24 @@ iso2rootfs: # http://1.1.1.1//dailybuilds/1970-01-01-00-00-00/openEuler-20.03-LTS-aarch64… dailybuild_iso_url_file: - ################# - # submit test yaml related fields - ################# +################# +# submit test yaml related fields +################# - # three required fields when submit test yaml - submit_name: +# three required fields when submit test yaml +secrets: + submit_name: submit_email: submit_token: - # submit target tests yaml - ## 1. You can add as many jobs as you like. - ## 2. The following three fields is required for every test job. - test1_yaml: - test1_os_mount: - test1_testbox: - ## the following three fields is required when your job.yaml and - ## script for this test are from the internet. - test1_git_url: - test1_git_yaml: - test1_git_script: +# submit target tests yaml +## 1. You can add as many jobs as you like. +## 2. The following three fields is required for every test job. +test1_yaml: +test1_os_mount: +test1_testbox: +## the following three fields is required when your job.yaml and +## script for this test are from the internet. +test1_git_url: +test1_git_yaml: +test1_git_script: -- 2.23.0

1 1

[PATCH v2 lkp-tests 2/2] sbin/submit: optimize submit -r
by Li Ping 05 Mar '21

05 Mar '21

[why] submit -m -r iperf.yaml submit /home/liping/lkp-tests/jobs/iperf.yaml, got job id=z9.1183160 submit /home/liping/lkp-tests/jobs/iperf.yaml, got job id=z9.1183161 query=>{"job_id":["z9.1183160","z9.1183161"]} connect to ws://172.168.131.2:11310/filter {"level_num":1,"level":"INFO","time":"2021-03-05T11:31:36.649+0800","job_id":"z9.1183161","message":"","job_state":"submit","result_root":"/srv/result/iperf/2021-03-05/vm-2p8g/openeuler-20.03-aarch64/udp-300/z9.1183161","status_code":200,"method":"POST","resource":"/submit_job","elapsed":"24.68ms"} {"level_num":1,"level":"INFO","time":"2021-03-05T11:31:39.699+0800","job_id":"z9.1183160","result_root":"/srv/result/iperf/2021-03-05/vm-2p8g/openeuler-20.03-aarch64/tcp-300/z9.1183160","job_state":"set result root","status_code":200,"method":"GET","resource":"/boot.ipxe/mac/0a-9f-90-36-ae-6b","elapsed":"8.75ms"} {"level_num":1,"level":"INFO","time":"2021-03-05T11:31:39.735+0800","job_id":"z9.1183160","job_state":"boot","status_code":200,"method":"GET","resource":"/boot.ipxe/mac/0a-9f-90-36-ae-6b","elapsed":"44.08ms"} ==> only get one result [how] get result_root directly from the msg not from fluentd data Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/monitor.rb | 36 ++++++++++-------------------------- sbin/monitor | 8 ++++++++ sbin/submit | 4 +++- 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/lib/monitor.rb b/lib/monitor.rb index 78582be6a..24191fa8d 100755 --- a/lib/monitor.rb +++ b/lib/monitor.rb @@ -11,7 +11,7 @@ require 'eventmachine' require 'json' class Monitor - attr_accessor :monitor_url, :query, :overrides, :action, :job, :result, :stop_query + attr_accessor :monitor_url, :query, :overrides, :action, :job, :result, :result_roots, :stop_query def initialize(monitor_url = '', query = {}, action = {}) @monitor_url = monitor_url @@ -25,9 +25,9 @@ class Monitor @exit_status_code = 0 @defaults = {} @result = [] + @result_roots = [] @stop_query = {} @reason = nil - @result_root = nil end def load_default @@ -95,29 +95,13 @@ class Monitor exec cmd end - def set_result_root(data) - return unless @action['mirror_result'] - return unless data['log'] - - data = JSON.parse(data['log']) - return unless data['result_root'] - - @result_root = data['result_root'] - end - - def lftp_mirror - @result_root.delete_prefix!('/srv') - srv_http_host = job['SRV_HTTP_HOST'] || 'api.compass-ci.openeuler.org' - srv_http_port = job['SRV_HTTP_PORT'] || '11300' - url = "http://#{srv_http_host}:#{srv_http_port}#{@result_root}" - system "lftp -c mirror #{url} >/dev/null 2>&1" - end - - def mirror_result(data) - if @result_root - lftp_mirror - else - set_result_root(data) + def mirror_result + @result_roots.each do |res| + res.to_s.delete_prefix!('/srv') + srv_http_host = job['SRV_HTTP_HOST'] || 'api.compass-ci.openeuler.org' + srv_http_port = job['SRV_HTTP_PORT'] || '11300' + url = "http://#{srv_http_host}:#{srv_http_port}#{res}" + system "lftp -c mirror #{url} >/dev/null 2>&1" end end @@ -163,7 +147,7 @@ class Monitor output(data) connect(data, ws) - mirror_result(data) + mirror_result stop(data, ws) if @action['stop'] end diff --git a/sbin/monitor b/sbin/monitor index def47bf6a..a2e677fc0 100755 --- a/sbin/monitor +++ b/sbin/monitor @@ -12,6 +12,7 @@ require 'yaml' opt_set_key_value = {} opt_action = {'output' => true} opt_job = {} +opt_results = [] opt_monitor_url = nil opt_timeout = 0 opt_stop_query = {} @@ -35,6 +36,12 @@ options = OptionParser.new do |opts| opt_action[action] = true end end + + opts.on('-r result_roots', '--results results', 'set monitor result_roots') do |result_roots| + result_roots.split(',').each do |res| + opt_results << res + end + end opts.separator ' action: output, output log' opts.separator ' action: connect, connect to the host when get ip' opts.separator ' action: stop, exit monitor client when get log' @@ -79,6 +86,7 @@ monitor = Monitor.new monitor.monitor_url = opt_monitor_url if opt_monitor_url monitor.overrides = opt_set_key_value monitor.action = opt_action +monitor.result_roots = opt_results monitor.job = opt_job monitor.stop_query = opt_stop_query diff --git a/sbin/submit b/sbin/submit index a6f0cd387..44fd0a7db 100755 --- a/sbin/submit +++ b/sbin/submit @@ -19,6 +19,7 @@ opt_monitor = false opt_monitor_query = {} opt_my_queue = false actions = ['output', 'stop'] +result_roots = [] options = OptionParser.new do |opts| opts.banner = 'Usage: submit [options] job1.yaml job2.yaml ...' @@ -197,6 +198,7 @@ ARGV.each do |jobfile| messages = scheduler_client.submit_job(job_json) JSON.parse(messages).each do |msg| if msg['message'].empty? + result_roots << msg['result_root'] job_ids << msg['job_id'].to_s puts("submit #{jobfile}, got job id=#{msg['job_id']}") else @@ -210,6 +212,6 @@ end if opt_monitor job_hash_list[0].delete('define_files') opt_monitor_query.merge!({'job_id' => job_ids}) - cmd = "#{LKP_SRC}/sbin/monitor -f '#{opt_monitor_query.to_json}' -j '#{job_hash_list[0].to_json}' -a #{actions.join(',')} -s 'job_state: extract_finished'" + cmd = "#{LKP_SRC}/sbin/monitor -f '#{opt_monitor_query.to_json}' -j '#{job_hash_list[0].to_json}' -a #{actions.join(',')} -r #{result_roots.join(',')} -s 'job_state: extract_finished'" exec cmd end -- 2.23.0

1 0

2025

2024

2023

2022

2021

2020

Compass-ci