- Compass-ci - mailweb.openeuler.org

[PATCH v2 compass-ci 2/3] lib/git_bisect.rb: fix: syntax error, unexpected string literal, expecting']'
by Cao Xueliang 24 Feb '21

24 Feb '21

need add ',' after 'upstream_url' => @upstream_url Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/git_bisect.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/git_bisect.rb b/lib/git_bisect.rb index dd0b782..6aedda5 100644 --- a/lib/git_bisect.rb +++ b/lib/git_bisect.rb @@ -113,7 +113,7 @@ class GitBisect 'work_dir' => @work_dir, 'bisect_error' => bisect_error, 'all_errors' => all_errors, - 'upstream_url' => @upstream_url + 'upstream_url' => @upstream_url, 'pkgbuild_repo' => @pkgbuild_repo, 'first_bad_commit_result_root' => first_bad_commit_result_root] end -- 2.23.0

1 0

[PATCH compass-ci 3/3] delimiter: use secrets field to submit process job
by Cao Xueliang 24 Feb '21

24 Feb '21

Before: Search account info from ES. After: Use secrets field to get the my_token. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter.rb | 8 ++++++++ src/delimiter/utils.rb | 10 ++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/delimiter.rb b/src/delimiter.rb index 1ea5de5..cb43cb4 100644 --- a/src/delimiter.rb +++ b/src/delimiter.rb @@ -5,8 +5,16 @@ require_relative './delimiter/delimiter' require_relative '../lib/config_account' +def config_secrets_yaml + account = YAML.load_file "#{ENV['HOME']}/.config/compass-ci/defaults/account.yaml" + lab = YAML.load_file "#{ENV['HOME']}/.config/compass-ci/include/lab/#{account['lab']}.yaml" + secrets = Hash["secrets" => lab] + File.open("#{ENV['HOME']}/.config/compass-ci/defaults/secrets.yaml", "w") { |f| YAML.dump(secrets, f) } +end + begin config_yaml('delimiter') + config_secrets_yaml delimiter = Delimiter.new delimiter.start_delimit rescue StandardError => e diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index 0930dd7..cd026f9 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -141,14 +141,12 @@ module Utils record_jobs(job['id'], job['upstream_commit']) job['suite'] = 'bisect' - job['my_name'] = account_info['my_name'] - job['my_email'] = account_info['my_email'] - job['my_token'] = account_info['my_token'] + job['my_name'] = ENV['my_name'] + job['my_email'] = ENV['my_email'] + job['my_token'] = ENV['secrets_my_token'] job['bad_job_id'] = job_id + job['queue'] = "#{job['testbox'].split('.')[0]}-bisect" - job.delete('tboxgroup') - job.delete('subqueue') - job.delete('queue') job.delete('id') return job -- 2.23.0

1 0

[PATCH compass-ci 2/3] lib/git_bisect.rb: fix: syntax error, unexpected string literal, expecting']'
by Cao Xueliang 24 Feb '21

24 Feb '21

need add ',' after 'upstream_url' => @upstream_url Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/git_bisect.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/git_bisect.rb b/lib/git_bisect.rb index dd0b782..6aedda5 100644 --- a/lib/git_bisect.rb +++ b/lib/git_bisect.rb @@ -113,7 +113,7 @@ class GitBisect 'work_dir' => @work_dir, 'bisect_error' => bisect_error, 'all_errors' => all_errors, - 'upstream_url' => @upstream_url + 'upstream_url' => @upstream_url, 'pkgbuild_repo' => @pkgbuild_repo, 'first_bad_commit_result_root' => first_bad_commit_result_root] end -- 2.23.0

1 0

[PATCH v2 compass-ci] doc/stats-rule.md
by Wang Chenglong 24 Feb '21

24 Feb '21

[Why] The rules to follow when writing the stats script. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- doc/stats-rule.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 doc/stats-rule.md diff --git a/doc/stats-rule.md b/doc/stats-rule.md new file mode 100644 index 0000000..0fb557a --- /dev/null +++ b/doc/stats-rule.md @@ -0,0 +1,42 @@ +${LKP_SERVER}/stats/${script} stat�� +========================================== + +## ��es�� + +1.��error_id�� + - original: {"changed": false, "failures": ["No package yum-plugin-priorities available."], "msg": "Failed to + install some of the specified packages", "rc": 1, "results": []} + + - error_id: error.No-package-yum-plugin-priorities-available: 1 + +2.�� + - ��error_id�� + # �� + # {"changed": false, "cmd": "csf -v", "msg": "[Errno 2] No such file or directory: b'csf': b'csf'", "rc": 2} + when /(No such file or directory: b'.+')/ + output_error $1 + end + + - ��jobs��'��'��es��error_id.��stats/${script}�� + +3.��error message�� + # original error message + # {"changed": false, "msg": "Failed to download packages: Cannot download 7.10.2/filebeat-7.10.2-aarch64.rpm: + # All mirrors were tried", "results": []} + when /(Failed to download packages: Cannot download .+):/ + output_error $1 + end + +4.��error_id�� + - �� + def common_error_id(line) + line = line.chomp + line.gsub!(/ /, '-') + line.gsub!(/:/, '') # error.:-'__mysql_packages'-is-undefined: 1 + line.gsub!(/,/, '') + line.gsub!(/'/, '') # Failed-to-download-metadata-for-repo-'dockerrepo: 1 + line.gsub!(/!/, '') # Distribution-openEuler-is-not-supported-by-this-role!: 1 + line.gsub!(/{%/, ' ') + line.gsub!(/%}/, ' ') + line + end -- 2.23.0

2 2

[PATCH lkp-tests 3/3] tests/sysbench-mysql: fix mysql_port, mysql_password default value
by Wei Jihui 24 Feb '21

24 Feb '21

Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- tests/sysbench-mysql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/sysbench-mysql b/tests/sysbench-mysql index 5dd8686f..d6f5dcda 100755 --- a/tests/sysbench-mysql +++ b/tests/sysbench-mysql @@ -20,9 +20,9 @@ : "${mysql_user:=root}" : "${mysql_host:=$server}" -: "${mysql_port:=$mysql_port}" +: "${mysql_port:=3306}" : "${mysql_db:=sysbench_1}" -: "${mysql_password:=$mysql_password}" +: "${mysql_password:=mysql_password}" : "${db_driver:=mysql}" : "${oltp_test_mode:=complex}" : "${oltp_tables_count:=1000}" -- 2.23.0

1 0

[PATCH v3 lkp-tests] jobs: add os-multi-dimension-arch.yaml
by Li Ping 24 Feb '21

24 Feb '21

1 0

[PATCH v2 compass-ci] container/assign-account: optimize config login permission
by Luan Shengde 24 Feb '21

24 Feb '21

optimize account login permission setting: before: set enable login: has key: enable_login key value: true set disable login has key: enable_login key value: false or has no key: enable_login now: set enable login: has key: enable_login key value: true set disable login: has key: enable_login key value: false keep the default login permission: has no key: enable_login Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/assign-account/get_account_info.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/assign-account/get_account_info.rb b/container/assign-account/get_account_info.rb index 7e0a5a7..228d468 100755 --- a/container/assign-account/get_account_info.rb +++ b/container/assign-account/get_account_info.rb @@ -136,7 +136,7 @@ class AccountStorage def permit_login_config(login_name) if @data.key?('enable_login') && @data['enable_login'] %x(usermod -s /usr/bin/zsh #{login_name}) - else + elsif @data.key?('enable_login') && ! @data['enable_login'] %x(usermod -s /sbin/nologin #{login_name}) end end -- 2.23.0

1 0

[PATCH lkp-tests] add tests script for openeuler os migration
by Bai Jing 24 Feb '21

24 Feb '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 114 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100755 tests/openeuler_docker diff --git a/tests/openeuler_docker b/tests/openeuler_docker new file mode 100755 index 000000000..f79610b34 --- /dev/null +++ b/tests/openeuler_docker @@ -0,0 +1,114 @@ +#!/bin/sh +# - dockerfile_url + +. $LKP_SRC/lib/debug.sh + +[ -n "$dockerfile_url" ] || die "dockerfile_url is empty" + +cd $BENCHMARK_ROOT + +fit_repo() +{ + yum reinstall -y git + git clone "$dockerfile_url" || exit 0 + file_dir=$(find ./ -type f -name "*Dockerfile*" | xargs grep -E '^FROM centos:(centos|)[78]') + [ -n "$file_dir" ] || exit 0 +} + +openeuler_ready() +{ + docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar +} + +centos_docker() +{ + flag=0 + for file in $(find ./ -type f -name "*Dockerfile*" | xargs grep -E '^FROM centos:(centos|)[78]' | awk -F: '{print $1}') + do + image=$(echo "$file" | awk -F '/' '{print $3}' | tr 'A-Z' 'a-z') + if [ -f "build.sh" ] && [ -x "build.sh" ]; then + ./build.sh > /dev/null 2> /dev/null && handle_dockerfile + else + docker_build > /dev/null 2> /dev/null && handle_dockerfile + fi + done +} + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$file" + +} + +handle_epel() +{ + grep -qw "epel-release" "$file" && { + sed -i "s/epel-release/bash/g" "$file" + } +} + +handle_epelrpm() +{ + + grep -qw "https://.*epel-release.*rpm" "$file" && { + sed -i "s|https://.*epel-release.*rpm|bash|g" "$file" + + } +} + +handle_groupadd() +{ + grep -qw "groupadd*" "$file" && { + sed -i '/groupadd*/i\ yum -y install shadow && \\' "$file" + } + +} + +handle_epel_repo() +{ + grep -qwE '.* wget .*(epel|CentOS-Base)\.repo' "$file" && { + sed 's|^|#|g' "$file" + } +} + +handle_dockerfile() +{ + flag=1 + handle_FROM + handle_epel + handle_groupadd + docker_build "-openeuler" + [ $? -ne 0 ] || docker_run "-openeuler" +} + +docker_build() +{ + docker build -f "$file" -t "${image}$1" . + [ $? -ne 0 ] && { + [ "$flag" == "1" ] && echo "${image}$1.build.fail: 1" + return 1 + } + [ "$flag" == "1" ] && echo "${image}$1.build.success: 1" + flag=0 +} + +docker_run() +{ + docker run -d --name "${image}" "${image}$1" + docker_stats=$(docker inspect ${image} | grep Running | awk -F ' ' '{print $2}' | tr -d ',') + [ $docker_stats == false ] && { + echo "${image}.run.fail: 1" + return 1 + } + + echo "${image}.run.success: 1" +} + +docker_test() +{ + fit_repo + openeuler_ready + centos_docker +} + +docker_test -- 2.23.0

1 1

[PATCH v2 lkp-tests] jobs/: add os-multi-dimension-arch.yaml
by Li Ping 24 Feb '21

24 Feb '21

3 7

[PATCH v3 compass-ci] answerback-email: add store lab to ES
by Luan Shengde 24 Feb '21

24 Feb '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/assign-account/answerback-email.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index 8dceee1..f573f0f 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -40,7 +40,8 @@ my_info = { 'my_commit_url' => nil, 'my_token' => nil, 'my_login_name' => nil, - 'my_ssh_pubkey' => [] + 'my_ssh_pubkey' => [], + 'lab' => LAB } # stdin_info is used to store infos added with option: @@ -179,7 +180,6 @@ def apply_account(my_info, conf_info) apply_info = {} apply_info.update my_info apply_info.update conf_info - apply_info['lab'] = LAB assign_account_url = "#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account" account_info_str = RestClient.post assign_account_url, apply_info.to_json -- 2.23.0

1 0

[PATCH v6 compass-ci] doc: optimize os_mount.md
by Yu Chuan 24 Feb '21

24 Feb '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- doc/job/os_mount.md | 255 +++++++++++++++++++++++++++++++------------- 1 file changed, 178 insertions(+), 77 deletions(-) diff --git a/doc/job/os_mount.md b/doc/job/os_mount.md index 226251a8bace..f4b28bb4b4c7 100644 --- a/doc/job/os_mount.md +++ b/doc/job/os_mount.md @@ -1,9 +1,9 @@ # Summary -========= +--------- -`os_mount` defines the mount method of testbox's root partition. +`os_mount` defines the rootfs type. -It has the following optional values: +The below types are supported: - nfs - cifs - initramfs @@ -11,84 +11,185 @@ It has the following optional values: - local Usage example: - - submit iperf.yaml testbox=vm-2p8g os_mount=nfs - - submit iperf.yaml testbox=vm-2p8g os_mount=cifs - - submit iperf.yaml testbox=vm-2p8g os_mount=initramfs - - submit iperf.yaml testbox=dc-8g os_mount=container - - submit iperf.yaml testbox=vm-2p8g os_mount=local - -# Optional Values -================= - -## Work flow when os_mount=local - -1. user submit job with os_mount: local - - optional {kernel_append_root}: - - save_root_partition: give an iconic string, then the root partition data used by this job will be retained. - and you can use the root partition data of this job in the futher job by {use_root_partition}. - - use_root_partition : give an iconic string, then the root partition data generated by the - pervious job which has the {save_root_partition} field will be used. - - root_partition_size: specify the size of root partition, *default is 10G*. - - - tips: - - value of {save_root_partition} and {use_root_partition} need to be iconic enough, uuid is recommended. - - value of {save_root_partition} and {use_root_partition} will be the suffix of logical volume. - -2. scheduler return the ipxe_str to testbox + + ```shell + submit -m -c borrow-1h.yaml testbox=vm-2p8g os_mount=nfs + submit -m -c borrow-1h.yaml testbox=vm-2p8g os_mount=cifs + submit -m -c borrow-1h.yaml testbox=vm-2p16g os_mount=initramfs + submit -m -c borrow-1h.yaml testbox=dc-8g os_mount=container + submit -m -c borrow-1h.yaml testbox=taishan200-2280-2s48p-256g os_mount=local + ``` + +# os_mount=local +---------------- + +`nfs`, `cifs`, `initramfs`, `container`, rootfs of them all are in memory. + +So if you want to run your job in disk file system, you need to specify `os_mount=local` in your job.yaml. + +BTW, the job's rootfs will be placed on a logical volume. + + +## temporary rootfs data by default + +If you only add `os_mount: local` in your job.yaml, the logical volume of rootfs will be deleted and re-created in initrd stage before every job executes. + +The brief flow is as follows: + ``` - dhcp - initrd http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/{timestamp}/initrd.lkp - initrd http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/{timestamp}/boot/vmlinuz - imgargs vmlinux root=/dev/mapper/os-openeuler_aarch64_20.03 rootfs_src={nfs_server_ip}:os/openeuler/aarch64/20.03-iso-snapshots/{timestamp} initrd=initrd.lkp {kernel_append_root} - boot + 1. boot and request scheduler for job. + 2. initrd stage: + - firstly, we need two logical volume: + ${src_lv} -- src logical volume: + - will never boot from this lv, just use it as a source data, and take a snapshot of it to get the real boot logical volume. + - src_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${timestamp} + - ${timestamp} is just a version number, which is the latest rootfs version on the nfs server in cluster. + ${boot_lv} -- boot logical volume: + - will boot from this lv. + - boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version} + - if ${src_lv} not exists: create it, and rsync the rootfs from cluster nfs server. + - if ${boot_lv} exists: delete it. + - create ${boot_lv} as the snapshot of ${src_lv}. + - switch root to ${boot_lv}. + 3. boot the rootfs and execute the job. ``` -3. dracut step of boot +## persistent rootfs data + +When you need to persist the rootfs data of a job, and use it in the subsequent job(s), two fields in `kernel_append_root` will help you: `save_root_partition`, `use_root_partition`. + +The brief flow is as follows: + + ``` + 1. boot and request scheduler for job. + 2. initrd stage: + - firstly, we need two logical volume: + ${src_lv} -- src logical volume: + - if have ${use_root_partition}, src_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${use_root_partition} + - if no ${use_root_partition}, src_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${timestamp} + ${boot_lv} -- boot logical volume: + - if have ${save_root_partition}, boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${save_root_partition} + - if no ${save_root_partition}, boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version} + - if ${src_lv} not exists: + - if have ${use_root_partition}, exit 1. + - if no ${use_root_partition}, create ${src_lv}, and rsync the rootfs from cluster nfs server. + - if ${boot_lv} != ${src_lv}: + - if ${boot_lv} exists: delete it + - create ${boot_lv} as the snapshot of ${src_lv}. + - switch root to ${boot_lv}. + 3. boot the rootfs and execute the job. ``` - if have {use_root_partition}; then - base_lv_name=openeuler_aarch64_20.03_{value_of_use_root_partition} - lvdisplay /dev/mappaer/os-{base_lv_name} > /dev/null || exit 1 - - if have {save_root_partition}; then - boot_lv_name=openeuler_aarch64_20.03_{value_of_save_root_partition} - - if {save_root_partition} == {use_root_partition}; then - boot from /dev/mappaer/os-{boot_lv_name} - fi - - lvremove -f /dev/mappaer/os-{boot_lv_name} - lvcreate /dev/mappaer/os-{boot_lv_name} from /dev/mappaer/os-{base_lv_name} || exit 1 - boot from /dev/mappaer/os-{boot_lv_name} - else - boot_lv_name=openeuler_aarch64_20.03_{value_of_use_root_partition} - boot from /dev/mappaer/os-{boot_lv_name} - fi - - else: - base_lv_name=openeuler_aarch64_20.03_{timestamp} - lvdisplay /dev/mappaer/os-{base_lv_name} > /dev/null || { - # create logical volume - lvcreate -L {root_partition_size} -n {base_lv_name} os || exit 1 - - # rsync nfsroot to lvm:/dev/mapper/os-{base_lv_name} - mount -t nfs {nfs_server_ip}:os/openeuler/aarch64/20.03-iso-snapshots/{timestamp} /mnt - mkdir /mnt1 && mount /dev/mapper/os-{base_lv_name} /mnt1 - cp -a /mnt/. /mnt1/ - umount /mnt /mnt1 - - # change premission of lvm:/dev/mapper/os-{base_lv_name} to readonly - lvchange -p r /dev/mapper/os-{base_lv_name} - } - - boot_lv_name=openeuler_aarch64_20.03 - if have {save_root_partition}; then - boot_lv_name=openeuler_aarch64_20.03_{value_of_save_root_partition} - fi - - lvremove -f /dev/mappaer/os-{boot_lv_name} - lvcreate /dev/mappaer/os-{boot_lv_name} from /dev/mappaer/os-{base_lv_name} || exit 1 - boot from /dev/mappaer/os-{boot_lv_name} - fi + +Demo usage: + ``` + - in 20210218, you submit a job-20210218.yaml, and you want to persist the + rootfs data of job-20210218.yaml so that it can be used by the subsequent + jobs. + Then you need add the follow field in your job-20210218.yaml: + kernel_append_root: save_root_partition=zhangsan_local_for_iperf_20210218 + + - in 20210219, you submit a job-20210219.yaml, and you want to use the rootfs + data of job-20210218.yaml. + Then you need add the follow field in your job-20210219.yaml: + kernel_append_root: use_root_partition=zhangsan_local_for_iperf_20210218 + ``` + +Notes: + - obviously, `save_root_partition` and `use_root_partition` must be the same, and unique. + - you must `save_root_partition` firstly, then you can `use_root_partition`. otherwise, your job will fail. + +## work flow + +1. user submit job + + job fields: + ```yaml + os: openeuler + os_arch: aarch64 + os_version: 20.03 + os_mount: local + kernel_append_root: use_root_partition=zhangsan_local_for_iperf_20210218 save_root_partition=zhangsan_local_for_iperf_20210219 + ``` + +2. scheduler return the ipxe_response to testbox + + ``` + #!ipxe + dhcp + initrd http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/${timestamp}/initrd.lkp + kernel http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/${timestamp}/boot/vmlinuz root=/dev/mapper/os-openeuler_aarch64_20.03 rootfs_src=${nfs_server_ip}:os/openeuler/aarch64/20.03-iso-snapshots/${timestamp} initrd=initrd.lkp ${kernel_append_root} + boot + ``` + +3. dracut step of boot + + ```bash + analyze_kernel_cmdline_params() + { + for i in $(cat /proc/cmdline) + do + [ "$i" =~ "rootfs_src=" ] && export ROOTFS_SRC=${i#rootfs_src=} + done + + [ -z "${ROOTFS_SRC}" ] && die "cannot find var in kernel cmdline params: rootfs_src" + + export TIMESTAMP=$(basename ${ROOTFS_SRC}) + } + + rsync_src_lv() + { + local src_lv=$1 + + lvdisplay ${src_lv} > /dev/null && return + + # create logical volume + lvcreate --size 10G --name $(basename ${src_lv}) os || exit + + # rsync nfsroot to ${src_lv} + mount -t nfs ${ROOTFS_SRC} /mnt + mkdir /mnt1 && mount ${src_lv} /mnt1 + cp -a /mnt/. /mnt1/ + umount /mnt /mnt1 + + # change permission of ${src_lv} to readonly + lvchange --permission r ${src_lv} + } + + snapshot_boot_lv() + { + local src_lv=$1 + local boot_lv=$2 + + [ "$src_lv" == "$boot_lv" ] && return + + lvremove --force ${boot_lv} + lvcreate --size 10G --name $(basename ${boot_lv}) --snapshot ${src_lv} || exit + } + + main() + { + analyze_kernel_cmdline_params + + if [ -z "${use_root_partition}" ]; then + src_lv="/dev/mapper/os-openeuler_aarch64_20.03_${TIMESTAMP}" + rsync_src_lv ${src_lv} + else + src_lv="/dev/mapper/os-openeuler_aarch64_20.03_${use_root_partition}" + lvdisplay ${src_lv} > /dev/null || exit + fi + + if [ -z "${save_root_partition}" ]; then + boot_lv="/dev/mapper/os-openeuler_aarch64_20.03" + else + boot_lv="/dev/mapper/os-openeuler_aarch64_20.03_${save_root_partition}" + fi + + snapshot_boot_lv ${src_lv} ${boot_lv} + boot_from_boot_lv + } + + main + ``` 4. execute the job -- 2.23.0

1 0

[PATCH v5 compass-ci] doc: optimize os_mount.md
by Yu Chuan 24 Feb '21

24 Feb '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- doc/job/os_mount.md | 264 ++++++++++++++++++++++++++++++-------------- 1 file changed, 180 insertions(+), 84 deletions(-) diff --git a/doc/job/os_mount.md b/doc/job/os_mount.md index 226251a8bace..64e86b0ecb5f 100644 --- a/doc/job/os_mount.md +++ b/doc/job/os_mount.md @@ -1,94 +1,190 @@ -# Summary -========= +Summary +======= -`os_mount` defines the mount method of testbox's root partition. +`os_mount` defines the rootfs type. -It has the following optional values: - - nfs - - cifs - - initramfs - - container - - local +The below types are supported: + - nfs + - cifs + - initramfs + - container + - local Usage example: - - submit iperf.yaml testbox=vm-2p8g os_mount=nfs - - submit iperf.yaml testbox=vm-2p8g os_mount=cifs - - submit iperf.yaml testbox=vm-2p8g os_mount=initramfs - - submit iperf.yaml testbox=dc-8g os_mount=container - - submit iperf.yaml testbox=vm-2p8g os_mount=local - -# Optional Values -================= - -## Work flow when os_mount=local - -1. user submit job with os_mount: local - - optional {kernel_append_root}: - - save_root_partition: give an iconic string, then the root partition data used by this job will be retained. - and you can use the root partition data of this job in the futher job by {use_root_partition}. - - use_root_partition : give an iconic string, then the root partition data generated by the - pervious job which has the {save_root_partition} field will be used. - - root_partition_size: specify the size of root partition, *default is 10G*. - - - tips: - - value of {save_root_partition} and {use_root_partition} need to be iconic enough, uuid is recommended. - - value of {save_root_partition} and {use_root_partition} will be the suffix of logical volume. + ``` + submit -m -c borrow-1h.yaml testbox=vm-2p8g os_mount=nfs + submit -m -c borrow-1h.yaml testbox=vm-2p8g os_mount=cifs + submit -m -c borrow-1h.yaml testbox=vm-2p16g os_mount=initramfs + submit -m -c borrow-1h.yaml testbox=dc-8g os_mount=container + submit -m -c borrow-1h.yaml testbox=taishan200-2280-2s48p-256g os_mount=local + ``` + +os_mount=local +============== + +`nfs`, `cifs`, `initramfs`, `container`, rootfs of them all are in memory. +So if you want to run your job in disk file system, you need to specify +`os_mount=local` in your job.yaml. + +BTW, the job's rootfs will be placed on a logical volume. + + +## temporary rootfs data by default + +If you only add `os_mount: local` in your job.yaml, the logical volume of rootfs +will be deleted and re-created in initrd stage before every job executes. + +The brief flow is as follows: + ``` + 1. boot and request scheduler for job. + 2. initrd stage: + - firstly, we need two logical volume: + ${src_lv} -- src logical volume: + - will never boot from this lv, just use it as a source data, and take a + snapshot of it to get the real boot logical volume. + - src_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${timestamp} + - ${timestamp} is just a version number, which is the latest rootfs version + on the nfs server in cluster. + ${boot_lv} -- boot logical volume: + - will boot from this lv. + - boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version} + - if ${src_lv} not exists: create it, and rsync the rootfs from cluster nfs server. + - if ${boot_lv} exists: delete it + - create ${boot_lv} as the snapshot of ${src_lv}. + - switch root to ${boot_lv}. + 3. boot the rootfs and execute the job. + ``` + +## persistent rootfs data + +When you need to persist the rootfs data of a job, and use it in the subsequent job(s), +two fields in `kernel_append_root` will help you: `save_root_partition`, `use_root_partition`. + +The brief flow is as follows: + ``` + 1. boot and request scheduler for job. + 2. initrd stage: + - firstly, we need two logical volume: + ${src_lv} -- src logical volume: + - if have ${use_root_partition}, src_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${use_root_partition} + - if no ${use_root_partition}, src_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${timestamp} + ${boot_lv} -- boot logical volume: + - if have ${save_root_partition}, boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${save_root_partition} + - if no ${save_root_partition}, boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version} + - if ${src_lv} not exists: + - if have ${use_root_partition}, exit 1. + - if no ${use_root_partition}, create ${src_lv}, and rsync the rootfs from cluster nfs server. + - if ${boot_lv} != ${src_lv}: + - if ${boot_lv} exists: delete it + - create ${boot_lv} as the snapshot of ${src_lv}. + - switch root to ${boot_lv}. + 3. boot the rootfs and execute the job. + ``` + +Demo usage: + ``` + - in 20210218, you submit a job-20210218.yaml, and you want to persist the rootfs data of + job-20210218.yaml so that it can be used by the subsequent jobs. + Then you need add the follow field in your job-20210218.yaml: + kernel_append_root: save_root_partition=zhangsan_local_for_iperf_20210218 + + - in 20210219, you submit a job-20210219.yaml, and you want to use the rootfs data of + job-20210218.yaml. + Then you need add the follow field in your job-20210219.yaml: + kernel_append_root: use_root_partition=zhangsan_local_for_iperf_20210218 + ``` + +Notes: + - obviously, ${save_root_partition} and ${use_root_partition} must be the same, and unique. + - you must 'save_root_partition' firstly, then you can 'use_root_partition'. otherwise, your job will fail. + +## work flow + +1. user submit job + job fields: + ``` + os: openeuler + os_arch: aarch64 + os_version: 20.03 + os_mount: local + kernel_append_root: use_root_partition=zhangsan_local_for_iperf_20210218 save_root_partition=zhangsan_local_for_iperf_20210219 + ``` 2. scheduler return the ipxe_str to testbox - ``` - dhcp - initrd http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/{timestamp}/initrd.lkp - initrd http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/{timestamp}/boot/vmlinuz - imgargs vmlinux root=/dev/mapper/os-openeuler_aarch64_20.03 rootfs_src={nfs_server_ip}:os/openeuler/aarch64/20.03-iso-snapshots/{timestamp} initrd=initrd.lkp {kernel_append_root} - boot - ``` + ``` + dhcp + initrd http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/${timestamp}/initrd.lkp + kernel http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/${timestamp}/boot/vmlinuz root=/dev/mapper/os-openeuler_aarch64_20.03 rootfs_src=${nfs_server_ip}:os/openeuler/aarch64/20.03-iso-snapshots/${timestamp} initrd=initrd.lkp ${kernel_append_root} + boot + ``` 3. dracut step of boot - ``` - if have {use_root_partition}; then - base_lv_name=openeuler_aarch64_20.03_{value_of_use_root_partition} - lvdisplay /dev/mappaer/os-{base_lv_name} > /dev/null || exit 1 - - if have {save_root_partition}; then - boot_lv_name=openeuler_aarch64_20.03_{value_of_save_root_partition} - - if {save_root_partition} == {use_root_partition}; then - boot from /dev/mappaer/os-{boot_lv_name} - fi - - lvremove -f /dev/mappaer/os-{boot_lv_name} - lvcreate /dev/mappaer/os-{boot_lv_name} from /dev/mappaer/os-{base_lv_name} || exit 1 - boot from /dev/mappaer/os-{boot_lv_name} - else - boot_lv_name=openeuler_aarch64_20.03_{value_of_use_root_partition} - boot from /dev/mappaer/os-{boot_lv_name} - fi - - else: - base_lv_name=openeuler_aarch64_20.03_{timestamp} - lvdisplay /dev/mappaer/os-{base_lv_name} > /dev/null || { - # create logical volume - lvcreate -L {root_partition_size} -n {base_lv_name} os || exit 1 - - # rsync nfsroot to lvm:/dev/mapper/os-{base_lv_name} - mount -t nfs {nfs_server_ip}:os/openeuler/aarch64/20.03-iso-snapshots/{timestamp} /mnt - mkdir /mnt1 && mount /dev/mapper/os-{base_lv_name} /mnt1 - cp -a /mnt/. /mnt1/ - umount /mnt /mnt1 - - # change premission of lvm:/dev/mapper/os-{base_lv_name} to readonly - lvchange -p r /dev/mapper/os-{base_lv_name} - } - - boot_lv_name=openeuler_aarch64_20.03 - if have {save_root_partition}; then - boot_lv_name=openeuler_aarch64_20.03_{value_of_save_root_partition} - fi - - lvremove -f /dev/mappaer/os-{boot_lv_name} - lvcreate /dev/mappaer/os-{boot_lv_name} from /dev/mappaer/os-{base_lv_name} || exit 1 - boot from /dev/mappaer/os-{boot_lv_name} - fi - ``` + ``` + analyze_kernel_cmdline_params() + { + for i in $(cat /proc/cmdline) + do + [ "$i" =~ "rootfs_src=" ] && export ROOTFS_SRC=${i#rootfs_src=} + done + + [ -z "${ROOTFS_SRC}" ] && die "cannot find var in kernel cmdline params: rootfs_src" + + export TIMESTAMP=$(basename ${ROOTFS_SRC}) + } + + rsync_src_lv() + { + local src_lv=$1 + + lvdisplay ${src_lv} > /dev/null && return + + # create logical volume + lvcreate --size 10G --name $(basename ${src_lv}) os || exit + + # rsync nfsroot to ${src_lv} + mount -t nfs ${ROOTFS_SRC} /mnt + mkdir /mnt1 && mount ${src_lv} /mnt1 + cp -a /mnt/. /mnt1/ + umount /mnt /mnt1 + + # change permission of ${src_lv} to readonly + lvchange --permission r ${src_lv} + } + + snapshot_boot_lv() + { + local src_lv=$1 + local boot_lv=$2 + + [ "$src_lv" == "$boot_lv" ] && return + + lvremove --force ${boot_lv} + lvcreate --size 10G --name $(basename ${boot_lv}) --snapshot ${src_lv} || exit + } + + main() + { + analyze_kernel_cmdline_params + + if [ -z "${use_root_partition}" ]; then + src_lv="/dev/mapper/os-openeuler_aarch64_20.03_${TIMESTAMP}" + rsync_src_lv ${src_lv} + else + src_lv="/dev/mapper/os-openeuler_aarch64_20.03_${use_root_partition}" + lvdisplay ${src_lv} > /dev/null || exit + fi + + if [ -z "${save_root_partition}" ]; then + boot_lv="/dev/mapper/os-openeuler_aarch64_20.03" + else + boot_lv="/dev/mapper/os-openeuler_aarch64_20.03_${save_root_partition}" + fi + + snapshot_boot_lv ${src_lv} ${boot_lv} + boot_from_boot_lv + } + + main + ``` 4. execute the job -- 2.23.0

2 2

[PATCH compass-ci] doc/manual/submit-job.md: fix link error
by Wei Jihui 24 Feb '21

24 Feb '21

the links file name has changed to english. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- doc/manual/submit-job.en.md | 2 +- doc/manual/submit-job.zh.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/submit-job.en.md b/doc/manual/submit-job.en.md index 2748fbe..de73f8f 100644 --- a/doc/manual/submit-job.en.md +++ b/doc/manual/submit-job.en.md @@ -2,7 +2,7 @@ ### Prerequisites -The Compass-CI client has been installed. For details, see [Installing the Local Compass-CI Client](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/本地…. +The Compass-CI client has been installed. For details, see [Installing the Local Compass-CI Client](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/in…. ### Purpose diff --git a/doc/manual/submit-job.zh.md b/doc/manual/submit-job.zh.md index 92faa5c..77e9566 100644 --- a/doc/manual/submit-job.zh.md +++ b/doc/manual/submit-job.zh.md @@ -2,7 +2,7 @@ ### 前提条件 -已经按照[本地安装compass-ci客户端.md](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/本地安装compass-ci客户端.md )完成安装 +已经按照[本地安装compass-ci客户端](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/install-cci-client.md )完成安装 ### 概述 -- 2.23.0

1 0

[PATCH compass-ci] doc/manual/submit-job.md: fix link error
by Wei Jihui 24 Feb '21

24 Feb '21

the links file name has changed to english. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- doc/manual/submit-job.en.md | 2 +- doc/manual/submit-job.zh.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/submit-job.en.md b/doc/manual/submit-job.en.md index 2748fbe..de73f8f 100644 --- a/doc/manual/submit-job.en.md +++ b/doc/manual/submit-job.en.md @@ -2,7 +2,7 @@ ### Prerequisites -The Compass-CI client has been installed. For details, see [Installing the Local Compass-CI Client](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/��…. +The Compass-CI client has been installed. For details, see [Installing the Local Compass-CI Client](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/in…. ### Purpose diff --git a/doc/manual/submit-job.zh.md b/doc/manual/submit-job.zh.md index 92faa5c..77e9566 100644 --- a/doc/manual/submit-job.zh.md +++ b/doc/manual/submit-job.zh.md @@ -2,7 +2,7 @@ ### �� -��[��compass-ci��.md](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/��compass-ci��.md )�� +��[��compass-ci��](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/install-cci-client.md )�� ### �� -- 2.23.0

1 0

[PATCH compass-ci] doc/stats-rule.md
by Wang Chenglong 24 Feb '21

24 Feb '21

[Why] The rules to follow when writing the stats script. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- doc/stats-rule.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 doc/stats-rule.md diff --git a/doc/stats-rule.md b/doc/stats-rule.md new file mode 100644 index 0000000..94affa7 --- /dev/null +++ b/doc/stats-rule.md @@ -0,0 +1,37 @@ +${LKP_SERVER}/stats/${script} stat�� +========================================== + +## ��es�� + +1.��error_id�� + - original: {"changed": false, "failures": ["No package yum-plugin-priorities available."], "msg": "Failed to + install some of the specified packages", "rc": 1, "results": []} + + - error_id: error.No-package-yum-plugin-priorities-available: 1 + +2.�� + - catch all kind errors + ��error_id�� + ��jobs��'��'��es��error_id.��stats/${script}�� + +3.��error message�� + # original error message + # {"changed": false, "msg": "Failed to download packages: Cannot download 7.10.2/filebeat-7.10.2-aarch64.rpm: + # All mirrors were tried", "results": []} + when /(Failed to download packages: Cannot download .+):/ + output_error $1 + end + +4.��error_id�� + - �� + def common_error_id(line) + line = line.chomp + line.gsub!(/ /, '-') + line.gsub!(/:/, '') # error.:-'__mysql_packages'-is-undefined: 1 + line.gsub!(/,/, '') + line.gsub!(/'/, '') # Failed-to-download-metadata-for-repo-'dockerrepo: 1 + line.gsub!(/!/, '') # Distribution-openEuler-is-not-supported-by-this-role!: 1 + line.gsub!(/{%/, ' ') + line.gsub!(/%}/, ' ') + line + end -- 2.23.0

2 2

[PATCH lab-z9] hosts/*: add default hdd_partitions
by Cui Yili 24 Feb '21

24 Feb '21

Signed-off-by: Cui Yili <2268260388(a)qq.com> --- hosts/vm-1p1g | 1 + hosts/vm-2p16g | 1 + hosts/vm-2p1g | 1 + hosts/vm-2p4g | 1 + hosts/vm-2p8g | 1 + 5 files changed, 5 insertions(+) diff --git a/hosts/vm-1p1g b/hosts/vm-1p1g index 3af3646..b278e80 100644 --- a/hosts/vm-1p1g +++ b/hosts/vm-1p1g @@ -4,3 +4,4 @@ template: kvm nr_node: 1 nr_cpu: 1 memory: 1G +hdd_partitions: /dev/vda diff --git a/hosts/vm-2p16g b/hosts/vm-2p16g index 10847a7..cd49afc 100644 --- a/hosts/vm-2p16g +++ b/hosts/vm-2p16g @@ -4,3 +4,4 @@ template: kvm nr_node: 1 nr_cpu: 2 memory: 16G +hdd_partitions: /dev/vda diff --git a/hosts/vm-2p1g b/hosts/vm-2p1g index 0293ce9..4fffd2a 100644 --- a/hosts/vm-2p1g +++ b/hosts/vm-2p1g @@ -4,3 +4,4 @@ template: kvm nr_node: 1 nr_cpu: 2 memory: 1G +hdd_partitions: /dev/vda diff --git a/hosts/vm-2p4g b/hosts/vm-2p4g index 219c4c9..41d0a3b 100644 --- a/hosts/vm-2p4g +++ b/hosts/vm-2p4g @@ -4,3 +4,4 @@ template: kvm nr_node: 1 nr_cpu: 2 memory: 4G +hdd_partitions: /dev/vda diff --git a/hosts/vm-2p8g b/hosts/vm-2p8g index 6249322..8bc5cad 100644 --- a/hosts/vm-2p8g +++ b/hosts/vm-2p8g @@ -4,3 +4,4 @@ template: kvm nr_node: 1 nr_cpu: 2 memory: 8G +hdd_partitions: /dev/vda -- 2.23.0

2 1

[PATCH compass-ci] doc/manual/submit-job.md: fix link error
by Wei Jihui 24 Feb '21

24 Feb '21

the links file name has changed to english. Signed-off-by: Wei Jihui <weijihuiall(a)163.com> --- doc/manual/submit-job.en.md | 2 +- doc/manual/submit-job.zh.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/submit-job.en.md b/doc/manual/submit-job.en.md index 2748fbe..de73f8f 100644 --- a/doc/manual/submit-job.en.md +++ b/doc/manual/submit-job.en.md @@ -2,7 +2,7 @@ ### Prerequisites -The Compass-CI client has been installed. For details, see [Installing the Local Compass-CI Client](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/��…. +The Compass-CI client has been installed. For details, see [Installing the Local Compass-CI Client](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/in…. ### Purpose diff --git a/doc/manual/submit-job.zh.md b/doc/manual/submit-job.zh.md index 92faa5c..77e9566 100644 --- a/doc/manual/submit-job.zh.md +++ b/doc/manual/submit-job.zh.md @@ -2,7 +2,7 @@ ### �� -��[��compass-ci��.md](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/��compass-ci��.md )�� +��[��compass-ci��](https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/install-cci-client.md )�� ### �� -- 2.23.0

1 0

[PATCH v4 compass-ci] doc: optimize os_mount.md
by Yu Chuan 24 Feb '21

24 Feb '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- doc/job/os_mount.md | 265 ++++++++++++++++++++++++++++++-------------- 1 file changed, 181 insertions(+), 84 deletions(-) diff --git a/doc/job/os_mount.md b/doc/job/os_mount.md index 226251a8bace..1e6b349a136d 100644 --- a/doc/job/os_mount.md +++ b/doc/job/os_mount.md @@ -1,94 +1,191 @@ -# Summary -========= +Summary +======= -`os_mount` defines the mount method of testbox's root partition. +`os_mount` defines the rootfs type. -It has the following optional values: - - nfs - - cifs - - initramfs - - container - - local +The below types are supported: + - nfs + - cifs + - initramfs + - container + - local Usage example: - - submit iperf.yaml testbox=vm-2p8g os_mount=nfs - - submit iperf.yaml testbox=vm-2p8g os_mount=cifs - - submit iperf.yaml testbox=vm-2p8g os_mount=initramfs - - submit iperf.yaml testbox=dc-8g os_mount=container - - submit iperf.yaml testbox=vm-2p8g os_mount=local - -# Optional Values -================= - -## Work flow when os_mount=local - -1. user submit job with os_mount: local - - optional {kernel_append_root}: - - save_root_partition: give an iconic string, then the root partition data used by this job will be retained. - and you can use the root partition data of this job in the futher job by {use_root_partition}. - - use_root_partition : give an iconic string, then the root partition data generated by the - pervious job which has the {save_root_partition} field will be used. - - root_partition_size: specify the size of root partition, *default is 10G*. - - - tips: - - value of {save_root_partition} and {use_root_partition} need to be iconic enough, uuid is recommended. - - value of {save_root_partition} and {use_root_partition} will be the suffix of logical volume. + ``` + submit -m -c borrow-1h.yaml testbox=vm-2p8g os_mount=nfs + submit -m -c borrow-1h.yaml testbox=vm-2p8g os_mount=cifs + submit -m -c borrow-1h.yaml testbox=vm-2p16g os_mount=initramfs + submit -m -c borrow-1h.yaml testbox=dc-8g os_mount=container + submit -m -c borrow-1h.yaml testbox=taishan200-2280-2s48p-256g os_mount=local + ``` + +os_mount=local +============== + +`nfs`, `cifs`, `initramfs`, `container`, rootfs of them all are in memory. +So if you want to run your job in disk file system, you need to specify +`os_mount=local` in your job.yaml. + +BTW, the job's rootfs will be placed on a logical volume. + +## description + +### temporary rootfs data *Defaultly* + +If you only add `os_mount: local` in your job.yaml, the logical volume of rootfs +will be deleted and re-created in initrd stage before every job executes. + +The brief flow is as follows: + ``` + 1. boot and request scheduler for job. + 2. initrd stage: + - firstly, we need two logical volume: + ${src_lv} -- src logical volume: + - will never boot from this lv, just use it as a source data, and take a + snapshot of it to get the real boot logical volume. + - src_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${timestamp} + - ${timestamp} is just a version number, which is the latest rootfs version + on the nfs server in cluster. + ${boot_lv} -- boot logical volume: + - will boot from this lv. + - boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version} + - if ${src_lv} not exists: create it, and rsync the rootfs from cluster nfs server. + - if ${boot_lv} exists: delete it + - create ${boot_lv} as the snapshot of ${src_lv}. + - switch root to ${boot_lv}. + 3. boot the rootfs and execute the job. + ``` + +### persistent rootfs data + +When you need to persist the rootfs data of a job, and use it in the subsequent job(s), +two fields in `kernel_append_root` will help you: `save_root_partition`, `use_root_partition`. + +The brief flow is as follows: + ``` + 1. boot and request scheduler for job. + 2. initrd stage: + - firstly, we need two logical volume: + ${src_lv} -- src logical volume: + - if have ${use_root_partition}, src_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${use_root_partition} + - if no ${use_root_partition}, src_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${timestamp} + ${boot_lv} -- boot logical volume: + - if have ${save_root_partition}, boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${save_root_partition} + - if no ${save_root_partition}, boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version} + - if ${src_lv} not exists: + - if have ${use_root_partition}, exit 1. + - if no ${use_root_partition}, create ${src_lv}, and rsync the rootfs from cluster nfs server. + - if ${boot_lv} != ${src_lv}: + - if ${boot_lv} exists: delete it + - create ${boot_lv} as the snapshot of ${src_lv}. + - switch root to ${boot_lv}. + 3. boot the rootfs and execute the job. + ``` + +Demo usage: + ``` + - in 20210218, you submit a job-20210218.yaml, and you want to persist the rootfs data of + job-20210218.yaml so that it can be used by the subsequent jobs. + Then you need add the follow field in your job-20210218.yaml: + kernel_append_root: save_root_partition=zhangsan_local_for_iperf_20210218 + + - in 20210219, you submit a job-20210219.yaml, and you want to use the rootfs data of + job-20210218.yaml. + Then you need add the follow field in your job-20210219.yaml: + kernel_append_root: use_root_partition=zhangsan_local_for_iperf_20210218 + ``` + +Notes: + - obviously, ${save_root_partition} and ${use_root_partition} must be the same, and unique. + - you must 'save_root_partition' firstly, then you can 'use_root_partition'. otherwise, your job will fail. + +## work flow + +1. user submit job + job fields: + ``` + os: openeuler + os_arch: aarch64 + os_version: 20.03 + os_mount: local + kernel_append_root: use_root_partition=zhangsan_local_for_iperf_20210218 save_root_partition=zhangsan_local_for_iperf_20210219 + ``` 2. scheduler return the ipxe_str to testbox - ``` - dhcp - initrd http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/{timestamp}/initrd.lkp - initrd http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/{timestamp}/boot/vmlinuz - imgargs vmlinux root=/dev/mapper/os-openeuler_aarch64_20.03 rootfs_src={nfs_server_ip}:os/openeuler/aarch64/20.03-iso-snapshots/{timestamp} initrd=initrd.lkp {kernel_append_root} - boot - ``` + ``` + dhcp + initrd http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/${timestamp}/initrd.lkp + kernel http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/${timestamp}/boot/vmlinuz root=/dev/mapper/os-openeuler_aarch64_20.03 rootfs_src=${nfs_server_ip}:os/openeuler/aarch64/20.03-iso-snapshots/${timestamp} initrd=initrd.lkp ${kernel_append_root} + boot + ``` 3. dracut step of boot - ``` - if have {use_root_partition}; then - base_lv_name=openeuler_aarch64_20.03_{value_of_use_root_partition} - lvdisplay /dev/mappaer/os-{base_lv_name} > /dev/null || exit 1 - - if have {save_root_partition}; then - boot_lv_name=openeuler_aarch64_20.03_{value_of_save_root_partition} - - if {save_root_partition} == {use_root_partition}; then - boot from /dev/mappaer/os-{boot_lv_name} - fi - - lvremove -f /dev/mappaer/os-{boot_lv_name} - lvcreate /dev/mappaer/os-{boot_lv_name} from /dev/mappaer/os-{base_lv_name} || exit 1 - boot from /dev/mappaer/os-{boot_lv_name} - else - boot_lv_name=openeuler_aarch64_20.03_{value_of_use_root_partition} - boot from /dev/mappaer/os-{boot_lv_name} - fi - - else: - base_lv_name=openeuler_aarch64_20.03_{timestamp} - lvdisplay /dev/mappaer/os-{base_lv_name} > /dev/null || { - # create logical volume - lvcreate -L {root_partition_size} -n {base_lv_name} os || exit 1 - - # rsync nfsroot to lvm:/dev/mapper/os-{base_lv_name} - mount -t nfs {nfs_server_ip}:os/openeuler/aarch64/20.03-iso-snapshots/{timestamp} /mnt - mkdir /mnt1 && mount /dev/mapper/os-{base_lv_name} /mnt1 - cp -a /mnt/. /mnt1/ - umount /mnt /mnt1 - - # change premission of lvm:/dev/mapper/os-{base_lv_name} to readonly - lvchange -p r /dev/mapper/os-{base_lv_name} - } - - boot_lv_name=openeuler_aarch64_20.03 - if have {save_root_partition}; then - boot_lv_name=openeuler_aarch64_20.03_{value_of_save_root_partition} - fi - - lvremove -f /dev/mappaer/os-{boot_lv_name} - lvcreate /dev/mappaer/os-{boot_lv_name} from /dev/mappaer/os-{base_lv_name} || exit 1 - boot from /dev/mappaer/os-{boot_lv_name} - fi - ``` + ``` + analysis_kernel_cmdline_params() + { + for i in $(cat /proc/cmdline) + do + [ "$i" =~ "rootfs_src=" ] && export ROOTFS_SRC=${i#rootfs_src=} + done + + [ -z "${ROOTFS_SRC}" ] && die "cannot find var in kernel cmdline params: rootfs_src" + + export TIMESTAMP=$(basename ${ROOTFS_SRC}) + } + + rsync_src_lv() + { + local src_lv=$1 + + lvdisplay ${src_lv} > /dev/null && return + + # create logical volume + lvcreate --size 10G --name $(basename ${src_lv}) os || exit + + # rsync nfsroot to ${src_lv} + mount -t nfs ${ROOTFS_SRC} /mnt + mkdir /mnt1 && mount ${src_lv} /mnt1 + cp -a /mnt/. /mnt1/ + umount /mnt /mnt1 + + # change permission of ${src_lv} to readonly + lvchange --permission r ${src_lv} + } + + snapshot_boot_lv() + { + local src_lv=$1 + local boot_lv=$2 + + [ "$src_lv" == "$boot_lv" ] && return + + lvremove --force ${boot_lv} + lvcreate --size 10G --name $(basename ${boot_lv}) --snapshot ${src_lv} || exit + } + + main() + { + analysis_kernel_cmdline_params + + if [ -z "${use_root_partition}" ]; then + src_lv="/dev/mapper/os-openeuler_aarch64_20.03_${TIMESTAMP}" + rsync_src_lv ${src_lv} + else + src_lv="/dev/mapper/os-openeuler_aarch64_20.03_${use_root_partition}" + lvdisplay ${src_lv} > /dev/null || exit + fi + + if [ -z "${save_root_partition}" ]; then + boot_lv="/dev/mapper/os-openeuler_aarch64_20.03" + else + boot_lv="/dev/mapper/os-openeuler_aarch64_20.03_${save_root_partition}" + fi + + snapshot_boot_lv ${src_lv} ${boot_lv} + boot_from_boot_lv + } + + main() + ``` 4. execute the job -- 2.23.0

2 2

[PATCH lkp-tests] jobs/: add os-multi-dimension-arch.yaml
by Li Ping 23 Feb '21

23 Feb '21

2 2

[PATCH lkp-tests] lib: export secrets.yaml in job_init
by Cao Xueliang 23 Feb '21

23 Feb '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/job-init.sh | 2 ++ lib/job.sh | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/job-init.sh b/lib/job-init.sh index 1fb898872..27d3c6d0b 100755 --- a/lib/job-init.sh +++ b/lib/job-init.sh @@ -1,4 +1,5 @@ . $LKP_SRC/lib/upload.sh +. $LKP_SRC/lib/job.sh mount_cgroup() { @@ -270,4 +271,5 @@ job_init() job_redirect_stdout_stderr job_env + read_secret_vars } diff --git a/lib/job.sh b/lib/job.sh index dc65f68d8..87ca7ee67 100755 --- a/lib/job.sh +++ b/lib/job.sh @@ -1,8 +1,20 @@ #!/bin/sh +read_secret_vars() +{ + read_yaml_vars "$(dirname $job_file)/secrets.yaml" secrets_ +} + read_env_vars() { - [ -f "$TMP/env.yaml" ] || return 0 + read_yaml_vars "$TMP/env.yaml" +} + +read_yaml_vars() +{ + local file=$1 + local prefix=$2 + [ -f "$file" ] || return 0 local key local val @@ -11,8 +23,8 @@ read_env_vars() do [ "${key%[a-zA-Z0-9_]:}" != "$key" ] || continue key=${key%:} - export "$key=$val" - done < $TMP/env.yaml + export "$prefix$key=$val" + done < $file return 0 } -- 2.23.0

1 0

find latest github git pushes
by Wu Fengguang 23 Feb '21

23 Feb '21

Hi Yuanchao, I researched github and have 2 finding 1) a registered user can watch a number of repos and get notifications. however the notification type does not include "git push/new commit". There is only "new pull request", which looks much less useful for us. 2) we can still take advantage of its search API to find latest updated repos, like this https://github.com/search?l=&o=desc&q=pushed%3A%3E2021-02-21+filename%3AREA… 7,543 repository results clean-code-craft-tcq-1/modular-c-AnPra7 modular-c-AnPra7 created by GitHub Classroom C MIT license Updated 3 minutes ago trpo2021/chessviz-Jamesss1337 chessviz-Jamesss1337 created by GitHub Classroom C Updated 3 minutes ago mubu43/ExploringC C Updated 3 minutes ago BlankyShi/gitTest gitTest C Updated 4 minutes ago aabelque/malloc C Updated 4 minutes ago ythan88/r-card C Updated 4 minutes ago koipkoi/pokeemerald-kr A korean translation project for Pokémon Emerald. C Updated 4 minutes ago OpenMPT/openmpt Official read-only git mirror of the OpenMPT and libopenmpt subversion repository at https://source.openmpt.org/. Thi… libopenmpt tracker audio music demoscene music-creation protracker impulsetracker screamtracker fasttracker 223 C Updated 4 minutes ago silabs-DenverL/ADCxSyncCapture ADC0 and ADC1 Synchronization Capture C MIT license Updated 4 minutes ago gopro/gopro-lib-node.gl OpenGL engine for building and rendering graph-based scenes 32 C Apache-2.0 license Updated 4 minutes ago Thanks, Fengguang

1 2

[PATCH compass-ci] 2-network/iptables: fix iptables holding the xtables lock
by Cui Yili 23 Feb '21

23 Feb '21

[why] error: Another app is currently holding the xtables lock. Perhaps you want to use the -w option? [how] add lock wait time. iptables -w 10... Signed-off-by: Cui Yili <2268260388(a)qq.com> --- sparrow/2-network/iptables | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sparrow/2-network/iptables b/sparrow/2-network/iptables index 48794f5..73f9749 100755 --- a/sparrow/2-network/iptables +++ b/sparrow/2-network/iptables @@ -8,9 +8,9 @@ BR0_IFACE=br0 BR0_SUBNET=172.18.0.0/16 # iptables -t nat -F -iptables -I FORWARD 1 -j ACCEPT -iptables -t nat -A POSTROUTING -o "$PUB_IFACE" -s $BR0_SUBNET -j MASQUERADE -iptables -t nat -A POSTROUTING -o $BR0_IFACE -d $BR0_SUBNET -j MASQUERADE +iptables -w 10 -I FORWARD 1 -j ACCEPT +iptables -w 10 -t nat -A POSTROUTING -o "$PUB_IFACE" -s $BR0_SUBNET -j MASQUERADE +iptables -w 10 -t nat -A POSTROUTING -o $BR0_IFACE -d $BR0_SUBNET -j MASQUERADE command -v firewall-cmd > /dev/null || { exit 0 -- 2.23.0

1 0

[PATCH compass-ci] support-os-os_arch-os_version.md: delete the cifs method of archlinux
by Wang Chenglong 23 Feb '21

23 Feb '21

[Why] When start archlinux with CIFS, the system will get mangled names error. So only show the NFS method for user. Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- doc/support-os-os_arch-os_version.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/support-os-os_arch-os_version.md b/doc/support-os-os_arch-os_version.md index 5d30486..93fb47b 100644 --- a/doc/support-os-os_arch-os_version.md +++ b/doc/support-os-os_arch-os_version.md @@ -18,5 +18,5 @@ centos aarch64 7 container, 7.8.2003 cifs/nfs, debian aarch64 sid cifs/nfs, ,initramfs x86 sid cifs/nfs, -archlinux aarch64 2020-11-12 cifs/nfs, +archlinux aarch64 2020-11-12 nfs, ``` -- 2.23.0

1 0

[PATCH v3 compass-ci] doc: optimize os_mount.md
by Yu Chuan 23 Feb '21

23 Feb '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- doc/job/os_mount.md | 263 ++++++++++++++++++++++++++++++-------------- 1 file changed, 179 insertions(+), 84 deletions(-) diff --git a/doc/job/os_mount.md b/doc/job/os_mount.md index 226251a8bace..0d0afd6469a8 100644 --- a/doc/job/os_mount.md +++ b/doc/job/os_mount.md @@ -1,94 +1,189 @@ -# Summary -========= +Summary +======= -`os_mount` defines the mount method of testbox's root partition. +`os_mount` defines the rootfs type. -It has the following optional values: - - nfs - - cifs - - initramfs - - container - - local +The below types are supported: + - nfs + - cifs + - initramfs + - container + - local Usage example: - - submit iperf.yaml testbox=vm-2p8g os_mount=nfs - - submit iperf.yaml testbox=vm-2p8g os_mount=cifs - - submit iperf.yaml testbox=vm-2p8g os_mount=initramfs - - submit iperf.yaml testbox=dc-8g os_mount=container - - submit iperf.yaml testbox=vm-2p8g os_mount=local - -# Optional Values -================= - -## Work flow when os_mount=local - -1. user submit job with os_mount: local - - optional {kernel_append_root}: - - save_root_partition: give an iconic string, then the root partition data used by this job will be retained. - and you can use the root partition data of this job in the futher job by {use_root_partition}. - - use_root_partition : give an iconic string, then the root partition data generated by the - pervious job which has the {save_root_partition} field will be used. - - root_partition_size: specify the size of root partition, *default is 10G*. - - - tips: - - value of {save_root_partition} and {use_root_partition} need to be iconic enough, uuid is recommended. - - value of {save_root_partition} and {use_root_partition} will be the suffix of logical volume. + ``` + submit -m -c borrow-1h.yaml testbox=vm-2p8g os_mount=nfs + submit -m -c borrow-1h.yaml testbox=vm-2p8g os_mount=cifs + submit -m -c borrow-1h.yaml testbox=vm-2p16g os_mount=initramfs + submit -m -c borrow-1h.yaml testbox=dc-8g os_mount=container + submit -m -c borrow-1h.yaml testbox=taishan200-2280-2s48p-256g os_mount=local + ``` + +os_mount=local +============== + +`nfs`, `cifs`, `initramfs`, `container`, rootfs of them all are in memory. +So if you want to run your job in disk file system, you need to specify +`os_mount=local` in your job.yaml. + +BTW, the job's rootfs will be placed on a logical volume. + +## description + +### temporary rootfs data *Defaultly* + +If you only add `os_mount: local` in your job.yaml, the logical volume of rootfs +will be deleted and re-created in initrd stage before every job executes. + +The brief flow is as follow: + ``` + 1. boot and request scheduler for job. + 2. initrd stage: + - firstly, we need two logical volume: + ${src_lv} -- src logical volume: + - will never boot form this lv, just use it as a source data, and take a + snapshot of it to get the real boot logical volume. + - srv_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${timestamp} + ${boot_lv} -- boot logical volume: + - will boot from this. + - boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version} + - if ${src_lv} not exists: create it, and rsync the rootfs from cluster nfs server. + - if ${boot_lv} exists: delete it + - create ${boot_lv} as the snapshot of ${src_lv}. + - switch root to ${boot_lv}. + 3. boot the rootfs and execute the job. + ``` + +### persistent rootfs data + +When you need to persist the rootfs data of a job, and use it in the subsequent job(s), +two fields in `kernel_append_root` will help you: `save_root_partition`, `use_root_partition`. + +The brief flow is as follow: + ``` + 1. boot and request scheduler for job. + 2. initrd stage: + - firstly, we need two logical volume: + ${src_lv} -- src logical volume: + - if have ${use_root_partition}, src_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${use_root_partition} + - if no ${use_root_partition}, src_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${timestamp} + ${boot_lv} -- boot logical volume: + - if have ${save_root_partition}, boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version}_${save_root_partition} + - if no ${save_root_partition}, boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version} + - if ${src_lv} not exists: + - if have ${use_root_partition}, exit 1. + - if no ${use_root_partition}, create ${src_lv}, and rsync the rootfs from cluster nfs server. + - if ${boot_lv} != ${src_lv}: + - if ${boot_lv} exists: delete it + - create ${boot_lv} as the snapshot of ${src_lv}. + - switch root to ${boot_lv}. + 3. boot the rootfs and execute the job. + ``` + +Demo usage: + ``` + - in 20210218, you submit a job-20210218.yaml, and you want to persist the rootfs data of + job-20210218.yaml so that it can be used by the subsequent jobs. + Then you need add the follow field in your job-20210218.yaml: + kernel_append_root: save_root_partition=zhangsan_local_for_iperf_20210218 + + - in 20210219, you submit a job-20210219.yaml, and you want to use the rootfs data of + job-20210218.yaml. + Then you need add the follow field in your job-20210219.yaml: + kernel_append_root: use_root_partition=zhangsan_local_for_iperf_20210218 + ``` + +Notes: + - obviously, ${save_root_partition} and ${use_root_partition} must be the same, and unique. + - you must 'save_root_partition' firstly, then you can 'use_root_partition'. otherwise, your job will fail. + +## work flow + +1. user submit job + job fields: + ``` + os: openeuler + os_arch: aarch64 + os_version: 20.03 + os_mount: local + kernel_append_root: use_root_partition=zhangsan_local_for_iperf_20210218 save_root_partition=zhangsan_local_for_iperf_20210219 + ``` 2. scheduler return the ipxe_str to testbox - ``` - dhcp - initrd http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/{timestamp}/initrd.lkp - initrd http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/{timestamp}/boot/vmlinuz - imgargs vmlinux root=/dev/mapper/os-openeuler_aarch64_20.03 rootfs_src={nfs_server_ip}:os/openeuler/aarch64/20.03-iso-snapshots/{timestamp} initrd=initrd.lkp {kernel_append_root} - boot - ``` + ``` + dhcp + initrd http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/${timestamp}/initrd.lkp + kernel http://${http_server_ip}:${http_server_port}/os/openeuler/aarch64/20.03-iso-snapshots/${timestamp}/boot/vmlinuz root=/dev/mapper/os-openeuler_aarch64_20.03 rootfs_src=${nfs_server_ip}:os/openeuler/aarch64/20.03-iso-snapshots/${timestamp} initrd=initrd.lkp ${kernel_append_root} + boot + ``` 3. dracut step of boot - ``` - if have {use_root_partition}; then - base_lv_name=openeuler_aarch64_20.03_{value_of_use_root_partition} - lvdisplay /dev/mappaer/os-{base_lv_name} > /dev/null || exit 1 - - if have {save_root_partition}; then - boot_lv_name=openeuler_aarch64_20.03_{value_of_save_root_partition} - - if {save_root_partition} == {use_root_partition}; then - boot from /dev/mappaer/os-{boot_lv_name} - fi - - lvremove -f /dev/mappaer/os-{boot_lv_name} - lvcreate /dev/mappaer/os-{boot_lv_name} from /dev/mappaer/os-{base_lv_name} || exit 1 - boot from /dev/mappaer/os-{boot_lv_name} - else - boot_lv_name=openeuler_aarch64_20.03_{value_of_use_root_partition} - boot from /dev/mappaer/os-{boot_lv_name} - fi - - else: - base_lv_name=openeuler_aarch64_20.03_{timestamp} - lvdisplay /dev/mappaer/os-{base_lv_name} > /dev/null || { - # create logical volume - lvcreate -L {root_partition_size} -n {base_lv_name} os || exit 1 - - # rsync nfsroot to lvm:/dev/mapper/os-{base_lv_name} - mount -t nfs {nfs_server_ip}:os/openeuler/aarch64/20.03-iso-snapshots/{timestamp} /mnt - mkdir /mnt1 && mount /dev/mapper/os-{base_lv_name} /mnt1 - cp -a /mnt/. /mnt1/ - umount /mnt /mnt1 - - # change premission of lvm:/dev/mapper/os-{base_lv_name} to readonly - lvchange -p r /dev/mapper/os-{base_lv_name} - } - - boot_lv_name=openeuler_aarch64_20.03 - if have {save_root_partition}; then - boot_lv_name=openeuler_aarch64_20.03_{value_of_save_root_partition} - fi - - lvremove -f /dev/mappaer/os-{boot_lv_name} - lvcreate /dev/mappaer/os-{boot_lv_name} from /dev/mappaer/os-{base_lv_name} || exit 1 - boot from /dev/mappaer/os-{boot_lv_name} - fi - ``` + ``` + analysis_kernel_cmdline_params() + { + for i in $(cat /proc/cmdline) + do + [ "$i" =~ "rootfs_src=" ] && export ROOTFS_SRC=${i#rootfs_src=} + done + + [ -z "${ROOTFS_SRC}" ] && die "cannot find var in kernel cmdline params: rootfs_src" + + export TIMESTAMP=$(basename ${ROOTFS_SRC}) + } + + rsync_src_lv() + { + local src_lv=$1 + + lvdisplay ${src_lv} > /dev/null && return + + # create logical volume + lvcreate --size 10G --name $(basename ${src_lv}) os || exit + + # rsync nfsroot to ${src_lv} + mount -t nfs ${ROOTFS_SRC} /mnt + mkdir /mnt1 && mount ${src_lv} /mnt1 + cp -a /mnt/. /mnt1/ + umount /mnt /mnt1 + + # change permission of ${src_lv} to readonly + lvchange --permission r ${src_lv} + } + + snapshot_boot_lv() + { + local src_lv=$1 + local boot_lv=$2 + + [ "$src_lv" == "$boot_lv" ] && return + + lvremove --force ${boot_lv} + lvcreate --size 10G --name $(basename ${boot_lv}) --snapshot ${src_lv} || exit + } + + main() + { + analysis_kernel_cmdline_params + + if [ -z "${use_root_partition}" ]; then + src_lv="/dev/mapper/os-openeuler_aarch64_20.03_${TIMESTAMP}" + rsync_src_lv ${src_lv} + else + src_lv="/dev/mapper/os-openeuler_aarch64_20.03_${use_root_partition}" + lvdisplay ${src_lv} > /dev/null || exit + fi + + if [ -z "${save_root_partition}" ]; then + boot_lv="/dev/mapper/os-openeuler_aarch64_20.03" + else + boot_lv="/dev/mapper/os-openeuler_aarch64_20.03_${save_root_partition}" + fi + + snapshot_boot_lv ${src_lv} ${boot_lv} + boot_from_boot_lv + } + + main() + ``` 4. execute the job -- 2.23.0

2 2

[PATCH v4 lkp-tests] stats/ansible_test: stats result for ansible migrating
by Wang Chenglong 23 Feb '21

23 Feb '21

[How] ansible_stats_openeuler < ${job_output} Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 135 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 124 insertions(+), 11 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 4c7d07b3e..bd60a3ebc 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,15 +1,128 @@ #!/usr/bin/env ruby -while (line = STDIN.gets) - case line.chomp! - when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ - puts "ansible_test.total.nr_ok: #{$1}" - puts "ansible_test.total.nr_changed: #{$2}" - puts "ansible_test.total.nr_failed: #{$4}" - puts "ansible_test.total.nr_skipped: #{$5}" - puts "ansible_test.total.nr_rescued: #{$6}" - puts "ansible_test.total.nr_ignored: #{$7}" - when /playbook_run_on_fail/ - puts line +require 'json' + +def parse_msg_1(ansible_failed_json) + case ansible_failed_json.chomp + + # When running some plug-in, the file was not found + when /No file was found when using (.+)\./ + puts "files.error.no-file-was-found-when-using.#{$1}: 1" + puts "files.error.no-file-was-found-when-using.#{$1}.message: #{@ansible_failed_info}" + + # {"changed": false, "cmd": "csf -v", "msg": "[Errno 2] No such file or directory: b'csf': b'csf'", "rc": 2} + when /No such file or directory: b (.+) :/ + puts "error.no-such-file-or-directory.#{$1}: 1" + puts "error.no-such-file-or-directory.#{$1}.message: #{@ansible_failed_info}" + + # The task includes an option with an undefined variable + when /: (.+) is undefined/ + puts "syntax.error.an-undefined-variable.#{$1}: 1" + puts "syntax.error.an-undefined-variable.#{$1}.message: #{@ansible_failed_info}" + + # {"changed": true, "cmd": ["packaging/installer/install-required-packages.sh", "--non-interactive", "netdata"], + # "msg": "non-zero return code", "rc": 1} + when /non-zero return code/ + puts "error.non-zero-return-code.#{ansible_failed_json}: 1" + puts "error.non-zero-return-code.#{ansible_failed_json}.message: #{@ansible_failed_info}" + + # Some service could not find the requested host + when /Could not find the requested service (.+): host/ + puts "request.error.could-not-find-service-host.#{$1}: 1" + puts "request.error.could-not-find-service-host.#{$1}.messgae: #{@ansible_failed_info}" + + # Unable to start some services + when /Unable to start service (.+):/ + puts "service.error.unable-to-start-service.#{$1}: 1" + puts "service.error.unable-to-start-service.#{$1}.message: #{@ansible_failed_info}" + end +end + +def parse_msg_2(ansible_failed_json) + case ansible_failed_json.chomp + + # {"changed": false, "msg": "Failed to download packages: Cannot download 7.10.2/filebeat-7.10.2-aarch64.rpm: + # All mirrors were tried", "results": []} + when /Failed to download packages: Cannot download (.+):/ + puts "download.error.failed-to-download-package.#{$1}: 1" + puts "download.error.failed-to-download-package.#{$1}.message: #{@ansible_failed_info}" + + # {"changed": false, "msg": "Failure downloading https://rpms.remirepo.net/enterprise/remi-release-20.rpm, + # HTTP Error 404: Not Found"} + when /Failure downloading (.+), / + puts "download.error.Failure-downloading.[#{$1}]: 1" + puts "download.error.Failure-downloading.[#{$1}]: #{@ansible_failed_info}" + + # Failed to download the yum repodata + # {"changed": false, "msg": "Failed to download metadata for repo 'docker-ce-stable': Cannot download repomd.xml", + # "rc": 1, "results": []} + when /Failed to download metadata for repo (.+) :/ + puts "download.error.failed-to-download-metadata.#{$1}: 1" + puts "download.error.failed-to-download-metadata.#{$1}.message: #{@ansible_failed_info}" + + # Some role in not sipported this system + # {"changed": false, "msg": "Distribution openEuler is not supported by this role!"} + when /(.+) is not supported/ + puts "system.error.not-supported.#{$1.gsub(/ /, '-')}: 1" + puts "system.error.not-supported.message: #{@ansible_failed_info}" + + # failed to create temporary content file: The read operation timed out. + when /The read operation timed out/ + puts 'connection.error.read-timed-out: 1' + puts "connection.error.read-timed-out.message: #{@ansible_failed_info}" end end + +def parse_failures(ansible_failed_json) + ansible_failed_json.each do |items| + case items.chomp + # Failed to install some of the specified packages. + # {"changed": false, "failures": ["No package erlang available."], "msg": + # "Failed to install some of the specified packages", "rc": 1, "results": []} + when /No package (.+) available/ + puts "yum.error.no-package-available.#{$1}: 1" + puts "yum.error.no-package-available.#{$1}.message: #{@ansible_failed_info}" + end + end +end + +def parse_message(ansible_failed_json) + case ansible_failed_json.chomp + + # Could not find or access ansible yaml + # {"ansible_facts": {}, "ansible_included_var_files": [], "changed": false, "message": + # "Could not find or access 'openEuler.yml' Searched in: + # /root/.ansible/roles/ansible-unzip/vars/openEuler.yml /root/.ansible/roles/ansible-unzip/openEuler.yml i + # /root/.ansible/roles/ansible-unzip/tasks/vars/openEuler.yml /root/.ansible/roles/ansible-unzip/tasks/openEuler.yml + # /root/.ansible/vars/openEuler.yml /root/.ansible/openEuler.yml on the Ansible Controller. + # If you are using a module and expect the file to exist on the remote, see the remote_src option"} + when /Could not find or access (.+) / + puts "yaml.error.not-find-or-access-yaml.#{$1}: 1" + puts "yaml.error.not-find-or-access-yaml.#{$1}.message: #{@ansible_failed_info}" + end +end + +def common_error_id(line) + line = line.chomp + line.gsub(/'/, ' ') + line.gsub(/{%/, ' ') + line.gsub(/%}/, ' ') + line.gsub(/{},/, '[],') + line.gsub(/\\n\\t/, ' ') + line +end + +while (line = STDIN.gets) + line = common_error_id(line) + next unless line =~ /(FAILED!|failed:).*({.*})/ + + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? + + ansible_failed_json = JSON.load(@ansible_failed_info) + + parse_msg_1 ansible_failed_json['msg'] unless ansible_failed_json['msg'].nil? + parse_msg_2 ansible_failed_json['msg'] unless ansible_failed_json['msg'].nil? + parse_failures ansible_failed_json['failures'] unless ansible_failed_json['failures'].nil? + parse_message ansible_failed_json['message'] unless ansible_failed_json['message'].nil? +end -- 2.23.0

4 6