- Compass-ci - mailweb.openeuler.org

[PATCH v2 compass-ci 2/3] sbin/auto_submit: change the path to do git command
by Li Yuanchao 10 Mar '21

10 Mar '21

As the path of repo mirrors under /srv/git changes, the path to execute git command should change too. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/sbin/auto_submit b/sbin/auto_submit index 857af29..faebb2f 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -44,14 +44,16 @@ class AutoSubmit submit_argv end - def submit_one_job(submit_argv, git_repo, branch, commit_id) - inactive_time = %x(git -C /srv/git/#{git_repo}.git log --pretty=format:"%cr" -1 #{commit_id}) + def submit_one_job(submit_argv, git_repo, belong, branch, commit_id) + mirror_dir = "/srv/git/#{belong}/#{git_repo}.git" + inactive_time = %x(git -C #{mirror_dir} log --pretty=format:"%cr" -1 #{commit_id}) return if inactive_time =~ /(month|year)/ real_argvs = Array.new(submit_argv) real_argvs.push("upstream_branch=#{branch}") real_argvs.push("upstream_commit=#{commit_id}") - tag = %x(git -C /srv/git/#{git_repo}.git tag --points-at #{commit_id}) + real_argvs.push("upstream_dir=#{belong}") + tag = %x(git -C #{mirror_dir} tag --points-at #{commit_id}) real_argvs.push("upstream_tag=#{tag}") unless tag.empty? system(real_argvs.join(' ')) @@ -60,16 +62,17 @@ class AutoSubmit def submit_specific_branch(submit_argv, newrefs_info, branches) branches.each do |branch_name| branch = "refs/heads/#{branch_name}" - next unless newrefs_info['new_refs']['heads'][branch] + commit_id = newrefs_info['new_refs']['heads'][branch] + next unless commit_id - submit_one_job(submit_argv, newrefs_info['git_repo'], branch_name, newrefs_info['new_refs']['heads'][branch]) + submit_one_job(submit_argv, newrefs_info['git_repo'], newrefs_info['belong'], branch_name, commit_id) end end - def submit_non_specific(non_specific_commits, git_repo, submit_argv) - non_specific_commits.each do |branch, commit_id| + def submit_non_specific(newrefs_info, submit_argv) + newrefs_info['new_refs']['heads'].each do |branch, commit_id| branch = branch.delete_prefix('refs/heads/') - submit_one_job(submit_argv, git_repo, branch, commit_id) + submit_one_job(submit_argv, newrefs_info['git_repo'], newrefs_info['belong'], branch, commit_id) end end @@ -96,7 +99,7 @@ class AutoSubmit if element['branches'] submit_specific_branch(argvs, newrefs_info, element['branches']) else - submit_non_specific(newrefs_info['new_refs']['heads'], newrefs_info['git_repo'], argvs) + submit_non_specific(newrefs_info, argvs) end end end -- 2.23.0

1 0

[PATCH v2 compass-ci 1/3] lib/git_mirror.rb: don't count new ref if it is not new
by Li Yuanchao 10 Mar '21

10 Mar '21

When repos are added, new_refs_count will increase. But many repos are not updated that day, that cause new_refs_count increase much more than real new refs number. Now only count those really new. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 53c9e59..8f23af3 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -194,7 +194,7 @@ class MirrorMain feedback_info[:new_refs] = new_refs send_message(feedback_info) - new_refs_log(git_repo, new_refs[:heads].length) + new_refs_log(git_repo, new_refs[:heads].length) if last_commit_new?(git_repo) end def do_push(fork_key) @@ -327,7 +327,7 @@ class MirrorMain def update_fork_stat(git_repo, possible_new_refs) update_stat_fetch(git_repo) - update_stat_new_refs(git_repo) if possible_new_refs + update_stat_new_refs(git_repo) if possible_new_refs && last_commit_new?(git_repo) es_repo_update(git_repo) end end @@ -521,4 +521,11 @@ class MirrorMain nr_new_branch: nr_new_branch }.to_json) end + + def last_commit_new?(git_repo) + inactive_time = %x(git -C /srv/git/#{@git_info[git_repo]['belong']}/#{git_repo}.git log --pretty=format:"%cr" -1) + return false if inactive_time =~ /(day|week|month|year)/ + + return true + end end -- 2.23.0

1 0

[PATCH v2 lkp-tests 11/11] install-iso-pre: add step: submit install-iso-sync-rootfs.yaml
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index 830c6bfd8a6a..ef4a0659fe02 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -236,7 +236,22 @@ submit_install_iso_firstboot_yaml() submit_install_iso_sync_rootfs_yaml() { - : + log_info "start submit install-iso-sync-rootfs.yaml" + + local sync_rootfs_yaml="${LKP_SRC}/jobs/install-iso-sync-rootfs.yaml" + [ -f "${sync_rootfs_yaml}" ] || die "cannot find ${sync_rootfs_yaml}" + + ${LKP_SRC}/sbin/submit \ + ${sync_rootfs_yaml} \ + suite=install-iso-sync-rootfs-${ROOTFS_SUFFIX} \ + iso_os=${iso_os} \ + iso_arch=${iso_arch} \ + iso_version=${iso_version} \ + NFS_SERVER_HOST=${NFS_SERVER_HOST} \ + NFS_ROOT_DIR=${NFS_ROOT_DIR} \ + ROOTFS_SUFFIX=${ROOTFS_SUFFIX} \ + VG_NAME=${VG_NAME} \ + testbox=$testbox || die "submit ${sync_rootfs_yaml} failed" } main() -- 2.23.0

1 0

[PATCH v2 lkp-tests 10/11] install-iso-pre: add step: submit install-iso-firstboot.yaml
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index cc939ba20ce8..830c6bfd8a6a 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -218,7 +218,20 @@ submit_install_iso_yaml() submit_install_iso_firstboot_yaml() { - : + log_info "start submit install-iso-firstboot.yaml" + + local firstboot_yaml="${LKP_SRC}/jobs/install-iso-firstboot.yaml" + [ -f "${firstboot_yaml}" ] || die "cannot find ${firstboot_yaml}" + + ${LKP_SRC}/sbin/submit \ + ${firstboot_yaml} \ + suite=install-iso-firstboot-${ROOTFS_SUFFIX} \ + os=${iso_os} \ + os_arch=${iso_arch} \ + os_version=${iso_version} \ + os_mount=local \ + kernel_custom_params="src_lv_suffix=${ROOTFS_SUFFIX} boot_lv_suffix=${ROOTFS_SUFFIX}" \ + testbox=$testbox || die "submit ${firstboot_yaml} failed" } submit_install_iso_sync_rootfs_yaml() -- 2.23.0

1 0

[PATCH v2 lkp-tests 09/11] install-iso-pre: add step: submit install-iso.yaml
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index 000c1fd6811b..cc939ba20ce8 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -185,9 +185,35 @@ prepare_submit() EOF } +prepare_install_iso_yaml() +{ + log_info "start prepare install-iso.yaml" + + local tftp_root="tftp://${TFTP_SERVER_HOST}/EFI/BOOT/${iso_os}/${iso_arch}/${iso_version}" + local custom_ipxe + + custom_ipxe="#!ipxe\\\\n" + custom_ipxe="${custom_ipxe}\\\\ndhcp" + custom_ipxe="${custom_ipxe}\\\\ninitrd ${tftp_root}/initrd.img" + custom_ipxe="${custom_ipxe}\\\\nkernel ${tftp_root}/vmlinux ro inst.geoloc=0 console=ttyS0 smmu.bypassdev=0x1000:0x17 smmu.bypassde1000:0x15 viff video=VGA-1:640x480-32@60me initrd=initrd.img ip=dhcp inst.ks=http://${HTTP_SERVER_HOST}:${HTTP_SERVER_PORT}/os/kickstart/ks/${iso_os}_${iso_arch}_${iso_version}.cfg" + custom_ipxe="${custom_ipxe}\\\\nboot\\\\n" + + log_info "custom_ipxe: $custom_ipxe" + sed -i "/custom_ipxe:/ccustom_ipxe: \"${custom_ipxe}\"" ${install_iso_yaml} +} + submit_install_iso_yaml() { - : + log_info "start submit install-iso.yaml" + + local install_iso_yaml="${LKP_SRC}/jobs/install-iso.yaml" + [ -f "${install_iso_yaml}" ] || die "cannot find ${install_iso_yaml}" + + prepare_install_iso_yaml + + ${LKP_SRC}/sbin/submit \ + ${install_iso_yaml} \ + testbox=$testbox || die "submit ${install_iso_yaml} failed" } submit_install_iso_firstboot_yaml() -- 2.23.0

1 0

[PATCH v2 lkp-tests 08/11] install-iso-pre: add step: prepare_submit
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- distro/depends/install-iso-pre | 2 ++ tests/install-iso-pre | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/distro/depends/install-iso-pre b/distro/depends/install-iso-pre index 6f88eb919896..4569e1e3cdc0 100644 --- a/distro/depends/install-iso-pre +++ b/distro/depends/install-iso-pre @@ -1 +1,3 @@ lvm2 +ruby +git diff --git a/tests/install-iso-pre b/tests/install-iso-pre index 7b9e6fd2107c..000c1fd6811b 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -166,7 +166,23 @@ prepare_ks() prepare_submit() { - : + ${LKP_SRC}/sbin/install-dependencies.sh + + export CCI_REPOS="/c" + + mkdir -p $CCI_REPOS && cd $CCI_REPOS + git clone $LAB_Z9_URL || die "clone lab-z9 git repo failed: $LAB_Z9_URL" + + local config_yaml="/etc/compass-ci/defaults/sparrow.yaml" + mkdir -p $(dirname $config_yaml) + cat >> $config_yaml <<-EOF + SCHED_HOST: ${SCHED_HOST} + SCHED_PORT: ${SCHED_PORT} + my_name: ${my_name} + my_email: ${my_email} + my_token: ${MY_TOKEN} + lab: z9 + EOF } submit_install_iso_yaml() -- 2.23.0

1 0

[PATCH v2 lkp-tests 07/11] install-iso-pre: add step: prepare_ks
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 72 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index d800704f648c..7b9e6fd2107c 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -89,9 +89,79 @@ prepare_lv() create_lv } +generate_ks() +{ + log_info "start generate $ks_file" + + local nic=$(ip route get ${LKP_SERVER} | awk '{print $3; exit}') + + cat > ${ks_file} <<-EOF + # Use network installation + url --url="http://${HTTP_SERVER_HOST}:${HTTP_SERVER_PORT}/${HTTP_ROOT_DIR}/${iso_os}/${iso_arch}/${iso_version}/" + + # Network information + network --bootproto=dhcp --device=${nic} --ipv6=auto --activate + network --hostname=localhost.localdomain + + # Specifies a list of disks for the installation program to use. All other disks are ignored. + ignoredisk --only-use=${rootfs_disk#/dev/} + + # if installation target is a logical volume, bootloader should be none. + bootloader --location=none + logvol / --vgname=${VG_NAME} --name=${LV_NAME} --useexisting + + # Root password + rootpw --iscrypted ${ROOTFS_ROOT_PASSWD} + + # Reboot after installation + reboot --eject + + # Use text mode install + text + + # Keyboard layouts + keyboard --vckeymap=us --xlayouts='us' + + # System language + lang en_US.UTF-8 + + # Do not configure the X Window System + skipx + + # System services + services --enabled="chronyd" + + # System timezone + timezone Asia/Shanghai --isUtc + + %packages + @^minimal-environment + + %end + + %anaconda + pwpolicy root --minlen=8 --minquality=1 --notstrict --nochanges --notempty + pwpolicy user --minlen=8 --minquality=1 --notstrict --nochanges --emptyok + pwpolicy luks --minlen=8 --minquality=1 --notstrict --nochanges --notempty + %end + EOF +} + +upload_ks() +{ + log_info "start upload ks to rsync-server: ${RSYNC_SERVER_HOST}:${RSYNC_SERVER_PORT}" + + rsync -vrtopg --port ${RSYNC_SERVER_PORT} ${ks_file} ${RSYNC_SERVER_HOST}::ks/ || die "upload ks to rsync-server failed, please check." +} + prepare_ks() { - : + log_info "start prepare kickstart file" + + local ks_file=/tmp/${iso_os}_${iso_arch}_${iso_version}.ks + + generate_ks + upload_ks } prepare_submit() -- 2.23.0

1 0

[PATCH v2 lkp-tests 06/11] install-iso-pre: add step: prepare_lv
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- distro/depends/install-iso-pre | 1 + tests/install-iso-pre | 46 +++++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 distro/depends/install-iso-pre diff --git a/distro/depends/install-iso-pre b/distro/depends/install-iso-pre new file mode 100644 index 000000000000..6f88eb919896 --- /dev/null +++ b/distro/depends/install-iso-pre @@ -0,0 +1 @@ +lvm2 diff --git a/tests/install-iso-pre b/tests/install-iso-pre index 4817f6d8eb37..d800704f648c 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -40,9 +40,53 @@ check_params() done } +create_vg() +{ + log_info "start create vg:$VG_NAME" + + local disk + + for disk in "${rootfs_disk[@]}" + do + log_info "start handle disk: $disk" + pvdisplay ${disk} > /dev/null || pvcreate ${disk} || die "create pv failed: $disk" + + if vgdisplay ${VG_NAME} > /dev/null; then + vgextend ${VG_NAME} ${disk} + else + vgcreate ${VG_NAME} ${disk} + fi + done + +} + +create_lv() +{ + log_info "start create lv:$LV_NAME" + + lvcreate -y -L 10G -n ${LV_NAME} ${VG_NAME} || die "create logical volume failed." +} + prepare_lv() { - : + log_info "start prepare logical volume" + + # activate lvm + vgchange -ay + + # VG_NAME defines in job.yaml + export VG_NAME + export LV_NAME="${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX}" + + # lv existed + lvdisplay /dev/mapper/${VG_NAME}-${LV_NAME} > /dev/null && { + log_info "iso already install to this testbox, so rootfs maybe already generated ok. please check it." + exit 0 + } + + # lv not existed + vgdisplay ${VG_NAME} > /dev/null || create_vg + create_lv } prepare_ks() -- 2.23.0

1 0

[PATCH v2 lkp-tests 05/11] feat: step 4 of iso2rootfs: add entry job
by Yu Chuan 10 Mar '21

10 Mar '21

Framework of install-iso-pre. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/install-iso-pre.yaml | 52 +++++++++++++++++++++++ tests/install-iso-pre | 86 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 jobs/install-iso-pre.yaml create mode 100755 tests/install-iso-pre diff --git a/jobs/install-iso-pre.yaml b/jobs/install-iso-pre.yaml new file mode 100644 index 000000000000..a9dbe46b4bf3 --- /dev/null +++ b/jobs/install-iso-pre.yaml @@ -0,0 +1,52 @@ +suite: install-iso-pre + +install-iso-pre: + ################# + # iso info + ################# + iso_os: + iso_arch: + iso_version: + + ################# + # http server info, which stored the installation source + ################# + HTTP_SERVER_HOST: + HTTP_SERVER_PORT: + HTTP_ROOT_DIR: + + ################# + # rsync-server info, used for upload ks + ################# + RSYNC_SERVER_HOST: + RSYNC_SERVER_PORT: + + ################# + # tftp server info, used for kickstart auto install + ################# + TFTP_SERVER_HOST: + + ################# + # nfs server info, used for store rootfs + ################# + NFS_SERVER_HOST: + NFS_ROOT_DIR: + + ################# + # result rootfs suffix, format: $(date "+%Y%m%d%H%M%S") + ################# + ROOTFS_SUFFIX: + + ################# + # name of volume group which stores the logical volume of rootfs + ################# + VG_NAME: + + ################# + # lab_z9 git repo clone url, used for find host file when submit + ################# + LAB_Z9_URL: + +secrets: + ROOTFS_ROOT_PASSWD: + MY_TOKEN: diff --git a/tests/install-iso-pre b/tests/install-iso-pre new file mode 100755 index 000000000000..4817f6d8eb37 --- /dev/null +++ b/tests/install-iso-pre @@ -0,0 +1,86 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +set -e + +. $LKP_SRC/lib/debug.sh +. $LKP_SRC/lib/log.sh + +check_params() +{ + log_info "start check params" + + local required_vars=( + "iso_os" + "iso_arch" + "iso_version" + "ROOTFS_SUFFIX" + "VG_NAME" + + "HTTP_SERVER_HOST" + "HTTP_SERVER_PORT" + "HTTP_ROOT_DIR" + "ROOTFS_ROOT_PASSWD" + "rootfs_disk" + + "RSYNC_SERVER_HOST" + "RSYNC_SERVER_PORT" + "TFTP_SERVER_HOST" + "NFS_SERVER_HOST" + "NFS_ROOT_DIR" + + "LAB_Z9_URL" + "MY_TOKEN" + ) + + for i in "${required_vars[@]}" + do + [ -n "$(eval echo \$$i)" ] || die "cannot get value of var: $i" + done +} + +prepare_lv() +{ + : +} + +prepare_ks() +{ + : +} + +prepare_submit() +{ + : +} + +submit_install_iso_yaml() +{ + : +} + +submit_install_iso_firstboot_yaml() +{ + : +} + +submit_install_iso_sync_rootfs_yaml() +{ + : +} + +main() +{ + check_params + + prepare_lv + prepare_ks + prepare_submit + + submit_install_iso_yaml + submit_install_iso_firstboot_yaml + submit_install_iso_sync_rootfs_yaml +} + +main -- 2.23.0

1 0

[PATCH v2 lkp-tests 04/11] install-iso-sync-rootfs: add step: sync_rootfs
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- distro/depends/install-iso-sync-rootfs | 2 + tests/install-iso-sync-rootfs | 53 +++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 distro/depends/install-iso-sync-rootfs diff --git a/distro/depends/install-iso-sync-rootfs b/distro/depends/install-iso-sync-rootfs new file mode 100644 index 000000000000..2abb6b833291 --- /dev/null +++ b/distro/depends/install-iso-sync-rootfs @@ -0,0 +1,2 @@ +nfs-common +lvm2 diff --git a/tests/install-iso-sync-rootfs b/tests/install-iso-sync-rootfs index e5de4d72240e..34096229ccfb 100755 --- a/tests/install-iso-sync-rootfs +++ b/tests/install-iso-sync-rootfs @@ -7,6 +7,20 @@ set -e . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/log.sh +SKIP_SYNC_DIRS=( + "srv" + "opt" + "mnt" + "media" + "home" + "lost+found" + "dev" + "proc" + "run" + "sys" + "var/log" +) + check_params() { log_info "start check params" @@ -28,9 +42,46 @@ check_params() done } +prepare_exclude_file() +{ + local exclude_file=$1 + + rm -rf $exclude_file + for i in "${SKIP_SYNC_DIRS[@]}" + do + echo $i >> $exclude_file + done +} + sync_rootfs_to_nfs_server() { - : + # mount nfs rootfs + local nfs_rootfs_mount_point=/tmp/nfs_rootfs_${ROOTFS_SUFFIX} + + mkdir -p ${nfs_rootfs_mount_point} + mount -t nfs ${NFS_SERVER_HOST}:${NFS_ROOT_DIR}/${os}/${os_arch}/${os_version}-iso-${ROOTFS_SUFFIX} ${nfs_rootfs_mount_point} || die "mount nfs rootfs failed" + + # activate the lvm + vgchange -ay + + # mount local logical volume + local lv_rootfs_mount_point=/tmp/lv_rootfs_${ROOTFS_SUFFIX} + + mkdir -p ${lv_rootfs_mount_point} + mount /dev/mapper/${VG_NAME}-${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX} ${lv_rootfs_mount_point} || die "mount lv rootfs failed" + + # sync rootfs + local exclude_file=/tmp/exclude_file_${ROOTFS_SUFFIX} + + prepare_exclude_file "$exclude_file" + rsync -az ${lv_rootfs_mount_point}/. ${nfs_rootfs_mount_point} --exclude-from=$exclude_file || die "copy out rootfs failed" + cd ${nfs_rootfs_mount_point} && mkdir -p "${SKIP_SYNC_DIRS[@]}" + + # umount nfs rootfs + umount $nfs_rootfs_mount_point || log_warn "umount failed: nfs_rootfs_mount_point" + + # umount local logical volume + umount $lv_rootfs_mount_point || log_warn "umount failed: $lv_rootfs_mount_point" } main() -- 2.23.0

1 0