- Compass-ci - mailweb.openeuler.org

[PATCH v2 compass-ci 2/3] container/dracut-initrd: set sysroot in local mount situation
by Xu Xijian 11 Mar '21

11 Mar '21

When mount local filesystem in os starting, we will create or use a local existed logical volume to replace remote filesystem. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- .../dracut-initrd/bin/set-local-sysroot.sh | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 container/dracut-initrd/bin/set-local-sysroot.sh diff --git a/container/dracut-initrd/bin/set-local-sysroot.sh b/container/dracut-initrd/bin/set-local-sysroot.sh new file mode 100644 index 0000000..dc9000b --- /dev/null +++ b/container/dracut-initrd/bin/set-local-sysroot.sh @@ -0,0 +1,83 @@ +#!/bin/sh + +analyse_kernel_cmdline_params() { + rootfs="$(getarg root=)" + # example: $nfs_server_ip:/os/${os}/${os_arch}/${os_version}-snapshots/20210310005959 + rootfs_src=$(echo $"$rootfs" | sed 's/\///') + timestamp="$(basename $root)" + os="$(echo $root_src | awk -F '/|-' '{print $2}')" + os_arch="$(echo $root_src | awk -F '/|-' '{print $3}')" + os_version="$(echo $root_src | awk -F '/|-' '{print $4}')" + os_info="${os}_${os_arch}_${os_version}" + export rootfs_src timestamp os_info +} + +sync_src_lv() { + local src_lv="$1" + + [ -e "$src_lv" ] && return + + # create logical volume + src_lv_devname="$(basename $src_lv)" + lvm lvcreate -y -L 10G --name "${src_lv_devname#os-}" os + mke2fs -t ext4 -F "$src_lv" + + # sync nfsroot to $src_lv + mkdir -p /mnt1 && mount -t nfs "$rootfs_src" /mnt1 + mkdir -p /mnt2 && mount "$src_lv" /mnt2 + cp -a /mnt1/. /mnt2/ + umount /mnt1 /mnt2 + + # change permission of "$src_lv" to readonly + lvm lvchange --permission r "$src_lv" +} + +snapshot_boot_lv() { + local src_lv="$1" + local boot_lv="$2" + + [ "$src_lv" == "$boot_lv" ] && return + + lvm lvremove --force "$boot_lv" + boot_lv_devname="$(basename $boot_lv)" + lvm lvcreate --size 10G --name ${boot_lv_devname#os-} --snapshot "$src_lv" +} + +set_sysroot() { + boot_lv="$1" + umount "$NEWROOT" + mount "$boot_lv" "$NEWROOT" +} + +if ! getargbool 0 local; then + exit 0 +fi + +analyse_kernel_cmdline_params + +sed -i "s/^locking_type = .*/locking_type = 1/" /etc/lvm/lvm.conf + +use_root_partition="$(getarg use_root_partition=)" +if [ -n "$use_root_partition" ]; then + src_lv="$use_root_partition" # /dev/mapper/os-${os_info}_${timestamp} + sync_src_lv "$src_lv" +else + src_lv="/dev/mapper/os-${os_info}_$timestamp" + [ -e "src_lv" ] || { + echo "warn dracut: FATAL: no src_lv with local mount, reboot" + reboot + } +fi + +save_root_partition="$(getarg save_root_partition=)" +if [ -n "$save_root_partition" ]; then + boot_lv="$save_root_partition" +else + boot_lv="/dev/mapper/os-${os_info}" +fi + +snapshot_boot_lv "$src_lv" "$boot_lv" + +set_sysroot "$boot_lv" + +exit 0 -- 2.23.0

1 0

[PATCH v2 compass-ci 1/3] container/dracut-initrd: add handling of pre-mount stage into setup-dracut.sh
by Xu Xijian 11 Mar '21

11 Mar '21

Add the content of set-local-sysroot.sh into the tail of dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- container/dracut-initrd/bin/setup-dracut.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/container/dracut-initrd/bin/setup-dracut.sh b/container/dracut-initrd/bin/setup-dracut.sh index 849480d..45e895c 100755 --- a/container/dracut-initrd/bin/setup-dracut.sh +++ b/container/dracut-initrd/bin/setup-dracut.sh @@ -18,3 +18,6 @@ cp -a /usr/local/bin/cifs-lib.sh /usr/lib/dracut/modules.d/95cifs/ cat overlay-lkp.sh >> /usr/lib/dracut/modules.d/90overlay-root/overlay-mount.sh sed -i "/install() {/a\ inst /usr/bin/awk" /usr/lib/dracut/modules.d/40network/module-setup.sh sed -i "/install() {/a\ inst /sbin/mke2fs" /usr/lib/dracut/modules.d/98dracut-systemd/module-setup.sh +pre_mount_file="/usr/lib/dracut/modules.d/98dracut-systemd/dracut-pre-mount.sh" +[ "$(sed -n '$p' $pre_mount_file)" = "exit 0" ] && sed -i '$d' "$pre_mount_file" +cat set-local-sysroot.sh >> "$pre_mount_file" -- 2.23.0

1 0

[PATCH v2 compass-ci 3/3] lib/git_mirror.rb: github download workaround
by Li Yuanchao 10 Mar '21

10 Mar '21

There is a problem to download from github by http or https. fatal: unable to access 'https://github.com/liyc-github/AvxToNeon/': Failed to connect to github.com port 443: Connection timed out When it occurs, change url to "git://xxx". Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 8f23af3..50a6c75 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -39,19 +39,27 @@ class GitMirror return url end + def stderr_443?(stderr_info) + return true if stderr_info.include?('Failed to connect to github.com port 443') + + return false + end + def git_clone(url, mirror_dir) url = get_url(Array(url)[0]) 10.times do - %x(git clone --mirror --depth 1 #{url} #{mirror_dir} 2>&1) + stderr = %x(git clone --mirror --depth 1 #{url} #{mirror_dir} 2>&1) return true if File.directory?(mirror_dir) && File.exist?("#{mirror_dir}/config") + + url = "git://#{url.split('://')[1]}" if stderr_443?(stderr) end return false end def git_fetch(mirror_dir) if File.exist?("#{mirror_dir}/shallow") + FileUtils.rm("#{mirror_dir}/shallow.lock") if File.exist?("#{mirror_dir}/shallow.lock") %x(git -C #{mirror_dir} fetch --unshallow 2>&1) - return false end fetch_info = %x(git -C #{mirror_dir} fetch 2>&1) @@ -59,6 +67,11 @@ class GitMirror if fetch_info.include?(ERR_MESSAGE) && Dir.empty?(mirror_dir) FileUtils.rmdir(mirror_dir) end + + if stderr_443?(fetch_info) + url = %x(git -C #{mirror_dir} ls-remote --get-url origin).chomp + %x(git -C #{mirror_dir} remote set-url origin git://#{url.split('://')[1]}) + end return fetch_info.include? '->' end @@ -66,7 +79,7 @@ class GitMirror url = get_url(Array(url)[0]) git_url = %x(git -C #{mirror_dir} ls-remote --get-url origin).chomp - return true if url == git_url + return true if git_url == url || git_url.include?(url.split('://')[1]) return false end -- 2.23.0

1 0

[PATCH v2 compass-ci 2/3] sbin/auto_submit: change the path to do git command
by Li Yuanchao 10 Mar '21

10 Mar '21

As the path of repo mirrors under /srv/git changes, the path to execute git command should change too. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/sbin/auto_submit b/sbin/auto_submit index 857af29..faebb2f 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -44,14 +44,16 @@ class AutoSubmit submit_argv end - def submit_one_job(submit_argv, git_repo, branch, commit_id) - inactive_time = %x(git -C /srv/git/#{git_repo}.git log --pretty=format:"%cr" -1 #{commit_id}) + def submit_one_job(submit_argv, git_repo, belong, branch, commit_id) + mirror_dir = "/srv/git/#{belong}/#{git_repo}.git" + inactive_time = %x(git -C #{mirror_dir} log --pretty=format:"%cr" -1 #{commit_id}) return if inactive_time =~ /(month|year)/ real_argvs = Array.new(submit_argv) real_argvs.push("upstream_branch=#{branch}") real_argvs.push("upstream_commit=#{commit_id}") - tag = %x(git -C /srv/git/#{git_repo}.git tag --points-at #{commit_id}) + real_argvs.push("upstream_dir=#{belong}") + tag = %x(git -C #{mirror_dir} tag --points-at #{commit_id}) real_argvs.push("upstream_tag=#{tag}") unless tag.empty? system(real_argvs.join(' ')) @@ -60,16 +62,17 @@ class AutoSubmit def submit_specific_branch(submit_argv, newrefs_info, branches) branches.each do |branch_name| branch = "refs/heads/#{branch_name}" - next unless newrefs_info['new_refs']['heads'][branch] + commit_id = newrefs_info['new_refs']['heads'][branch] + next unless commit_id - submit_one_job(submit_argv, newrefs_info['git_repo'], branch_name, newrefs_info['new_refs']['heads'][branch]) + submit_one_job(submit_argv, newrefs_info['git_repo'], newrefs_info['belong'], branch_name, commit_id) end end - def submit_non_specific(non_specific_commits, git_repo, submit_argv) - non_specific_commits.each do |branch, commit_id| + def submit_non_specific(newrefs_info, submit_argv) + newrefs_info['new_refs']['heads'].each do |branch, commit_id| branch = branch.delete_prefix('refs/heads/') - submit_one_job(submit_argv, git_repo, branch, commit_id) + submit_one_job(submit_argv, newrefs_info['git_repo'], newrefs_info['belong'], branch, commit_id) end end @@ -96,7 +99,7 @@ class AutoSubmit if element['branches'] submit_specific_branch(argvs, newrefs_info, element['branches']) else - submit_non_specific(newrefs_info['new_refs']['heads'], newrefs_info['git_repo'], argvs) + submit_non_specific(newrefs_info, argvs) end end end -- 2.23.0

1 0

[PATCH v2 compass-ci 1/3] lib/git_mirror.rb: don't count new ref if it is not new
by Li Yuanchao 10 Mar '21

10 Mar '21

When repos are added, new_refs_count will increase. But many repos are not updated that day, that cause new_refs_count increase much more than real new refs number. Now only count those really new. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 53c9e59..8f23af3 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -194,7 +194,7 @@ class MirrorMain feedback_info[:new_refs] = new_refs send_message(feedback_info) - new_refs_log(git_repo, new_refs[:heads].length) + new_refs_log(git_repo, new_refs[:heads].length) if last_commit_new?(git_repo) end def do_push(fork_key) @@ -327,7 +327,7 @@ class MirrorMain def update_fork_stat(git_repo, possible_new_refs) update_stat_fetch(git_repo) - update_stat_new_refs(git_repo) if possible_new_refs + update_stat_new_refs(git_repo) if possible_new_refs && last_commit_new?(git_repo) es_repo_update(git_repo) end end @@ -521,4 +521,11 @@ class MirrorMain nr_new_branch: nr_new_branch }.to_json) end + + def last_commit_new?(git_repo) + inactive_time = %x(git -C /srv/git/#{@git_info[git_repo]['belong']}/#{git_repo}.git log --pretty=format:"%cr" -1) + return false if inactive_time =~ /(day|week|month|year)/ + + return true + end end -- 2.23.0

1 0

[PATCH v2 lkp-tests 11/11] install-iso-pre: add step: submit install-iso-sync-rootfs.yaml
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index 830c6bfd8a6a..ef4a0659fe02 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -236,7 +236,22 @@ submit_install_iso_firstboot_yaml() submit_install_iso_sync_rootfs_yaml() { - : + log_info "start submit install-iso-sync-rootfs.yaml" + + local sync_rootfs_yaml="${LKP_SRC}/jobs/install-iso-sync-rootfs.yaml" + [ -f "${sync_rootfs_yaml}" ] || die "cannot find ${sync_rootfs_yaml}" + + ${LKP_SRC}/sbin/submit \ + ${sync_rootfs_yaml} \ + suite=install-iso-sync-rootfs-${ROOTFS_SUFFIX} \ + iso_os=${iso_os} \ + iso_arch=${iso_arch} \ + iso_version=${iso_version} \ + NFS_SERVER_HOST=${NFS_SERVER_HOST} \ + NFS_ROOT_DIR=${NFS_ROOT_DIR} \ + ROOTFS_SUFFIX=${ROOTFS_SUFFIX} \ + VG_NAME=${VG_NAME} \ + testbox=$testbox || die "submit ${sync_rootfs_yaml} failed" } main() -- 2.23.0

1 0

[PATCH v2 lkp-tests 10/11] install-iso-pre: add step: submit install-iso-firstboot.yaml
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index cc939ba20ce8..830c6bfd8a6a 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -218,7 +218,20 @@ submit_install_iso_yaml() submit_install_iso_firstboot_yaml() { - : + log_info "start submit install-iso-firstboot.yaml" + + local firstboot_yaml="${LKP_SRC}/jobs/install-iso-firstboot.yaml" + [ -f "${firstboot_yaml}" ] || die "cannot find ${firstboot_yaml}" + + ${LKP_SRC}/sbin/submit \ + ${firstboot_yaml} \ + suite=install-iso-firstboot-${ROOTFS_SUFFIX} \ + os=${iso_os} \ + os_arch=${iso_arch} \ + os_version=${iso_version} \ + os_mount=local \ + kernel_custom_params="src_lv_suffix=${ROOTFS_SUFFIX} boot_lv_suffix=${ROOTFS_SUFFIX}" \ + testbox=$testbox || die "submit ${firstboot_yaml} failed" } submit_install_iso_sync_rootfs_yaml() -- 2.23.0

1 0

[PATCH v2 lkp-tests 09/11] install-iso-pre: add step: submit install-iso.yaml
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index 000c1fd6811b..cc939ba20ce8 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -185,9 +185,35 @@ prepare_submit() EOF } +prepare_install_iso_yaml() +{ + log_info "start prepare install-iso.yaml" + + local tftp_root="tftp://${TFTP_SERVER_HOST}/EFI/BOOT/${iso_os}/${iso_arch}/${iso_version}" + local custom_ipxe + + custom_ipxe="#!ipxe\\\\n" + custom_ipxe="${custom_ipxe}\\\\ndhcp" + custom_ipxe="${custom_ipxe}\\\\ninitrd ${tftp_root}/initrd.img" + custom_ipxe="${custom_ipxe}\\\\nkernel ${tftp_root}/vmlinux ro inst.geoloc=0 console=ttyS0 smmu.bypassdev=0x1000:0x17 smmu.bypassde1000:0x15 viff video=VGA-1:640x480-32@60me initrd=initrd.img ip=dhcp inst.ks=http://${HTTP_SERVER_HOST}:${HTTP_SERVER_PORT}/os/kickstart/ks/${iso_os}_${iso_arch}_${iso_version}.cfg" + custom_ipxe="${custom_ipxe}\\\\nboot\\\\n" + + log_info "custom_ipxe: $custom_ipxe" + sed -i "/custom_ipxe:/ccustom_ipxe: \"${custom_ipxe}\"" ${install_iso_yaml} +} + submit_install_iso_yaml() { - : + log_info "start submit install-iso.yaml" + + local install_iso_yaml="${LKP_SRC}/jobs/install-iso.yaml" + [ -f "${install_iso_yaml}" ] || die "cannot find ${install_iso_yaml}" + + prepare_install_iso_yaml + + ${LKP_SRC}/sbin/submit \ + ${install_iso_yaml} \ + testbox=$testbox || die "submit ${install_iso_yaml} failed" } submit_install_iso_firstboot_yaml() -- 2.23.0

1 0

[PATCH v2 lkp-tests 08/11] install-iso-pre: add step: prepare_submit
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- distro/depends/install-iso-pre | 2 ++ tests/install-iso-pre | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/distro/depends/install-iso-pre b/distro/depends/install-iso-pre index 6f88eb919896..4569e1e3cdc0 100644 --- a/distro/depends/install-iso-pre +++ b/distro/depends/install-iso-pre @@ -1 +1,3 @@ lvm2 +ruby +git diff --git a/tests/install-iso-pre b/tests/install-iso-pre index 7b9e6fd2107c..000c1fd6811b 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -166,7 +166,23 @@ prepare_ks() prepare_submit() { - : + ${LKP_SRC}/sbin/install-dependencies.sh + + export CCI_REPOS="/c" + + mkdir -p $CCI_REPOS && cd $CCI_REPOS + git clone $LAB_Z9_URL || die "clone lab-z9 git repo failed: $LAB_Z9_URL" + + local config_yaml="/etc/compass-ci/defaults/sparrow.yaml" + mkdir -p $(dirname $config_yaml) + cat >> $config_yaml <<-EOF + SCHED_HOST: ${SCHED_HOST} + SCHED_PORT: ${SCHED_PORT} + my_name: ${my_name} + my_email: ${my_email} + my_token: ${MY_TOKEN} + lab: z9 + EOF } submit_install_iso_yaml() -- 2.23.0

1 0

[PATCH v2 lkp-tests 07/11] install-iso-pre: add step: prepare_ks
by Yu Chuan 10 Mar '21

10 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 72 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index d800704f648c..7b9e6fd2107c 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -89,9 +89,79 @@ prepare_lv() create_lv } +generate_ks() +{ + log_info "start generate $ks_file" + + local nic=$(ip route get ${LKP_SERVER} | awk '{print $3; exit}') + + cat > ${ks_file} <<-EOF + # Use network installation + url --url="http://${HTTP_SERVER_HOST}:${HTTP_SERVER_PORT}/${HTTP_ROOT_DIR}/${iso_os}/${iso_arch}/${iso_version}/" + + # Network information + network --bootproto=dhcp --device=${nic} --ipv6=auto --activate + network --hostname=localhost.localdomain + + # Specifies a list of disks for the installation program to use. All other disks are ignored. + ignoredisk --only-use=${rootfs_disk#/dev/} + + # if installation target is a logical volume, bootloader should be none. + bootloader --location=none + logvol / --vgname=${VG_NAME} --name=${LV_NAME} --useexisting + + # Root password + rootpw --iscrypted ${ROOTFS_ROOT_PASSWD} + + # Reboot after installation + reboot --eject + + # Use text mode install + text + + # Keyboard layouts + keyboard --vckeymap=us --xlayouts='us' + + # System language + lang en_US.UTF-8 + + # Do not configure the X Window System + skipx + + # System services + services --enabled="chronyd" + + # System timezone + timezone Asia/Shanghai --isUtc + + %packages + @^minimal-environment + + %end + + %anaconda + pwpolicy root --minlen=8 --minquality=1 --notstrict --nochanges --notempty + pwpolicy user --minlen=8 --minquality=1 --notstrict --nochanges --emptyok + pwpolicy luks --minlen=8 --minquality=1 --notstrict --nochanges --notempty + %end + EOF +} + +upload_ks() +{ + log_info "start upload ks to rsync-server: ${RSYNC_SERVER_HOST}:${RSYNC_SERVER_PORT}" + + rsync -vrtopg --port ${RSYNC_SERVER_PORT} ${ks_file} ${RSYNC_SERVER_HOST}::ks/ || die "upload ks to rsync-server failed, please check." +} + prepare_ks() { - : + log_info "start prepare kickstart file" + + local ks_file=/tmp/${iso_os}_${iso_arch}_${iso_version}.ks + + generate_ks + upload_ks } prepare_submit() -- 2.23.0

1 0