- Compass-ci - mailweb.openeuler.org

[PATCH v2 compass-ci] lib/rpm-repo.rb: do not move to pub directory until test completed
by Li Ping 20 Apr '21

20 Apr '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/rpm-repo.rb | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/rpm-repo.rb b/lib/rpm-repo.rb index 658a5270..c8d6b087 100755 --- a/lib/rpm-repo.rb +++ b/lib/rpm-repo.rb @@ -6,6 +6,7 @@ require 'fileutils' require 'json' require './mq_client' +require_relative 'json_logger.rb' MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || '172.17.0.1' MQ_PORT = ENV['MQ_PORT'] || 5672 @@ -29,31 +30,47 @@ MQ_PORT = ENV['MQ_PORT'] || 5672 # change /srv/rpm/pub/**/repodate # @mq.ack(info) class HandleRepo + @@upload_dir_prefix = "/srv/rpm/upload/" def initialize @mq = MQClient.new(MQ_HOST, MQ_PORT) - @update = [] + @log = JSONLogger.new end def handle_new_rpm queue = @mq.queue("update_repo") queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| - rpm_info = JSON.parse(msg) - rpm_info["upload_rpms"].each do |rpm| - rpm_path = File.dirname(rpm).sub("upload", "testing") - FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) + begin + rpm_info = JSON.parse(msg) + check_upload_rpms(rpm_info) + rpm_info["upload_rpms"].each do |rpm| + rpm_path = File.dirname(rpm).sub("upload", "testing") + FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) - dest = File.join(rpm_path.to_s, File.basename(rpm)) - @update << dest - FileUtils.mv(rpm, dest) - system("createrepo --update $(dirname #{rpm_path})") + dest = File.join(rpm_path.to_s, File.basename(rpm)) + FileUtils.mv(rpm, dest) + system("createrepo --update $(dirname #{rpm_path})") + end + @mq.ack(info) + rescue StandardError => e + @log.warn({ + "error message": e.message + }.to_json) + @mq.ack(info) end - update_pub_dir - @mq.ack(info) end end - def update_pub_dir - @update.each do |rpm| + def check_upload_rpms(data) + raise JSON.dump({ "errcode" => "200", "errmsg" => "no upload_rpms params" }) unless data.key?("upload_rpms") + raise JSON.dump({ "errcode" => "200", "errmsg" => "upload_rpms params type error" }) if data["upload_rpms"].class != Array + data["upload_rpms"].each do |rpm| + raise JSON.dump({ "errcode" => "200", "errmsg" => "#{rpm} not exist" }) unless File.exist?(rpm) + raise JSON.dump({ "errcode" => "200", "errmsg" => "the upload directory is incorrect" }) unless File.dirname(rpm).start_with?(@@upload_dir_prefix) + end + end + + def update_pub_dir(update) + update.each do |rpm| pub_path = File.dirname(rpm).sub("testing", "pub") FileUtils.mkdir_p(pub_path) unless File.directory?(pub_path) -- 2.23.0

1 0

[PATCH compass-ci 3/3] service/kibana-logging: configuring ES authentication information
by Wu Zhende 20 Apr '21

20 Apr '21

logging-es set up the certification. kibana-logging needs to configure the connection information to connect to the logging-es. When logging to the kibana web page, need to enter the account and password to enhance security. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana-logging/Dockerfile | 7 +++++-- container/kibana-logging/build | 10 +++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/container/kibana-logging/Dockerfile b/container/kibana-logging/Dockerfile index 970eb5a..7085618 100644 --- a/container/kibana-logging/Dockerfile +++ b/container/kibana-logging/Dockerfile @@ -5,8 +5,11 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} -# docker image borrowed from hub.docker.com/r/gagara/kibana-oss-arm64 +ARG USER +ARG PASSWORD MAINTAINER Wu Zhende <wuzhende666(a)163.com> -RUN sed -i 's/server.host: "0"/server.host: "0.0.0.0"/' config/kibana.yml +RUN sed -i 's/server.host: "0"/server.host: "0.0.0.0"/' config/kibana.yml && \ + sed -i "6i\elasticsearch.username: ${USER}" config/kibana.yml && \ + sed -i "6i\elasticsearch.password: ${PASSWORD}" config/kibana.yml diff --git a/container/kibana-logging/build b/container/kibana-logging/build index 9c75aeb..238a88d 100755 --- a/container/kibana-logging/build +++ b/container/kibana-logging/build @@ -3,8 +3,16 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig' +names = Set.new %w[ + LOGGING_ES_USER + LOGGING_ES_PASSWORD +] + +defaults = relevant_service_account(names) + docker_skip_rebuild "kibana:7.11.1" BASE_IMAGE_DICT = { @@ -13,4 +21,4 @@ BASE_IMAGE_DICT = { BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] -system "docker build -t kibana:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} ." +system "docker build -t kibana:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg USER=#{defaults['LOGGING_ES_USER']} --build-arg PASSWORD=#{defaults['LOGGING_ES_PASSWORD']} ." -- 2.23.0

1 0

[PATCH compass-ci 2/3] service/master-fluentd: configuring ES authentication information
by Wu Zhende 20 Apr '21

20 Apr '21

logging-es set up the certification. fluentd needs to configure the connection information to connect to the logging-es. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/master-fluentd/docker-fluentd.conf | 2 ++ container/master-fluentd/start | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/container/master-fluentd/docker-fluentd.conf b/container/master-fluentd/docker-fluentd.conf index 410f2eb..73d7103 100644 --- a/container/master-fluentd/docker-fluentd.conf +++ b/container/master-fluentd/docker-fluentd.conf @@ -42,6 +42,8 @@ @type elasticsearch host "#{ENV['LOGGING_ES_HOST']}" port "#{ENV['LOGGING_ES_PORT']}" + user "#{ENV['LOGGING_ES_USER']}" + password "#{ENV['LOGGING_ES_PASSWORD']}" suppress_type_name true flush_interval 1s num_threads 10 diff --git a/container/master-fluentd/start b/container/master-fluentd/start index a05881c..27aece9 100755 --- a/container/master-fluentd/start +++ b/container/master-fluentd/start @@ -9,11 +9,17 @@ require_relative '../defconfig.rb' names = Set.new %w[ LOGGING_ES_HOST LOGGING_ES_PORT + LOGGING_ES_USER + LOGGING_ES_PASSWORD ] +account_info = relevant_service_account(names) + defaults = relevant_defaults(names) defaults['LOGGING_ES_HOST'] ||= '172.17.0.1' defaults['LOGGING_ES_PORT'] ||= '9202' +defaults.merge!(account_info) + env = docker_env(defaults) docker_rm 'master-fluentd' -- 2.23.0

1 0

[PATCH lab-z9] add new info for a125
by Bai Jing 20 Apr '21

20 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- hosts/taishan200-2280-2s64p-256g--a125 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/hosts/taishan200-2280-2s64p-256g--a125 b/hosts/taishan200-2280-2s64p-256g--a125 index 5c5862e..f2cb708 100644 --- a/hosts/taishan200-2280-2s64p-256g--a125 +++ b/hosts/taishan200-2280-2s64p-256g--a125 @@ -2,11 +2,7 @@ memory: 256G nr_hdd_partitions: 2 hdd_partitions: - /dev/disk/by-id/scsi-35000039988707ca1 - - scsi-35000039988707ca10 - - scsi-35000039988707ca1-part* - - /dev/disk/by-id/scsi-35000039988707d25-part1 - - /dev/disk/by-id/scsi-35000039988707d25-part2 - - /dev/disk/by-id/scsi-35000039988707d25-part3 + - /dev/disk/by-id/scsi-35000039988707d25 mac_addr: - 44:67:47:85:f4:65 - 44:67:47:85:f4:66 -- 2.23.0

1 0

[PATCH v2 lkp-tests] sbin/submit: support nr_run
by Li Yuanchao 20 Apr '21

20 Apr '21

If nr_run is a integer and nr_run > 1, submit nr_run times. Else, submit once. [usage] submit xxx xxx nr_run=3 xxx.yaml Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/submit | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/sbin/submit b/sbin/submit index edb207cc0..bcc155f3e 100755 --- a/sbin/submit +++ b/sbin/submit @@ -195,16 +195,21 @@ ARGV.each do |jobfile| scheduler_client = SchedulerClient.new(job['SCHED_HOST'], job['SCHED_PORT']) # submit job + nr_run = job_hash['nr_run'] || 1 + nr_run = nr_run.to_i + nr_run = 1 if nr_run < 1 job_json = job_hash.to_json - messages = scheduler_client.submit_job(job_json) - JSON.parse(messages).each do |msg| - if msg['message'].empty? - result_roots << msg['result_root'] - job_ids << msg['job_id'].to_s - puts("submit #{jobfile}, got job id=#{msg['job_id']}") - else - opt_monitor = false - puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + nr_run.times do + messages = scheduler_client.submit_job(job_json) + JSON.parse(messages).each do |msg| + if msg['message'].empty? + result_roots << msg['result_root'] + job_ids << msg['job_id'].to_s + puts("submit #{jobfile}, got job id=#{msg['job_id']}") + else + opt_monitor = false + puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + end end end end -- 2.23.0

1 0

[PATCH compass-ci] sparrow/3-code/dev-env: set default value
by Liu Yinsi 20 Apr '21

20 Apr '21

[error] submit_id=e628930d-f8ce-4ea1-b323-f8593c6f60cf 2021-04-19 20:59:08 +0800 WARN -- skip non-executable /c/lkp-tests/tests/self-test submit /c/lkp-tests/jobs/host-info.yaml failed, got job id=0, error: The /srv/initrd/lkp/latest/lkp-aarch64.cgz does not exist. [why] when selftest, if not to define $lkp_initrd_user in /etc/compass-ci/defaults/*.yaml, when submit job, it will not to find /srv/initrd/lkp/$lkp_initrd_user/lkp-$arch.cgz. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/3-code/dev-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index 969889ed..e02de355 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -55,12 +55,14 @@ server_name=$(hostname | cut -f1 -d.) : ${MAILDIR:=/srv/cci/Maildir/.compass-ci} : ${SENT_MAILDIR:=/srv/cci/Maildir/.sent} : ${NETDATA_PORT:=20013} +: ${lkp_initrd_user:=latest} mkdir -p /etc/compass-ci/defaults cat > /etc/compass-ci/defaults/$server_name.yaml <<EOF SCHED_HOST: $sched_host SCHED_PORT: $sched_port lab: $lab +lkp_initrd_user: $lkp_initrd_user EOF [[ -f /etc/compass-ci/defaults/upstream-config ]] || { -- 2.23.0

1 0

[PATCH compass-ci] sparrow/1-storage/permission: set lkp owner and group
by Liu Yinsi 20 Apr '21

20 Apr '21

/srv/initrd/deps and /srv/initrd/pkg permission must be lkp:lkp, or build pkg and deps are unable to transfer to the directories. Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/1-storage/permission | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sparrow/1-storage/permission b/sparrow/1-storage/permission index db33a93..3c229c5 100755 --- a/sparrow/1-storage/permission +++ b/sparrow/1-storage/permission @@ -13,6 +13,9 @@ chmod_dir=( /srv/rabbitmq /c/git-repos /srv/tmp/public + /srv/initrd + /srv/initrd/pkg + /srv/initrd/deps ) chmod 775 "${chmod_dir[@]}" @@ -20,6 +23,8 @@ chmod 775 "${chmod_dir[@]}" chown_dir=( /srv/result /srv/initrd + /srv/initrd/pkg + /srv/initrd/deps /srv/es /srv/cache/netdata_cache /srv/cache/netdata_lib -- 2.23.0

1 0

[PATCH lkp-tests] sbin/submit: support nr_run
by Li Yuanchao 20 Apr '21

20 Apr '21

If nr_run is a integer and nr_run > 1, submit nr_run times. Else, submit once. [usage] submit xxx xxx nr_run=3 xxx.yaml Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/submit | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/sbin/submit b/sbin/submit index edb207cc0..2e6ad8960 100755 --- a/sbin/submit +++ b/sbin/submit @@ -195,16 +195,21 @@ ARGV.each do |jobfile| scheduler_client = SchedulerClient.new(job['SCHED_HOST'], job['SCHED_PORT']) # submit job - job_json = job_hash.to_json - messages = scheduler_client.submit_job(job_json) - JSON.parse(messages).each do |msg| - if msg['message'].empty? - result_roots << msg['result_root'] - job_ids << msg['job_id'].to_s - puts("submit #{jobfile}, got job id=#{msg['job_id']}") - else - opt_monitor = false - puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + nr_run = job_hash['nr_run'] || 1 + nr_run = nr_run.to_i + nr_run = 1 if nr_run < 1 + nr_run.times do + job_json = job_hash.to_json + messages = scheduler_client.submit_job(job_json) + JSON.parse(messages).each do |msg| + if msg['message'].empty? + result_roots << msg['result_root'] + job_ids << msg['job_id'].to_s + puts("submit #{jobfile}, got job id=#{msg['job_id']}") + else + opt_monitor = false + puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + end end end end -- 2.23.0

3 4

[PATCH lkp-tests 2/2] install-iso-firstboot: modify how job's IPXE response is retrieved
by Yu Chuan 19 Apr '21

19 Apr '21

[Why] Vmlinuz, initrd, reboot.cgz is all that is required for firstboot after ISO is installed. Therefore, the original os_mount=local is changed to custom_ipxe to obtain the ipxe response. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/install-iso-firstboot.yaml | 9 +-------- tests/install-iso-pre | 14 +++++++++----- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/jobs/install-iso-firstboot.yaml b/jobs/install-iso-firstboot.yaml index 7be9eaaab184..c170b8a07f8a 100644 --- a/jobs/install-iso-firstboot.yaml +++ b/jobs/install-iso-firstboot.yaml @@ -1,10 +1,3 @@ suite: install-iso-firstboot -sleep: 1 - -# os_mount must be equal to 'local' -os_mount: local - -# kernel_custom_params must have 'boot_lv_suffix' and 'src_lv_suffix', -# and they must be the same. so can boot from the specify logical volume -kernel_custom_params: +custom_ipxe: diff --git a/tests/install-iso-pre b/tests/install-iso-pre index 768ffb2cca45..c89895a994cc 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -225,14 +225,18 @@ submit_install_iso_firstboot_yaml() local firstboot_yaml="${LKP_SRC}/jobs/install-iso-firstboot.yaml" [ -f "${firstboot_yaml}" ] || die "cannot find ${firstboot_yaml}" + custom_ipxe="#!ipxe\\\\n" + custom_ipxe="${custom_ipxe}\\\\ndhcp" + custom_ipxe="${custom_ipxe}\\\\ninitrd http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/${iso_os}/${iso_arch}/${iso_version}/initrd.lkp" + custom_ipxe="${custom_ipxe}\\\\ninitrd http://${INITRD_HTTP_HOST}:${INITRD_HTTP_PORT}/initrd/custom_bootstrap/reboot.cgz" + custom_ipxe="${custom_ipxe}\\\\nkernel http://${OS_HTTP_HOST}:${OS_HTTP_PORT}/os/${iso_os}/${iso_arch}/${iso_version}/boot/vmlinuz ip=dhcp root=/dev/mapper/os-${iso_os}_${iso_arch}_${iso_version}_${ROOTFS_SUFFIX} initrd=initrd.lkp initrd=reboot.cgz local rw" + custom_ipxe="${custom_ipxe}\\\\nboot\\\\n" + + sed -i "/custom_ipxe:/ccustom_ipxe: \"${custom_ipxe}\"" $firstboot_yaml + ${LKP_SRC}/sbin/submit \ ${firstboot_yaml} \ suite=install-iso-firstboot-${ROOTFS_SUFFIX} \ - os=${iso_os} \ - os_arch=${iso_arch} \ - os_version=${iso_version} \ - os_mount=local \ - kernel_custom_params="src_lv_suffix=${ROOTFS_SUFFIX} boot_lv_suffix=${ROOTFS_SUFFIX}" \ testbox=$testbox || die "submit ${firstboot_yaml} failed" } -- 2.23.0

1 0

[PATCH lkp-tests 1/2] install-iso-pre: fix some bugs
by Yu Chuan 19 Apr '21

19 Apr '21

[What Bugs] 1. fix the use of secrets fields. 2. add mkfs.ext4 step after create logical volume. 3. vmlinux => vmlinuz 4. ks upload dir: - before: /srv/os/kickstart/ks/${iso_os}_${iso_arch}_${iso_version}.cfg - after: /srv/os/install/ks/${iso_os}_${iso_arch}_${iso_version}.ks 5. second-level hash problem when submit. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- tests/install-iso-pre | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/install-iso-pre b/tests/install-iso-pre index ef4a0659fe02..768ffb2cca45 100755 --- a/tests/install-iso-pre +++ b/tests/install-iso-pre @@ -21,7 +21,7 @@ check_params() "HTTP_SERVER_HOST" "HTTP_SERVER_PORT" "HTTP_ROOT_DIR" - "ROOTFS_ROOT_PASSWD" + "secrets_ROOTFS_ROOT_PASSWD" "rootfs_disk" "RSYNC_SERVER_HOST" @@ -31,7 +31,7 @@ check_params() "NFS_ROOT_DIR" "LAB_Z9_URL" - "MY_TOKEN" + "secrets_MY_TOKEN" ) for i in "${required_vars[@]}" @@ -65,6 +65,8 @@ create_lv() log_info "start create lv:$LV_NAME" lvcreate -y -L 10G -n ${LV_NAME} ${VG_NAME} || die "create logical volume failed." + + mkfs.ext4 /dev/mapper/${VG_NAME}-${LV_NAME} } prepare_lv() @@ -111,7 +113,7 @@ generate_ks() logvol / --vgname=${VG_NAME} --name=${LV_NAME} --useexisting # Root password - rootpw --iscrypted ${ROOTFS_ROOT_PASSWD} + rootpw --iscrypted ${secrets_ROOTFS_ROOT_PASSWD} # Reboot after installation reboot --eject @@ -180,7 +182,7 @@ prepare_submit() SCHED_PORT: ${SCHED_PORT} my_name: ${my_name} my_email: ${my_email} - my_token: ${MY_TOKEN} + my_token: ${secrets_MY_TOKEN} lab: z9 EOF } @@ -195,7 +197,7 @@ prepare_install_iso_yaml() custom_ipxe="#!ipxe\\\\n" custom_ipxe="${custom_ipxe}\\\\ndhcp" custom_ipxe="${custom_ipxe}\\\\ninitrd ${tftp_root}/initrd.img" - custom_ipxe="${custom_ipxe}\\\\nkernel ${tftp_root}/vmlinux ro inst.geoloc=0 console=ttyS0 smmu.bypassdev=0x1000:0x17 smmu.bypassde1000:0x15 viff video=VGA-1:640x480-32@60me initrd=initrd.img ip=dhcp inst.ks=http://${HTTP_SERVER_HOST}:${HTTP_SERVER_PORT}/os/kickstart/ks/${iso_os}_${iso_arch}_${iso_version}.cfg" + custom_ipxe="${custom_ipxe}\\\\nkernel ${tftp_root}/vmlinuz ro inst.geoloc=0 console=ttyS0 smmu.bypassdev=0x1000:0x17 smmu.bypassde1000:0x15 viff video=VGA-1:640x480-32@60me initrd=initrd.img ip=dhcp inst.ks=http://${HTTP_SERVER_HOST}:${HTTP_SERVER_PORT}/os/install/ks/${iso_os}_${iso_arch}_${iso_version}.ks" custom_ipxe="${custom_ipxe}\\\\nboot\\\\n" log_info "custom_ipxe: $custom_ipxe" @@ -244,13 +246,13 @@ submit_install_iso_sync_rootfs_yaml() ${LKP_SRC}/sbin/submit \ ${sync_rootfs_yaml} \ suite=install-iso-sync-rootfs-${ROOTFS_SUFFIX} \ - iso_os=${iso_os} \ - iso_arch=${iso_arch} \ - iso_version=${iso_version} \ - NFS_SERVER_HOST=${NFS_SERVER_HOST} \ - NFS_ROOT_DIR=${NFS_ROOT_DIR} \ - ROOTFS_SUFFIX=${ROOTFS_SUFFIX} \ - VG_NAME=${VG_NAME} \ + install-iso-sync-rootfs.iso_os=${iso_os} \ + install-iso-sync-rootfs.iso_arch=${iso_arch} \ + install-iso-sync-rootfs.iso_version=${iso_version} \ + install-iso-sync-rootfs.NFS_SERVER_HOST=${NFS_SERVER_HOST} \ + install-iso-sync-rootfs.NFS_ROOT_DIR=${NFS_ROOT_DIR} \ + install-iso-sync-rootfs.ROOTFS_SUFFIX=${ROOTFS_SUFFIX} \ + install-iso-sync-rootfs.VG_NAME=${VG_NAME} \ testbox=$testbox || die "submit ${sync_rootfs_yaml} failed" } -- 2.23.0

1 0