- Compass-ci - mailweb.openeuler.org

[PATCH v4 compass-ci] feat: build centos-7 initramfs image with one command
by Wang Chenglong 13 Oct '20

13 Oct '20

Use the tool to create a new centos-7 initramfs image with one command. and we also can customize the pre-installed software by configuring ./bin/install-list. --- container/osimage/centos-7/README.md | 13 +++++++ container/osimage/centos-7/bin/create-image | 38 +++++++++++++++++++ container/osimage/centos-7/bin/lib | 30 +++++++++++++++ .../osimage/centos-7/bin/packages-to-install | 2 + .../osimage/centos-7/bin/packages-to-remove | 8 ++++ container/osimage/centos-7/build | 29 ++++++++++++++ 6 files changed, 120 insertions(+) create mode 100644 container/osimage/centos-7/README.md create mode 100755 container/osimage/centos-7/bin/create-image create mode 100755 container/osimage/centos-7/bin/lib create mode 100644 container/osimage/centos-7/bin/packages-to-install create mode 100644 container/osimage/centos-7/bin/packages-to-remove create mode 100755 container/osimage/centos-7/build diff --git a/container/osimage/centos-7/README.md b/container/osimage/centos-7/README.md new file mode 100644 index 0000000..d1a0317 --- /dev/null +++ b/container/osimage/centos-7/README.md @@ -0,0 +1,13 @@ +# Use the tool to create a new centos-7 initramfs image. + +Usage: + cd ${CCI_SRC/rootfs/initramfs/centos/aarch64/7} + ./build + +Some configuration items: +./bin/packages-to-install + If you want to pre-install the software, you can write the package names in ./bin/packages-to-install. + +./bin/packages-to-remove + If you want remove some unnecessary files, you can write the names in ./bin/packages-to-remove + diff --git a/container/osimage/centos-7/bin/create-image b/container/osimage/centos-7/bin/create-image new file mode 100755 index 0000000..214204f --- /dev/null +++ b/container/osimage/centos-7/bin/create-image @@ -0,0 +1,38 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +# Configure rootfs +make_rootfs() +{ + yum repolist + yum -y install --skip-broken $(</root/bin/packages-to-install) + yum clean all + rm -rf $(</root/bin/packages-to-remove) + ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + cd /root/modules + zcat modules-5.8.0.1.cgz | cpio -idm +} + +# Config password +pre_config_rootfs() { + [ -n "$ROOT_NEW_PASSWD" ] && { + echo "Changing root password" + passwd_md5=$(openssl passwd -1 "$ROOT_NEW_PASSWD") + sed -i -r "s/^root:[^:]*:(.*)/root:${passwd_md5//\//\\/}:\1/" "$ROOTFS_DIR/etc/shadow" # Change the password in shadow + sed -i 's/[# ]PermitRootLogin.*/PermitRootLogin yes/' "$ROOTFS_DIR/etc/ssh/sshd_config" # Configure ssh service + } +} + +# Pack cgz package +pack_cgz() +{ + echo "Packing package. Please wait." + cd / + find ./ ! -path "./${image_name}" ! -path "./root/modules/modules-5.8.0.1.cgz" | cpio -o -Hnewc | gzip -9 > /${image_name} + chmod 644 /${image_name} +} + +make_rootfs +pre_config_rootfs +pack_cgz diff --git a/container/osimage/centos-7/bin/lib b/container/osimage/centos-7/bin/lib new file mode 100755 index 0000000..c3c470f --- /dev/null +++ b/container/osimage/centos-7/bin/lib @@ -0,0 +1,30 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +DIR="$(basename $(dirname $(realpath $0)))" +name="$(echo $DIR |awk -F "-" '{print $1}')" +version="$(echo $DIR |awk -F "-" '{print $2}')" +image="${name}:${version}" +image_name="${name}-${version}-$(date +"%Y%m%d").cgz" + +# Check password file +check_passwd_file() { + root_pwd_file="$HOME/.config/compass-ci/rootfs.passwd" + export ROOT_NEW_PASSWD= + [ -f "$root_pwd_file" ] || { + echo "[INFO] Please set the password file." + echo "$HOME/.config/compass-ci/rootfs.passwd" + exit 1 + } + + export ROOT_NEW_PASSWD=$(cat "$root_pwd_file") +} + +# Pull docker image +pull_docker() +{ + docker pull $image || return + echo "finish downloading image" +} + diff --git a/container/osimage/centos-7/bin/packages-to-install b/container/osimage/centos-7/bin/packages-to-install new file mode 100644 index 0000000..5d31c40 --- /dev/null +++ b/container/osimage/centos-7/bin/packages-to-install @@ -0,0 +1,2 @@ +openssh-server +openssl diff --git a/container/osimage/centos-7/bin/packages-to-remove b/container/osimage/centos-7/bin/packages-to-remove new file mode 100644 index 0000000..ce079b9 --- /dev/null +++ b/container/osimage/centos-7/bin/packages-to-remove @@ -0,0 +1,8 @@ +/.dockerenv +/lib/modules +/usr/share/doc +/usr/share/man +/usr/share/info +/usr/share/i18n +/usr/share/locale +/usr/share/terminfo diff --git a/container/osimage/centos-7/build b/container/osimage/centos-7/build new file mode 100755 index 0000000..5e46306 --- /dev/null +++ b/container/osimage/centos-7/build @@ -0,0 +1,29 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. ${CCI_SRC}/container/osimage/centos-7/bin/lib +. ${CCI_SRC}/container/defconfig.sh + +check_passwd_file "$root_pwd_file" + +DIR=$(dirname $(realpath $0)) +cmd=( + docker run + --name init_docker + -v $DIR/bin/:/root/bin + -v /srv/initrd/modules/:/root/modules:ro + -e ROOT_NEW_PASSWD=$ROOT_NEW_PASSWD + -e image_name=$image_name + $image + /root/bin/create-image +) + +pull_docker + +docker_rm init_docker +${cmd[@]} + +docker cp -a init_docker:/${image_name} ./ +echo "build finished!!!" + -- 2.23.0

3 12

[PATCH compass-ci] sparrow: add docker test
by Liu Yinsi 12 Oct '20

12 Oct '20

1. submit job 2. run docker test Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/6-test/docker | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/sparrow/6-test/docker b/sparrow/6-test/docker index 3d6796d..ab825d3 100755 --- a/sparrow/6-test/docker +++ b/sparrow/6-test/docker @@ -1,7 +1,13 @@ -#!/bin/bash -e +#!/usr/bin/env ruby # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true -## TODO -# 1.submit a job -# 2.run docker.sh +require_relative '../../providers/docker/docker' + +def run_job(hostname) + system "submit -s 'testbox: #{hostname}' #{ENV['CCI_SRC']}/user-client/jobs/iperf-sparrow.yaml" + main "#{hostname}" +end + +run_job "dc-1g" -- 2.23.0

2 4

[PATCH compass-ci] use local docker repository
by Xiao Shenwei 12 Oct '20

12 Oct '20

add DOCKER_REGISTRY_HOST, DOCKER_REGISTRY_PORT field into /etc/compass-ci/defaults/*.yaml, or $HOME/.config/compass-ci/defaults/*.yaml also, if failed from local registry, will pull from docker hub Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- providers/docker/docker.rb | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/providers/docker/docker.rb b/providers/docker/docker.rb index c75fc2c..eea0336 100755 --- a/providers/docker/docker.rb +++ b/providers/docker/docker.rb @@ -11,14 +11,17 @@ require_relative '../../container/defconfig' BASE_DIR = '/srv/dc' +names = Set.new %w[ + SCHED_HOST + SCHED_PORT + DOCKER_REGISTRY_HOST + DOCKER_REGISTRY_PORT +] +DEFAULTS = relevant_defaults(names) + def get_url(hostname) - names = Set.new %w[ - SCHED_HOST - SCHED_PORT - ] - defaults = relevant_defaults(names) - host = defaults['SCHED_HOST'] || '172.17.0.1' - port = defaults['SCHED_PORT'] || 3000 + host = DEFAULTS['SCHED_HOST'] || '172.17.0.1' + port = DEFAULTS['SCHED_PORT'] || 3000 "http://#{host}:#{port}/boot.container/hostname/#{hostname}" end @@ -59,7 +62,7 @@ def clean_dir(path) end def load_initrds(load_path, hash) - clean_dir(load_path) if Dir.exist?(load_path) + clean_dir(load_path) unless Dir.empty?(load_path) arch = RUBY_PLATFORM.split('-')[0] job_url = hash['job'] lkp_url = hash['lkp'] @@ -67,9 +70,18 @@ def load_initrds(load_path, hash) wget_cmd(load_path, lkp_url, "lkp-#{arch}.cgz") end +def pull(image_name) + host = DEFAULTS['DOCKER_REGISTRY_HOST'] || '172.17.0.1' + port = DEFAULTS['DOCKER_REGISTRY_PORT'] || '5001' + flag = system "docker pull #{host}:#{port}/#{image_name}" + unless flag + system "docker pull #{image_name}" + end +end + def run(hostname, load_path, hash) docker_image = hash['docker_image'] - system "docker pull #{docker_image}" + pull docker_image system( { 'hostname' => hostname, 'docker_image' => docker_image, 'load_path' => load_path }, ENV['CCI_SRC'] + '/providers/docker/run.sh' -- 2.23.0

2 4

[PATCH compass-ci] fix: scheduler: update job root_result error
by Tong Qunfeng 12 Oct '20

12 Oct '20

1) after job finished, we save update job field back to es. 2) if the job is consumed at "20201010", but finished at "20201011". then the result_root will be update to "/result/$suite/$tbox_group/20201011/$id". 3) but the result is really saved at "/result/$suite/$tbox_group/20201010/$id", 20201010 is the date when it is consumed. no need to set_default for an initialized job (when is is consumed). Signed-off-by: Tong Qunfeng <taxcom(a)tom.com> --- src/lib/job.cr | 40 +++++++++++++++++++++++++++++++++++++--- src/lib/sched.cr | 1 + 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 504bc19..be3446e 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -50,10 +50,13 @@ class Job @hash = job_content.as_h # init job with "-1", or use the original job_content["id"] - if "#{id}" == "" - @hash["id"] = JSON::Any.new("-1") + id == "-1" if "#{id}" == "" + + if initialized? + return if @hash["id"] == "#{id}" end + @hash["id"] = JSON::Any.new("#{id}") check_required_keys() set_defaults() end @@ -165,7 +168,7 @@ class Job self["os_dir"] = "#{os}/#{os_arch}/#{os_version}" end - private def set_result_root + def set_result_root update_tbox_group_from_testbox # id must exists, need update tbox_group date = Time.local.to_s("%F") self["result_root"] = "/result/#{suite}/#{tbox_group}/#{date}/#{id}" @@ -234,6 +237,37 @@ class Job end end + private def initialized? + initialized_keys = [] of String + + REQUIRED_KEYS.each do |key| + initialized_keys << key.to_s + end + + METHOD_KEYS.each do |key| + initialized_keys << key.to_s + end + + INIT_FIELD.each do |key, _value| + initialized_keys << key.to_s + end + + initialized_keys += ["result_service", + "LKP_SERVER", + "LKP_CGI_PORT", + "SCHED_HOST", + "SCHED_PORT"] + + initialized_keys.each do |key| + if @hash.has_key?(key) == false + return false + end + end + + return false if "#{@hash["id"]}" == "" + return true + end + private def vmlinuz if os_mount == "initramfs" return "#{INITRD_HTTP_PREFIX}" + diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 23332df..2fbf05d 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -451,6 +451,7 @@ class Sched if job job.update({"testbox" => testbox}) + job.set_result_root @redis.set_job(job) end return job -- 2.23.0

1 0

[PATCH compass-ci] container/remote-git: disable encode
by Lin Jiaxin 12 Oct '20

12 Oct '20

encode change binary file's md5sums. error: ==> Validating source files with md5sums... VUE.jar ... Passed vue ... Passed vue.png ... FAILED vue.desktop ... Passed ==> ERROR: One or more files did not pass the validity check! md5sum vue.png 7253b39732630ba58186d7b79263b2a3 vue.png correct: ==> Validating source files with md5sums... VUE.jar ... Passed vue ... Passed vue.png ... Passed vue.desktop ... Passed md5sum vue.png 9369da34c69baefc77a7baeaceb4e476 vue.png Signed-off-by: Lin Jiaxin <ljx.joe(a)qq.com> --- container/remote-git/remote-git.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/remote-git/remote-git.rb b/container/remote-git/remote-git.rb index 085f4db..44a8e18 100755 --- a/container/remote-git/remote-git.rb +++ b/container/remote-git/remote-git.rb @@ -41,7 +41,7 @@ post '/git_command' do # execute git command Dir.chdir(repo_path) _stdin, stdout, _stderr, wait_thr = Open3.popen3(*data['git_command']) - out = stdout.read.force_encoding('ISO-8859-1').encode('UTF-8') + out = stdout.read exit_code = wait_thr.value.to_i [200, headers.update({ 'errcode' => '0', 'exit_code' => exit_code.to_s }), out] -- 2.23.0

1 0

[PATCH v5 compass-ci 4/6] container/mail-robot: Dockerfile
by Luan Shengde 12 Oct '20

12 Oct '20

Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/mail-robot/Dockerfile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 container/mail-robot/Dockerfile diff --git a/container/mail-robot/Dockerfile b/container/mail-robot/Dockerfile new file mode 100644 index 0000000..0eddbc4 --- /dev/null +++ b/container/mail-robot/Dockerfile @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM debian +MAINTAINER shdluan(a)163.com +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && \ + apt-get install -y uuid-runtime curl ruby-listen ruby-json ruby-mail && \ + gem install fileutils elasticsearch activesupport && \ + ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + echo 'Asia/Shanghai' > /etc/timezone + +COPY mail-robot.rb /usr/local/bin/ + +COPY answerback-email.rb /usr/local/bin/ + +WORKDIR /usr/local/bin/ -- 2.23.0

2 1

[PATCH v5 compass-ci 3/6] container/mail-robot: build file
by Luan Shengde 12 Oct '20

12 Oct '20

Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/mail-robot/build | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 container/mail-robot/build diff --git a/container/mail-robot/build b/container/mail-robot/build new file mode 100755 index 0000000..e742191 --- /dev/null +++ b/container/mail-robot/build @@ -0,0 +1,6 @@ +#!/bin/sh +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +docker build -t mail-robot:latest . + -- 2.23.0

2 1

[PATCH compass-ci] src: remove hi1620 from vm-hi1620
by Bai Jing 12 Oct '20

12 Oct '20

Signed-off-by: Bai Jing <799286817(a)qq.com> --- src/delimiter/constants.rb | 2 +- src/features/jobs/right_iperf.yaml | 2 +- src/features/scheduler.feature | 2 +- src/features/step_definitions/scheduler_step.rb | 2 +- src/lib/web_backend.rb | 4 ++-- src/scheduler/README.md | 2 +- src/taskqueue/queue.cr | 2 +- src/taskqueue/redis_client.cr | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/delimiter/constants.rb b/src/delimiter/constants.rb index 7a17b36..dafc2c8 100644 --- a/src/delimiter/constants.rb +++ b/src/delimiter/constants.rb @@ -9,4 +9,4 @@ MONITOR_PORT = ENV['MONITOR_PORT'] || '11310' TMEP_GIT_BASE = '/c/public_git' DELIMITER_TASK_QUEUE = 'delimiter' BISECT_RUN_SCRIPT = "#{ENV['CCI_SRC']}/src/delimiter/find-commit/bisect_run_script.rb" -DELIMITER_TBOX_GROUP = 'vm-hi1620-2p8g--delimiter' +DELIMITER_TBOX_GROUP = 'vm-2p8g--delimiter' diff --git a/src/features/jobs/right_iperf.yaml b/src/features/jobs/right_iperf.yaml index 843564d..f81af38 100644 --- a/src/features/jobs/right_iperf.yaml +++ b/src/features/jobs/right_iperf.yaml @@ -3,7 +3,7 @@ #! jobs/iperf.yaml suite: iperf testcase: iperf -testbox: vm-hi1620-2p8g-chief +testbox: vm-2p8g--chief category: benchmark runtime: 30 cluster: cs-localhost diff --git a/src/features/scheduler.feature b/src/features/scheduler.feature index c335fb4..7c1a804 100644 --- a/src/features/scheduler.feature +++ b/src/features/scheduler.feature @@ -6,7 +6,7 @@ Feature: Scheduler Given prepared a job "right_iperf.yaml" When call with API: post "submit_job" job from add_job.sh Then return with job id - When call with API: put "set_host_mac" "vm-hi1620-2p8g-chief => ef-01-02-03-04-05" + When call with API: put "set_host_mac" "vm-2p8g--chief => ef-01-02-03-04-05" And call with API: get "boot.ipxe/mac/ef-01-02-03-04-05" Then return with basic ipxe boot parameter and initrd and kernel diff --git a/src/features/step_definitions/scheduler_step.rb b/src/features/step_definitions/scheduler_step.rb index 882ddd8..2872bc7 100644 --- a/src/features/step_definitions/scheduler_step.rb +++ b/src/features/step_definitions/scheduler_step.rb @@ -22,7 +22,7 @@ Then('return with job id') do end # set_host_mac -# host_mac => "vm-hi1620-2p8g-chief => ef-01-02-03-04-05" +# host_mac => "vm-2p8g--chief => ef-01-02-03-04-05" Given('call with API: put {string} {string}') do |url, host_mac| host_mac_params = host_mac.split(' ') url_with_params = "#{url}?hostname=#{host_mac_params[0]}\\&mac=#{host_mac_params[2]}" diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 4544690..31aa0c3 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -41,8 +41,8 @@ def es_count(query) ES_CLIENT.count(index: 'jobs*', body: query)['count'] end -# "vm-hi1620-2p8g-212" remove "-212" -# "vm-hi1620-2p8g-zzz" remove "-zzz" +# "vm-2p8g--212" remove "--212" +# "vm-2p8g--zzz" remove "--zzz" # "vm-git-bisect" don't remove "-bisect" def filter_tbox_group(es_result) result = Set.new diff --git a/src/scheduler/README.md b/src/scheduler/README.md index 5da1bde..fdc7027 100644 --- a/src/scheduler/README.md +++ b/src/scheduler/README.md @@ -413,7 +413,7 @@ you can simply call the scheduler's build and run script. vim $CCI_SRC/user-client/helper/iperf.yaml runtime: 30 # unit in seconds, the time can be shortened arch: aarch64 - testbox: vm-hi1620-2p8g-$USER # echo $USER for your own user name + testbox: vm-2p8g--$USER # echo $USER for your own user name ## 2.Submit job submit-job $_CCI_SRC/user-client/helper/iperf.yaml diff --git a/src/taskqueue/queue.cr b/src/taskqueue/queue.cr index 02236f5..674606e 100644 --- a/src/taskqueue/queue.cr +++ b/src/taskqueue/queue.cr @@ -183,7 +183,7 @@ class TaskQueue private def move_uuid_task2ready(queue_name) # get uuid lists - # uuid_keys ["queues/sched/vm-hi1620-2p8g/ee44b164-90e3-49a7-9798-5e7cc9bc7451", ] + # uuid_keys ["queues/sched/vm-2p8g/ee44b164-90e3-49a7-9798-5e7cc9bc7451", ] uuid_keys = get_uuid_keys("#{queue_name}") return unless uuid_keys # nil return unless uuid_keys.size > 0 diff --git a/src/taskqueue/redis_client.cr b/src/taskqueue/redis_client.cr index 3bac924..27d46cf 100644 --- a/src/taskqueue/redis_client.cr +++ b/src/taskqueue/redis_client.cr @@ -236,7 +236,7 @@ class TaskQueue private def get_uuid_keys(queue_name) return nil unless queue_name[0..5] == "sched/" - # search = "queues/sched/vm-hi1620-2p8g/ee44b164-90e3-49a7-9798-5e7cc9bc7451" + # search = "queues/sched/vm-2p8g/ee44b164-90e3-49a7-9798-5e7cc9bc7451" # only 3 matchs keyword: * [] ? search = "#{QUEUE_NAME_BASE}/#{queue_name}/[0-9a-eA-F\-]*" lua_script = "return redis.call('keys', KEYS[1])" -- 2.23.0

1 0

[PATCH v5 compass-ci 2/6] container/mail-robot: answerback-email.rb
by Luan Shengde 12 Oct '20

12 Oct '20

read email content check email available check if base url in upstream-repos return if base url not in upstream-repos check if commit url valid return if no match commit info check pub_key return if no pub_key assign account generate uuid write account/uuid to es send uuid to user Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/mail-robot/answerback-email.rb | 207 +++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100755 container/mail-robot/answerback-email.rb diff --git a/container/mail-robot/answerback-email.rb b/container/mail-robot/answerback-email.rb new file mode 100755 index 0000000..7664f37 --- /dev/null +++ b/container/mail-robot/answerback-email.rb @@ -0,0 +1,207 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'mail' +require_relative '/compass-ci/lib/es_client.rb' + +def build_message(sum_infos) + # To: #{sum_infos['email']} + message = <<~EMAIL_MESSAGE + To: shdluan(a)163.com + Message-ID: #{sum_infos['message_id']} + Subject: Account Ready + + Dear #{sum_infos['email'].split('@')[0]} + + Thank you for joining us. + You can use the following info to submit jobs: + + First add the info to: ~/.config/compass-ci/defaults/{USER}.yaml + + uuid: #{sum_infos['account_uuid']} + lab: z9 + + Second: download lkp-tests and dependencies + git clone https://gitee.com_/wu_fengguang/lkp-tests.git + cd lkp-tests + make install + source {HOME}/.bashrc && source {HOME}/.bash_profile + + Third: submit job + reference: https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/tutorial.md + how to write job yaml + submit jobs/netperf.yaml + + Repalce the fields {...} with your local environment variable. + + regards + compass-ci + EMAIL_MESSAGE + + return message +end + +def email_addr(mail_content) + email = mail_content.from[0] + + return email +end + +def url_addr(mail_content) + url_line = mail_content.part[0].body.decoded.split(/\n/).find { |line| line =~ %r{https?://} } + message = 'No commit url found' + + raise message unless url_line + + url = url_line.split[-1] + + return url +end + +def base_url_addr(url) + url_a = url.split('/') + url_a.pop 2 + base_url = url_a.join('/') + + return base_url +end + +def check_email_available(mail_content, email) + url = url_addr(mail_content) + + base_url = base_url_addr(url) + + upstream_dir = '/upstream-repos' + check_url_in_upstream_repos(upstream_dir, base_url) + check_url_commit(url, base_url, email) + + return url +end + +def check_url_in_upstream_repos(upstream_dir, base_url) + Dir.chdir(upstream_dir) + match_out = %x(grep -rn #{base_url}) + + message = 'The url is not in upstream-repo list' + + raise StandardError, message, base_url if match_out.empty? +end + +def check_url_commit(url, base_url, email) + hub_name = url.split('/')[2] + check_gitee_commit(url, base_url, email) if hub_name.eql? 'gitee.com' + check_non_gitee_commit(url, email) unless hub_name.eql? 'gitee.com' +end + +def check_non_gitee_commit(url, email) + url_fdback = %x(curl #{url}) + email_index = url_fdback.index email + + message = 'No email matched from the commit_id feedback.' + raise StandardError, message, url if email_index.nil? +end + +def check_gitee_commit(url, base_url, email) + repo_info = { + 'repos_dir' => '/repos_dir', + 'repo_dir' => url.split('/')[-3], + 'repo_url' => [base_url, 'git'].join('.'), + 'repo_path' => File.join('/repos_dir', url.split('/')[-3]), + 'commit_id' => url.split('/')[-1] + } + check_gitee_repo_commit(url, email, repo_info) +end + +def check_gitee_repo_commit(url, email, repo_info) + Dir.chdir repo_info['repos_dir'] + %x(/usr/bin/git clone #{repo_info['repo_url']}) + Dir.chdir repo_info['repo_path'] + email_index = %x(/usr/bin/git show #{repo_info['commit_id']}).index email + + Dir.chdir repo_info['repos_dir'] + FileUtils.rm_rf repo_info['repo_dir'] + + message = 'No email matched from commit_id feedback.' if email_index.nil? + raise StandardError, message, url if email_index.nil? +end + +def email_message_id(mail_content) + message_id = mail_content.message_id + return message_id +end + +def pub_key_value(mail_content) + pub_key = mail_content.part[1].body.decoded.split(/\r|\n/).join if mail_content.part[1].filename == 'id_rsa.pub' + + message = 'No pub_key found' + + raise message unless pub_key + + return pub_key +end + +def account_info(pub_key) + account_info_str = %x(curl -XGET "#{ENV['JUMPER_IP']}:#{ENV['JUMPER_PORT']}/assign_account" -d "pub_key: #{pub_key}") + account_info = JSON.parse account_info_str + + message = 'No available jumper account returned.' + raise StandardError, message, account_info if account_info['account'].empty? + + return account_info +end + +def setup_account_uuid + account_uuid = %x(uuidgen).chomp + + return account_uuid +end + +def sum_infos(mail_content) + email = email_addr(mail_content) + pub_key = pub_key_value(mail_content) + sum_infos = { + 'email' => email, + 'message_id' => email_message_id(mail_content), + 'url' => check_email_available(mail_content, email), + 'acct_infos' => account_info(pub_key), + 'account_uuid' => setup_account_uuid, + 'pub_key' => pub_key + } + store_account_info(sum_infos) + sum_infos +end + +def send_uuid(sum_infos) + message = build_message(sum_infos) + + send_mail_server = `/sbin/ip route |awk '/default/ {print $3}'`.chomp + %x(curl -XPOST "#{send_mail_server}:#{ENV['SEND_MAIL_PORT']}/send_mail_text" -d "#{message}") +end + +def send_account(mail_content) + sum_infos = sum_infos(mail_content) + + send_uuid(sum_infos) +end + +def store_account_info(sum_infos) + account_uuid_info = { + 'email' => sum_infos['email'], + 'name' => sum_infos['email'].split('@')[0], + 'uuid' => sum_infos['account_uuid'], + 'user' => sum_infos['acct_infos']['account'], + 'my_commit_url' => sum_infos['url'] + } + + es = ESClient.new(index: 'accounts') + es.put_source_by_id(sum_infos['email'], account_uuid_info) +end + +def read_mail_content(mail_file) + mail_content = Mail.read(mail_file) + + return mail_content +end -- 2.23.0

2 1

[PATCH v4 compass-ci] feat: build centos initramfs image with one command
by Wang Chenglong 12 Oct '20

12 Oct '20

Use the tool to create a new centos-$version initramfs image with one command. and we also can customize the pre-installed software by configuring ./bin/packages-to-install. --- container/osimage/centos/README.md | 15 +++++++ container/osimage/centos/bin/create-image | 38 +++++++++++++++++ container/osimage/centos/bin/lib | 41 +++++++++++++++++++ .../osimage/centos/bin/packages-to-install | 2 + .../osimage/centos/bin/packages-to-remove | 8 ++++ container/osimage/centos/run | 27 ++++++++++++ 6 files changed, 131 insertions(+) create mode 100644 container/osimage/centos/README.md create mode 100755 container/osimage/centos/bin/create-image create mode 100755 container/osimage/centos/bin/lib create mode 100644 container/osimage/centos/bin/packages-to-install create mode 100644 container/osimage/centos/bin/packages-to-remove create mode 100755 container/osimage/centos/run diff --git a/container/osimage/centos/README.md b/container/osimage/centos/README.md new file mode 100644 index 0000000..000ca05 --- /dev/null +++ b/container/osimage/centos/README.md @@ -0,0 +1,15 @@ +# Use the tool to create a new centos-${version} initramfs image. + +Usage: + cd ${CCI_SRC}/rootfs/initramfs/centos/aarch64/${version}/ + ./build + +Some configuration items: +./bin/packages-to-install + If you want to pre-install the software, you can write the package names in ./bin/packages-to-install. + +./bin/packages-to-remove + If you want remove some unnecessary files, you can write the names in ./bin/packages-to-remove. + +$HOME/.config/compass-ci/rootfs.passwd + Set the password for the image into this file. diff --git a/container/osimage/centos/bin/create-image b/container/osimage/centos/bin/create-image new file mode 100755 index 0000000..214204f --- /dev/null +++ b/container/osimage/centos/bin/create-image @@ -0,0 +1,38 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +# Configure rootfs +make_rootfs() +{ + yum repolist + yum -y install --skip-broken $(</root/bin/packages-to-install) + yum clean all + rm -rf $(</root/bin/packages-to-remove) + ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + cd /root/modules + zcat modules-5.8.0.1.cgz | cpio -idm +} + +# Config password +pre_config_rootfs() { + [ -n "$ROOT_NEW_PASSWD" ] && { + echo "Changing root password" + passwd_md5=$(openssl passwd -1 "$ROOT_NEW_PASSWD") + sed -i -r "s/^root:[^:]*:(.*)/root:${passwd_md5//\//\\/}:\1/" "$ROOTFS_DIR/etc/shadow" # Change the password in shadow + sed -i 's/[# ]PermitRootLogin.*/PermitRootLogin yes/' "$ROOTFS_DIR/etc/ssh/sshd_config" # Configure ssh service + } +} + +# Pack cgz package +pack_cgz() +{ + echo "Packing package. Please wait." + cd / + find ./ ! -path "./${image_name}" ! -path "./root/modules/modules-5.8.0.1.cgz" | cpio -o -Hnewc | gzip -9 > /${image_name} + chmod 644 /${image_name} +} + +make_rootfs +pre_config_rootfs +pack_cgz diff --git a/container/osimage/centos/bin/lib b/container/osimage/centos/bin/lib new file mode 100755 index 0000000..b4a59e2 --- /dev/null +++ b/container/osimage/centos/bin/lib @@ -0,0 +1,41 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +DIR="$(pwd)" +name="$(echo $DIR |awk -F "/" '{print $(NF - 2)}')" +version="$(echo $DIR |awk -F "/" '{print $NF}')" +image="${name}:${version}" +image_name="${name}-${version}-$(date +"%Y%m%d").cgz" + +# Check password file +check_passwd_file() { + root_pwd_file="$HOME/.config/compass-ci/rootfs.passwd" + export ROOT_NEW_PASSWD= + [ -f "$root_pwd_file" ] || { + echo "[INFO] Please set the password file." + echo "$HOME/.config/compass-ci/rootfs.passwd" + exit 1 + } + + export ROOT_NEW_PASSWD=$(cat "$root_pwd_file") +} + +# Pull docker image +pull_docker_image() +{ + docker pull $image + if [ $? = 0 ]; then + echo "finish downloading image" + else + echo "[INFO] Run command in false path ." + exit + fi +} + +# cp image package to host +cp_package() +{ + docker cp -a init_docker:/${image_name} $HOME/ + echo "result: $(ls $HOME/${image_name})" +} diff --git a/container/osimage/centos/bin/packages-to-install b/container/osimage/centos/bin/packages-to-install new file mode 100644 index 0000000..5d31c40 --- /dev/null +++ b/container/osimage/centos/bin/packages-to-install @@ -0,0 +1,2 @@ +openssh-server +openssl diff --git a/container/osimage/centos/bin/packages-to-remove b/container/osimage/centos/bin/packages-to-remove new file mode 100644 index 0000000..ce079b9 --- /dev/null +++ b/container/osimage/centos/bin/packages-to-remove @@ -0,0 +1,8 @@ +/.dockerenv +/lib/modules +/usr/share/doc +/usr/share/man +/usr/share/info +/usr/share/i18n +/usr/share/locale +/usr/share/terminfo diff --git a/container/osimage/centos/run b/container/osimage/centos/run new file mode 100755 index 0000000..47817b9 --- /dev/null +++ b/container/osimage/centos/run @@ -0,0 +1,27 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. ${CCI_SRC}/container/osimage/centos/bin/lib +. ${CCI_SRC}/container/defconfig.sh + +check_passwd_file "$root_pwd_file" + +DIR=$(dirname $(realpath $0)) +cmd=( + docker run + --name init_docker + -v $DIR/bin/:/root/bin + -v /srv/initrd/modules/:/root/modules:ro + -e ROOT_NEW_PASSWD=$ROOT_NEW_PASSWD + -e image_name=$image_name + $image + /root/bin/create-image +) + +pull_docker_image +${cmd[@]} +cp_package +docker_rm init_docker &> /dev/null +echo "build finished" + -- 2.23.0

2 2