- Compass-ci - mailweb.openeuler.org

[PATCH v4 compass-ci] feat: build centos initramfs image with one command
by Wang Chenglong 12 Oct '20

12 Oct '20

Use the tool to create a new centos-$version initramfs image with one command. and we also can customize the pre-installed software by configuring ./bin/packages-to-install. --- container/osimage/centos/README.md | 15 +++++++ container/osimage/centos/bin/create-image | 38 +++++++++++++++++ container/osimage/centos/bin/lib | 41 +++++++++++++++++++ .../osimage/centos/bin/packages-to-install | 2 + .../osimage/centos/bin/packages-to-remove | 8 ++++ container/osimage/centos/run | 27 ++++++++++++ 6 files changed, 131 insertions(+) create mode 100644 container/osimage/centos/README.md create mode 100755 container/osimage/centos/bin/create-image create mode 100755 container/osimage/centos/bin/lib create mode 100644 container/osimage/centos/bin/packages-to-install create mode 100644 container/osimage/centos/bin/packages-to-remove create mode 100755 container/osimage/centos/run diff --git a/container/osimage/centos/README.md b/container/osimage/centos/README.md new file mode 100644 index 0000000..000ca05 --- /dev/null +++ b/container/osimage/centos/README.md @@ -0,0 +1,15 @@ +# Use the tool to create a new centos-${version} initramfs image. + +Usage: + cd ${CCI_SRC}/rootfs/initramfs/centos/aarch64/${version}/ + ./build + +Some configuration items: +./bin/packages-to-install + If you want to pre-install the software, you can write the package names in ./bin/packages-to-install. + +./bin/packages-to-remove + If you want remove some unnecessary files, you can write the names in ./bin/packages-to-remove. + +$HOME/.config/compass-ci/rootfs.passwd + Set the password for the image into this file. diff --git a/container/osimage/centos/bin/create-image b/container/osimage/centos/bin/create-image new file mode 100755 index 0000000..214204f --- /dev/null +++ b/container/osimage/centos/bin/create-image @@ -0,0 +1,38 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +# Configure rootfs +make_rootfs() +{ + yum repolist + yum -y install --skip-broken $(</root/bin/packages-to-install) + yum clean all + rm -rf $(</root/bin/packages-to-remove) + ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + cd /root/modules + zcat modules-5.8.0.1.cgz | cpio -idm +} + +# Config password +pre_config_rootfs() { + [ -n "$ROOT_NEW_PASSWD" ] && { + echo "Changing root password" + passwd_md5=$(openssl passwd -1 "$ROOT_NEW_PASSWD") + sed -i -r "s/^root:[^:]*:(.*)/root:${passwd_md5//\//\\/}:\1/" "$ROOTFS_DIR/etc/shadow" # Change the password in shadow + sed -i 's/[# ]PermitRootLogin.*/PermitRootLogin yes/' "$ROOTFS_DIR/etc/ssh/sshd_config" # Configure ssh service + } +} + +# Pack cgz package +pack_cgz() +{ + echo "Packing package. Please wait." + cd / + find ./ ! -path "./${image_name}" ! -path "./root/modules/modules-5.8.0.1.cgz" | cpio -o -Hnewc | gzip -9 > /${image_name} + chmod 644 /${image_name} +} + +make_rootfs +pre_config_rootfs +pack_cgz diff --git a/container/osimage/centos/bin/lib b/container/osimage/centos/bin/lib new file mode 100755 index 0000000..b4a59e2 --- /dev/null +++ b/container/osimage/centos/bin/lib @@ -0,0 +1,41 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +DIR="$(pwd)" +name="$(echo $DIR |awk -F "/" '{print $(NF - 2)}')" +version="$(echo $DIR |awk -F "/" '{print $NF}')" +image="${name}:${version}" +image_name="${name}-${version}-$(date +"%Y%m%d").cgz" + +# Check password file +check_passwd_file() { + root_pwd_file="$HOME/.config/compass-ci/rootfs.passwd" + export ROOT_NEW_PASSWD= + [ -f "$root_pwd_file" ] || { + echo "[INFO] Please set the password file." + echo "$HOME/.config/compass-ci/rootfs.passwd" + exit 1 + } + + export ROOT_NEW_PASSWD=$(cat "$root_pwd_file") +} + +# Pull docker image +pull_docker_image() +{ + docker pull $image + if [ $? = 0 ]; then + echo "finish downloading image" + else + echo "[INFO] Run command in false path ." + exit + fi +} + +# cp image package to host +cp_package() +{ + docker cp -a init_docker:/${image_name} $HOME/ + echo "result: $(ls $HOME/${image_name})" +} diff --git a/container/osimage/centos/bin/packages-to-install b/container/osimage/centos/bin/packages-to-install new file mode 100644 index 0000000..5d31c40 --- /dev/null +++ b/container/osimage/centos/bin/packages-to-install @@ -0,0 +1,2 @@ +openssh-server +openssl diff --git a/container/osimage/centos/bin/packages-to-remove b/container/osimage/centos/bin/packages-to-remove new file mode 100644 index 0000000..ce079b9 --- /dev/null +++ b/container/osimage/centos/bin/packages-to-remove @@ -0,0 +1,8 @@ +/.dockerenv +/lib/modules +/usr/share/doc +/usr/share/man +/usr/share/info +/usr/share/i18n +/usr/share/locale +/usr/share/terminfo diff --git a/container/osimage/centos/run b/container/osimage/centos/run new file mode 100755 index 0000000..47817b9 --- /dev/null +++ b/container/osimage/centos/run @@ -0,0 +1,27 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. ${CCI_SRC}/container/osimage/centos/bin/lib +. ${CCI_SRC}/container/defconfig.sh + +check_passwd_file "$root_pwd_file" + +DIR=$(dirname $(realpath $0)) +cmd=( + docker run + --name init_docker + -v $DIR/bin/:/root/bin + -v /srv/initrd/modules/:/root/modules:ro + -e ROOT_NEW_PASSWD=$ROOT_NEW_PASSWD + -e image_name=$image_name + $image + /root/bin/create-image +) + +pull_docker_image +${cmd[@]} +cp_package +docker_rm init_docker &> /dev/null +echo "build finished" + -- 2.23.0

2 2

[PATCH v5 compass-ci 6/6] container/mail-robot: mail-robot.md
by Luan Shengde 12 Oct '20

12 Oct '20

guidance file for mail-robot Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- doc/mail-robot.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 doc/mail-robot.md diff --git a/doc/mail-robot.md b/doc/mail-robot.md new file mode 100644 index 0000000..e248b41 --- /dev/null +++ b/doc/mail-robot.md @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ + +# email monitor robot + +## purpose + +monitor received mails + check if the mail is for applying uuid/account + +## steps overview + +1. monitor mailbox dir + - check for new email file + +2. check email + - check email subject + - return if unmatched subject + - call answerback-email + - check email available + - check base_url for the oss_url in upstream-repos list + if not, it will raise "url not in upstream-repos list" error + - check if the email has commit for the url + - if gitee, git clone the repo and check + if non-gitee, curl url to check + - if not, raise "no commit found" error + - check pub_key + - check if there is a pub_key in the email + if not, it will raise 'no pub_key' error + - apply ssh account + - apply account uuid + - write account info to es + - send uuid to user + - mv mail file to drafts + +3. continue the monitor + - after 1 and 2, continue to monitor the mailbox + +4. service log + - use the following command to get the logs + docker logs -f --tail=100 fluentd |grep mail-robot -- 2.23.0

1 0

[PATCH v5 compass-ci 1/6] container/mail-robot: mail-robot.rb
by Luan Shengde 12 Oct '20

12 Oct '20

mail robot for application email monitor mailbox check new added email file in new check new mail's subject return if unmatched subject call answerback-email apply account apply uuid write account/uuid etc. to es send uuid to user Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/mail-robot/mail-robot.rb | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100755 container/mail-robot/mail-robot.rb diff --git a/container/mail-robot/mail-robot.rb b/container/mail-robot/mail-robot.rb new file mode 100755 index 0000000..2de0ac4 --- /dev/null +++ b/container/mail-robot/mail-robot.rb @@ -0,0 +1,41 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'yaml' +require 'listen' +require 'mail' +require 'fileutils' +require_relative 'answerback-email.rb' + +mail_inbox = '/Maildir/.compass-ci/new/' +mail_drafts = '/Maildir/.compass-ci/cur/' + +def monitor_dir(mail_inbox, mail_drafts) + listener = Listen.to(mail_inbox) do |_modified, added, _removed| + unless added.empty? + added.each do |mail_file| + begin + check_to_send_account(mail_file, mail_drafts) + rescue StandardError => e + puts e.message + end + end + end + end + listener.start + sleep +end + +def check_to_send_account(mail_file, mail_drafts) + mail_content = Mail.read(mail_file) + subject = mail_content.subject + return unless subject =~ /apply account/i + + send_account(mail_content) + FileUtils.mv(mail_file, mail_drafts) +end + +monitor_dir(mail_inbox, mail_drafts) -- 2.23.0

1 0

[PATCH v5 compass-ci 5/6] container/mail-robot: start file
by Luan Shengde 12 Oct '20

12 Oct '20

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/mail-robot/start | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100755 container/mail-robot/start diff --git a/container/mail-robot/start b/container/mail-robot/start new file mode 100755 index 0000000..a2e183d --- /dev/null +++ b/container/mail-robot/start @@ -0,0 +1,29 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative '../defconfig.rb' +docker_rm 'mail-robot' + +JUMPER_IP = '192.168.132.2' +JUMPER_PORT = '29999' +SEND_MAIL_PORT = '11311' + +cmd = %W[ + docker run + --restart=always + --name=mail-robot + -d + -e JUMPER_IP=#{JUMPER_IP} + -e JUMPER_PORT=#{JUMPER_PORT} + -e SEND_MAIL_PORT=#{SEND_MAIL_PORT} + -v /c/upstream-repos:/upstream-repos:ro + -v /srv/cci/Maildir:/Maildir:rw + -v /c/compass-ci:/compass-ci:ro + mail-robot +] + +cmd += ['bash', '-c', 'umask 002 && ruby ./mail-robot.rb'] + +system(*cmd) -- 2.23.0

1 0

[PATCH compass-ci] feat: a link for one command to build centos initramfs image
by Wang Chenglong 12 Oct '20

12 Oct '20

This link is used to one command to build centos-$version initramfs image Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- rootfs/initramfs/centos/aarch64/7/build | 1 + 1 file changed, 1 insertion(+) create mode 120000 rootfs/initramfs/centos/aarch64/7/build diff --git a/rootfs/initramfs/centos/aarch64/7/build b/rootfs/initramfs/centos/aarch64/7/build new file mode 120000 index 0000000..2d22759 --- /dev/null +++ b/rootfs/initramfs/centos/aarch64/7/build @@ -0,0 +1 @@ +../../../../../container/osimage/centos/run \ No newline at end of file -- 2.23.0

1 0

[PATCH compass-ci 2/2] container/srv-http allow access file with level 4 path
by Lu Weitao 12 Oct '20

12 Oct '20

problem: can not access file with level 4 path like: http://localhost:11300/result/unixbench/crystal.82998/boot-time.json cause analysis: we had forbidden access level 4 catalog and file solution: keep forbidden access level 4 catalog, allow access file with level 4 path. Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/srv-http/root/etc/nginx/conf.d/default.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/srv-http/root/etc/nginx/conf.d/default.conf b/container/srv-http/root/etc/nginx/conf.d/default.conf index 403b0dc..9c15d99 100644 --- a/container/srv-http/root/etc/nginx/conf.d/default.conf +++ b/container/srv-http/root/etc/nginx/conf.d/default.conf @@ -22,7 +22,7 @@ server { } } - location ~* ^/result/(([^/]+)?/?|[^/]+/[^/]+/[^/]+/?)$ { + location ~* ^/result/(([^/]+)?/?|[^/]+/[^/]+/[^/]+/)$ { deny all; } } -- 2.23.0

1 0

[PATCH compass-ci 1/2] container/srv-http fix Chinese garbled
by Lu Weitao 12 Oct '20

12 Oct '20

problem: Chinese file name and content display garbled characters Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/srv-http/root/etc/nginx/conf.d/default.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/container/srv-http/root/etc/nginx/conf.d/default.conf b/container/srv-http/root/etc/nginx/conf.d/default.conf index 2bf55b6..403b0dc 100644 --- a/container/srv-http/root/etc/nginx/conf.d/default.conf +++ b/container/srv-http/root/etc/nginx/conf.d/default.conf @@ -6,6 +6,8 @@ server { root /usr/share/nginx/html; index index.html; + charset utf-8,gbk; + location /favicon.ico { log_not_found off; } -- 2.23.0

1 0

[PATCH compass-ci] fix: scheduler: update_id need set_initrds_uri
by Tong Qunfeng 12 Oct '20

12 Oct '20

Signed-off-by: Tong Qunfeng <taxcom(a)tom.com> --- src/lib/job.cr | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index bf959c2..504bc19 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -382,9 +382,13 @@ class Job def update_id(id) @hash["id"] = JSON::Any.new(id) - # "result_root" is based on "id" + # "result_root" => "/result/#{suite}/#{tbox_group}/#{date}/#{id}" + # set_initrds_uri -> get_initrds -> common_initrds => ".../#{id}/job.cgz" + # + # "result_root, common_initrds" is associate with "id" # so when update id, we need redo set_ set_result_root() + set_initrds_uri() end def get_uuid_tag -- 2.23.0

1 0

[PATCH v4 compass-ci] feat: build centos-7 initramfs image with one command
by Wang Chenglong 12 Oct '20

12 Oct '20

Use the tool to create a new centos-7 initramfs image with one command. and we also can customize the pre-installed software by configuring ./bin/install-list. --- container/osimage/centos-7/README.md | 13 +++++++ container/osimage/centos-7/bin/create-image | 38 +++++++++++++++++++++ container/osimage/centos-7/bin/delete-list | 8 +++++ container/osimage/centos-7/bin/install-list | 2 ++ container/osimage/centos-7/bin/lib | 30 ++++++++++++++++ container/osimage/centos-7/build | 29 ++++++++++++++++ 6 files changed, 120 insertions(+) create mode 100644 container/osimage/centos-7/README.md create mode 100755 container/osimage/centos-7/bin/create-image create mode 100644 container/osimage/centos-7/bin/delete-list create mode 100644 container/osimage/centos-7/bin/install-list create mode 100755 container/osimage/centos-7/bin/lib create mode 100755 container/osimage/centos-7/build diff --git a/container/osimage/centos-7/README.md b/container/osimage/centos-7/README.md new file mode 100644 index 0000000..2c2f956 --- /dev/null +++ b/container/osimage/centos-7/README.md @@ -0,0 +1,13 @@ +# Use the tool to create a new centos-7 initramfs image. + +Usage: + cd ${CCI_SRC/rootfs/initramfs/centos/aarch64/7} + ./build + +Some configuration items: +./bin/install-list + If you want to pre-install the software, you can write the package names in ./bin/install-list. + +./bin/delete-list + If you want remove some unnecessary files, you can write the names in ./bin/delete-list + diff --git a/container/osimage/centos-7/bin/create-image b/container/osimage/centos-7/bin/create-image new file mode 100755 index 0000000..ad51986 --- /dev/null +++ b/container/osimage/centos-7/bin/create-image @@ -0,0 +1,38 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +# Configure rootfs +make_rootfs() +{ + yum repolist + yum -y install --skip-broken $(</root/bin/install-list) + yum clean all + rm -rf $(</root/bin/delete-list) + ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + cd /root/modules + zcat modules-5.8.0.1.cgz | cpio -idm +} + +# Config password +pre_config_rootfs() { + [ -n "$ROOT_NEW_PASSWD" ] && { + echo "Changing root password" + passwd_md5=$(openssl passwd -1 "$ROOT_NEW_PASSWD") + sed -i -r "s/^root:[^:]*:(.*)/root:${passwd_md5//\//\\/}:\1/" "$ROOTFS_DIR/etc/shadow" # Change the password in shadow + sed -i 's/[# ]PermitRootLogin.*/PermitRootLogin yes/' "$ROOTFS_DIR/etc/ssh/sshd_config" # Configure ssh service + } +} + +# Pack cgz package +pack_cgz() +{ + echo "Packing package. Please wait." + cd / + find ./ ! -path "./${image_name}" ! -path "./root/modules/modules-5.8.0.1.cgz" | cpio -o -Hnewc | gzip -9 > /${image_name} + chmod 644 /${image_name} +} + +make_rootfs +pre_config_rootfs +pack_cgz diff --git a/container/osimage/centos-7/bin/delete-list b/container/osimage/centos-7/bin/delete-list new file mode 100644 index 0000000..ce079b9 --- /dev/null +++ b/container/osimage/centos-7/bin/delete-list @@ -0,0 +1,8 @@ +/.dockerenv +/lib/modules +/usr/share/doc +/usr/share/man +/usr/share/info +/usr/share/i18n +/usr/share/locale +/usr/share/terminfo diff --git a/container/osimage/centos-7/bin/install-list b/container/osimage/centos-7/bin/install-list new file mode 100644 index 0000000..5d31c40 --- /dev/null +++ b/container/osimage/centos-7/bin/install-list @@ -0,0 +1,2 @@ +openssh-server +openssl diff --git a/container/osimage/centos-7/bin/lib b/container/osimage/centos-7/bin/lib new file mode 100755 index 0000000..c3c470f --- /dev/null +++ b/container/osimage/centos-7/bin/lib @@ -0,0 +1,30 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +DIR="$(basename $(dirname $(realpath $0)))" +name="$(echo $DIR |awk -F "-" '{print $1}')" +version="$(echo $DIR |awk -F "-" '{print $2}')" +image="${name}:${version}" +image_name="${name}-${version}-$(date +"%Y%m%d").cgz" + +# Check password file +check_passwd_file() { + root_pwd_file="$HOME/.config/compass-ci/rootfs.passwd" + export ROOT_NEW_PASSWD= + [ -f "$root_pwd_file" ] || { + echo "[INFO] Please set the password file." + echo "$HOME/.config/compass-ci/rootfs.passwd" + exit 1 + } + + export ROOT_NEW_PASSWD=$(cat "$root_pwd_file") +} + +# Pull docker image +pull_docker() +{ + docker pull $image || return + echo "finish downloading image" +} + diff --git a/container/osimage/centos-7/build b/container/osimage/centos-7/build new file mode 100755 index 0000000..5e46306 --- /dev/null +++ b/container/osimage/centos-7/build @@ -0,0 +1,29 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. ${CCI_SRC}/container/osimage/centos-7/bin/lib +. ${CCI_SRC}/container/defconfig.sh + +check_passwd_file "$root_pwd_file" + +DIR=$(dirname $(realpath $0)) +cmd=( + docker run + --name init_docker + -v $DIR/bin/:/root/bin + -v /srv/initrd/modules/:/root/modules:ro + -e ROOT_NEW_PASSWD=$ROOT_NEW_PASSWD + -e image_name=$image_name + $image + /root/bin/create-image +) + +pull_docker + +docker_rm init_docker +${cmd[@]} + +docker cp -a init_docker:/${image_name} ./ +echo "build finished!!!" + -- 2.23.0

2 4

[PATCH lkp-tests] /lkp-tests/jobs: add test-tools.yaml
by Li Ping 12 Oct '20

12 Oct '20

--- jobs/test-tools.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 jobs/test-tools.yaml diff --git a/jobs/test-tools.yaml b/jobs/test-tools.yaml new file mode 100644 index 00000000..43ca1343 --- /dev/null +++ b/jobs/test-tools.yaml @@ -0,0 +1,8 @@ +suite: test-tools +category: functional +os_mount: initramfs + +test-tools: + add: 1 + testsuite: smoke-testing + testcase: oe_test_dd_001 -- 2.23.0

1 0