- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci] add work dir /srv/tmp/public
by Liu Yinsi 17 Apr '21

17 Apr '21

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- sparrow/1-storage/tiny | 1 + 1 file changed, 1 insertion(+) diff --git a/sparrow/1-storage/tiny b/sparrow/1-storage/tiny index 0eb7d39..c7636b0 100755 --- a/sparrow/1-storage/tiny +++ b/sparrow/1-storage/tiny @@ -26,6 +26,7 @@ dirs=( /srv/cci/Maildir/.compass-ci/cur /srv/cci/Maildir/.compass-ci/sent /srv/rabbitmq + /srv/tmp/public ) mkdir -p "${dirs[@]}" -- 2.23.0

1 0

[PATCH compass-ci] container/assist-result: refactor get_job_stats to get_job_content
by Cao Xueliang 17 Apr '21

17 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assist-result/routes.rb | 6 +++--- .../views/{get_job_stats.rb => get_job_content.rb} | 5 ++--- lib/assist_result_client.rb | 8 ++++++++ src/delimiter/utils.rb | 7 +++++-- 4 files changed, 18 insertions(+), 8 deletions(-) rename container/assist-result/views/{get_job_stats.rb => get_job_content.rb} (67%) diff --git a/container/assist-result/routes.rb b/container/assist-result/routes.rb index 02956aa..287db25 100755 --- a/container/assist-result/routes.rb +++ b/container/assist-result/routes.rb @@ -7,7 +7,7 @@ require 'json' require 'sinatra' require_relative './views/get_job_yaml' -require_relative './views/get_job_stats' +require_relative './views/get_job_content' require_relative './views/check_job_credible' configure do @@ -36,9 +36,9 @@ post '/check_job_credible' do return [200, {'credible' => result}.to_json] end -get '/get_job_stats/:job_id' do +get '/get_job_content/:job_id' do begin - result = get_job_stats(params[:job_id]) + result = get_job_content(params[:job_id]) rescue StandardError => e return [400, e.backtrace.inspect] end diff --git a/container/assist-result/views/get_job_stats.rb b/container/assist-result/views/get_job_content.rb similarity index 67% rename from container/assist-result/views/get_job_stats.rb rename to container/assist-result/views/get_job_content.rb index d294df3..643beb5 100644 --- a/container/assist-result/views/get_job_stats.rb +++ b/container/assist-result/views/get_job_content.rb @@ -4,7 +4,6 @@ require_relative "#{ENV['CCI_SRC']}/lib/es_query" -def get_job_stats(job_id) - content = ESQuery.new.query_by_id(job_id) - return content['stats'] +def get_job_content(job_id) + return ESQuery.new.query_by_id(job_id) end diff --git a/lib/assist_result_client.rb b/lib/assist_result_client.rb index fcf5d9d..9c42b97 100644 --- a/lib/assist_result_client.rb +++ b/lib/assist_result_client.rb @@ -34,4 +34,12 @@ class AssistResult return JSON.parse(response.body) end + + def get_job_content(job_id) + resource = RestClient::Resource.new("http://#{@host}:#{@port}/get_job_content/#{job_id}") + response = resource.get() + return nil unless response.code == 200 + + return response.body + end end diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index dbc42ef..a39d9de 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -108,9 +108,12 @@ module Utils # this is a temporary solution, the extract container will be improved in future. def query_stats(job_id, times) (1..times).each do |i| - new_job = ESQuery.new.query_by_id(job_id) + job_content = AssistResult.new.get_job_content(job_id) puts "query stats times: #{i}" - return new_job['stats'] if new_job['stats'] + if job_content + job_content = JSON.parse(job_content) + return job_content['stats'] if job_content.key?('stats') + end sleep 60 end -- 2.23.0

1 0

[PATCH v2 lkp-tests] script to deploy ceph for multi server tests
by Luan Shengde 16 Apr '21

16 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/ceph-init | 155 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100755 tests/ceph-init diff --git a/tests/ceph-init b/tests/ceph-init new file mode 100755 index 000000000..59398d1bd --- /dev/null +++ b/tests/ceph-init @@ -0,0 +1,155 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# - cephuser_password +# - cephuser + +: "${cephuser:=cephuser}" +: "${cephuser_password:=cephuser_password}" + +### ===== on each node ===== +# stop firewalld and disable selinx +systemctl stop firewalld +systemctl disable firewalld + +sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/sysconfig/selinux + +until [[ -n $node_roles ]]; do + sleep 1 +done + +hosts=(node1 node2 node3) + +[[ $node_roles == node1 ]] && { + hostnamectl set-hostname node1 +} + +[[ $node_roles == node2 ]] && { + hostnamectl set-hostname node2 +} + +[[ $node_roles == node3 ]] && { + hostnamectl set-hostname node3 +} + +yum update && { + yum install ntp wget gcc make ceph lvm2 python3 -y +} + +# add user and set sudo permissions +useradd -d /home/$cephuser -m $cephuser && { + echo $cephuser:$cephuser_password | chpasswd +} + +cat >> /etc/sudoers <<-EOF +$cephuser ALL=(ALL) NOPASSWD:ALL +EOF + +# check and install pip +[[ $(command -v pip) ]] || { + until wget https://bootstrap.pypa.io/get-pip.py; do + sleep 2 + done + python get-pip.py +} + +pip install prettytable + +### ===== on ceph-deploy node ===== +# get node IPs is first +[[ $node_roles == node1 ]] || exit 0 + +until [[ -n $node1 ]]; do + sleep 1 +done + +until [[ -n $node2 ]]; do + sleep 1 +done + +until [[ -n $node3 ]]; do + sleep 1 +done + +# config hosts and set default ssh users +cat >> /etc/hosts <<-EOF +$node1 node1 node1 +$node2 node2 node2 +$node3 node3 node3 +EOF + +for host in "${hosts[@]}"; do + cat >> "$HOME/.ssh/config" <<-EOF + Host $host + Hostname $host + User $cephuser + EOF +done + +chmod 0600 "$HOME/.ssh/config" + +network=$(ip route | grep default | awk '{print $3}' | awk -F'.' '{print $1"."$2}') +net_mask=$(ip route | grep "^$network" | awk '{print $1}') + +# do the deploy until all the servers are up +# for host in "${host[@]}"; do +# until ping -c 1 -W 1 172.168.131.100> /dev/null; do +# sleep 2 +# done +# done + +# set server trust +[[ $(command -v sshpass) ]] || { + until wget http://sourceforge.net/projects/sshpass/files/latest/download -O sshpass.tar.gz; do + sleep 2 + done +} + +tar xzvf sshpass.tar.gz && { + cd sshpass-1.08 || exit 1 + ./configure + make + make install +} + +rm -rf "$HOME/.ssh/id_rsa" "$HOME/.ssh/id_rsa.pub" && { + ssh-keygen -f "$HOME/.ssh/id_rsa" -t rsa -N '' +} + +sed -i '/StrictHostKeyChecking/c StrictHostKeyChecking no' /etc/ssh/ssh_config +for host in "${hosts[@]}"; do +{ + sshpass -p $cephuser_password ssh-copy-id "$cephuser"@"$host" &> /dev/null +}& +done +wait + +# install +pip install ceph-deploy + +pip_dir=$(pip show ceph-deploy | awk '/^Location/ {print $NF}') +ceph_init_file="${pip_dir}/ceph_deploy/hosts/__init__.py" +[[ -f $ceph_init_file ]]&& { + sed -i "/distributions = {/a 'openeuler': fedora," "$ceph_init_file" +} + +# deploy ceph cluster +disk_name=$(sed -e '1d' -e '/^$/d' /proc/partitions | grep " .da" | awk '{print $NF}') +disk_device="/dev/${disk_name}" + +[[ -d "$HOME/ceph-deploy" ]] || { + mkdir -p "$HOME/ceph-deploy" +} + +cd "$HOME/ceph-deploy" && { + ceph-deploy new "${hosts[@]}" + echo "public_network = ${net_mask}" >> "$HOME/ceph-deploy/ceph.conf" + ceph-deploy mon create-initial + ceph-deploy --overwrite-conf admin "${hosts[@]}" + ceph-deploy mgr create "${hosts[@]}" + for host in "${hosts[@]}"; do + ceph-deploy osd create --data "$disk_device" "$host" + done + wait + ceph osd pool create ceph-osd-pool 128 +} -- 2.23.0

2 2

[PATCH v2 compass-ci] delimiter: consume bisect process job by the bad job tbox_group
by Cao Xueliang 16 Apr '21

16 Apr '21

Sometimes we set a queue in a job, then we delete the testbox who can consume the queue, if we submit the job to the queue, no testbox can consume it, so we consume the bisect process job by the bad job tbox_group. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter/utils.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index dbc42ef..1aac6b3 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -137,9 +137,10 @@ module Utils job['my_email'] = ENV['my_email'] job['my_token'] = ENV['secrets_my_token'] job['bad_job_id'] = job_id - job['queue'] = "#{job['testbox'].split('.')[0]}-bisect" + job['testbox'] = job['tbox_group'] job.delete('id') + job.delete('queue') return job end -- 2.23.0

1 0

[PATCH v1 compass-ci 1/2] delimiter: consume job by the bad job queue
by Cao Xueliang 16 Apr '21

16 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter/utils.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index dbc42ef..a0d4196 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -137,7 +137,6 @@ module Utils job['my_email'] = ENV['my_email'] job['my_token'] = ENV['secrets_my_token'] job['bad_job_id'] = job_id - job['queue'] = "#{job['testbox'].split('.')[0]}-bisect" job.delete('id') -- 2.23.0

1 1

[PATCH v1 compass-ci 2/2] container/etcd: fix Error reading socket: Connection reset by peer
by Cao Xueliang 16 Apr '21

16 Apr '21

We should pass the start script params to /bin/etcd program. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/etcd/entry.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/container/etcd/entry.sh b/container/etcd/entry.sh index 4bfea1e..16c8d00 100755 --- a/container/etcd/entry.sh +++ b/container/etcd/entry.sh @@ -3,6 +3,6 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. crond -/bin/etcd -exec "$@" +/bin/etcd "$@" +exec -- 2.23.0

1 0

[PATCH compass-ci 4/4] sparrow: expose ASSISTANT_PORT/HOST in scheduler
by Cao Xueliang 16 Apr '21

16 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- sparrow/3-code/dev-env | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index bb576c7..76686a5 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -28,6 +28,8 @@ server_name=$(hostname | cut -f1 -d.) : ${INITRD_HTTP_PORT:=8800} : ${assist_result_host:=$server_ip} : ${ASSIST_RESULT_PORT:=8102} +: ${assistant_host:=$server_ip} +: ${ASSISTANT_PORT:=8101} : ${result_webdav_host:=$server_ip} : ${RESULT_WEBDAV_PORT:=3080} : ${MASTER_FLUENTD_HOST:=$server_ip} @@ -107,6 +109,8 @@ INITRD_HTTP_HOST: $initrd_http_host INITRD_HTTP_PORT: $INITRD_HTTP_PORT ASSIST_RESULT_HOST: $assist_result_host ASSIST_RESULT_PORT: $ASSIST_RESULT_PORT +ASSISTANT_HOST: $assistant_host +ASSISTANT_PORT: $ASSISTANT_PORT RESULT_WEBDAV_HOST: $result_webdav_host RESULT_WEBDAV_PORT: $RESULT_WEBDAV_PORT MASTER_FLUENTD_HOST: $MASTER_FLUENTD_HOST @@ -149,6 +153,8 @@ INITRD_HTTP_HOST: INITRD_HTTP_PORT: ASSIST_RESULT_HOST: ASSIST_RESULT_PORT: +ASSISTANT_HOST: +ASSISTANT_PORT: RESULT_WEBDAV_HOST: RESULT_WEBDAV_PORT: DOCKER_REGISTRY_HOST: -- 2.23.0

1 0

[PATCH compass-ci 3/4] lib: add assistant_client and use it in mail_bisect_result
by Cao Xueliang 16 Apr '21

16 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/assistant_client.rb | 24 ++++++++++++++++++++++++ lib/constants.rb | 3 +++ lib/mail_bisect_result.rb | 4 ++-- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 lib/assistant_client.rb diff --git a/lib/assistant_client.rb b/lib/assistant_client.rb new file mode 100644 index 0000000..c951f2b --- /dev/null +++ b/lib/assistant_client.rb @@ -0,0 +1,24 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'base64' +require 'rest-client' +require_relative 'constants' + +class AssistantClient + def initialize(host = ASSISTANT_HOST, port = ASSISTANT_PORT) + @host = host + @port = port + end + + def get_mail_list(type) + resource = RestClient::Resource.new("http://#{@host}:#{@port}/get_mail_list/#{type}") + response = resource.get() + return nil unless response.code == 200 + + return JSON.parse(response.body) + end +end diff --git a/lib/constants.rb b/lib/constants.rb index 03b1e40..11927f9 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -20,6 +20,9 @@ SRV_HTTP_GIT_PORT ||= config['SRV_HTTP_GIT_PORT'] || ENV['SRV_HTTP_GIT_PORT'] || SRV_HTTP_DOMAIN ||= config['SRV_HTTP_DOMAIN'] || ENV['SRV_HTTP_DOMAIN'] || 'api.compass-ci.openeuler.org' +ASSISTANT_HOST ||= config['ASSISTANT_HOST'] || ENV['ASSISTANT_HOST'] || '172.17.0.1' +ASSISTANT_PORT ||= config['ASSISTANT_PORT'] || ENV['ASSISTANT_PORT'] || 8101 + ASSIST_RESULT_HOST ||= config['ASSIST_RESULT_HOST'] || ENV['ASSIST_RESULT_HOST'] || '172.17.0.1' ASSIST_RESULT_PORT ||= config['ASSIST_RESULT_PORT'] || ENV['ASSIST_RESULT_PORT'] || 8102 diff --git a/lib/mail_bisect_result.rb b/lib/mail_bisect_result.rb index 782c702..5a71225 100644 --- a/lib/mail_bisect_result.rb +++ b/lib/mail_bisect_result.rb @@ -6,7 +6,7 @@ require_relative 'git' require_relative 'es_query' require_relative 'constants' require_relative 'mail_client' -require_relative 'parse_mail_list' +require_relative 'assistant_client' require_relative 'assign_account_client' # compose and send email for bisect result @@ -25,7 +25,7 @@ class MailBisectResult end def parse_mail_info - mail_hash = parse_mail_list('delimiter') + mail_hash = AssistantClient.new.get_mail_list('delimiter') @to = mail_hash['to'] if mail_hash.key?('to') @bcc = mail_hash['bcc'] if mail_hash.key?('bcc') raise 'Need to add bcc email for bisect report.' unless @bcc -- 2.23.0

1 0

[PATCH compass-ci 2/4] container/assistant: refactor assistant container
by Cao Xueliang 16 Apr '21

16 Apr '21

1. Refactor assistant container. 2. Add get_mail_list function. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assistant/Dockerfile | 17 ++++---- container/assistant/build | 4 +- container/assistant/routes.rb | 20 ++++++--- container/assistant/start | 51 +++++++++++++++------- container/assistant/views/get_mail_list.rb | 9 ++++ 5 files changed, 69 insertions(+), 32 deletions(-) create mode 100644 container/assistant/views/get_mail_list.rb diff --git a/container/assistant/Dockerfile b/container/assistant/Dockerfile index 04d2da0..3299858 100644 --- a/container/assistant/Dockerfile +++ b/container/assistant/Dockerfile @@ -1,16 +1,15 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM debian +FROM alpine:3.11 MAINTAINER Cao Xueliang <caoxl78320(a)163.com> -ENV DEBIAN_FRONTEND noninteractive +RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories && \ + apk update && \ + apk add --no-cache 'ruby-dev' 'ruby-etc' \ + 'g++' 'gcc' 'pcre' 'libevent' 'make' 'git' 'bash' 'grep' 'coreutils' 'curl' 'util-linux' -RUN apt-get update && \ - apt-get install -y ruby-json ruby-sinatra curl - -COPY routes.rb /usr/local/bin/ -COPY views /usr/local/bin/views/ - -CMD ["/usr/local/bin/routes.rb"] +RUN umask 002 && \ + gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ + gem install rest-client activesupport json yaml elasticsearch sinatra puma terminal-table diff --git a/container/assistant/build b/container/assistant/build index 89a8a97..6f739d6 100755 --- a/container/assistant/build +++ b/container/assistant/build @@ -4,6 +4,6 @@ . ../defconfig.sh -docker_skip_rebuild "debian:assistant" +docker_skip_rebuild "assistant" -docker build -t debian:assistant . +docker build -t assistant . diff --git a/container/assistant/routes.rb b/container/assistant/routes.rb index 54d3670..232ec8c 100755 --- a/container/assistant/routes.rb +++ b/container/assistant/routes.rb @@ -8,24 +8,32 @@ require 'json' require 'open3' require_relative './views/locate_files' +require_relative './views/get_mail_list' -set :bind, '0.0.0.0' -set :port, 8101 +configure do + set :bind, '0.0.0.0' + set :port, ENV['ASSISTANT_PORT'] +end post '/locate_files' do request.body.rewind begin data = JSON.parse request.body.read - rescue JSON::ParserError - return [400, 'parse json params error'] + result = locate_files(data) + rescue StandardError => e + return [400, e.backtrace.inspect] end + [200, result.to_json] +end + +get '/get_mail_list/:type' do begin - result = locate_files(data) + result = get_mail_list(params[:type]) rescue StandardError => e return [400, e.backtrace.inspect] end - [200, result.to_json] + return [200, result.to_json] end diff --git a/container/assistant/start b/container/assistant/start index 5f7822e..87c1801 100755 --- a/container/assistant/start +++ b/container/assistant/start @@ -1,21 +1,42 @@ -#!/bin/bash +#!/usr/bin/env ruby # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true -. $CCI_SRC/container/defconfig.sh +require 'set' +require_relative '../defconfig.rb' -docker_rm assistant +names = Set.new %w[ + ASSISTANT_HOST + ASSISTANT_PORT +] -cmd=( - docker run - --restart=always - --name assistant - -u nobody - -d - -p 8101:8101 - -v /srv/initrd:/srv/initrd - -v /etc/localtime:/etc/localtime:ro - debian:assistant -) +defaults = relevant_defaults(names) +env = docker_env(defaults) -"${cmd[@]}" +DEFAULT_LKP = '/c/lkp-tests' +DEFAULT_CCI = '/c/compass-ci' +DEFAULT_CONFIG_DIR = '/etc/compass-ci' +ASSISTANT_PORT = defaults['ASSISTANT_PORT'] +docker_rm 'assistant' + +cmd = %w[ + docker run + --name assistant + --restart=always + -d +] + env + %W[ + -e LKP_SRC=#{DEFAULT_LKP} + -e CCI_SRC=#{DEFAULT_CCI} + -p #{ASSISTANT_PORT}:#{ASSISTANT_PORT} + -v #{ENV['LKP_SRC']}:#{DEFAULT_LKP} + -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} + -v #{DEFAULT_CONFIG_DIR}:#{DEFAULT_CONFIG_DIR}:ro + -v /etc/localtime:/etc/localtime:ro + -w #{DEFAULT_CCI}/container/assistant/ + assistant +] + +cmd += ['sh', '-c', 'umask 002 && ruby ./routes.rb'] + +system(*cmd) diff --git a/container/assistant/views/get_mail_list.rb b/container/assistant/views/get_mail_list.rb new file mode 100644 index 0000000..66a1872 --- /dev/null +++ b/container/assistant/views/get_mail_list.rb @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative "#{ENV['CCI_SRC']}/lib/parse_mail_list" + +def get_mail_list(type) + return parse_mail_list(type) +end -- 2.23.0

1 0

[PATCH compass-ci 1/4] delimiter: consume job by the bad job queue
by Cao Xueliang 16 Apr '21

16 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter/utils.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index dbc42ef..a0d4196 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -137,7 +137,6 @@ module Utils job['my_email'] = ENV['my_email'] job['my_token'] = ENV['secrets_my_token'] job['bad_job_id'] = job_id - job['queue'] = "#{job['testbox'].split('.')[0]}-bisect" job.delete('id') -- 2.23.0

1 0