- Compass-ci - mailweb.openeuler.org

[PATCH v8 lkp-tests] sshd: send emails to login services for users
by Zhang Yale 16 Oct '20

16 Oct '20

Provide a login port for user by email, determine whether to send the ssh_port message based on the uuid. Signed-off-by: Zhang Yale <zhangyale3(a)huawei.com> --- daemon/sshd | 89 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/daemon/sshd b/daemon/sshd index 98f31d77..1f7c4126 100755 --- a/daemon/sshd +++ b/daemon/sshd @@ -1,6 +1,16 @@ #!/bin/sh # pub_key # email +# uuid +# ssh_port_len +# ssh_port_base + +. $LKP_SRC/lib/http.sh + +sshr_ip=$LKP_SERVER +sshr_port=5050 +[ -n "$sshr_port_len" ] || sshr_port_len=2000 +[ -n "$sshr_port_base" ] || sshr_port_base=30000 run_ssh() { @@ -11,40 +21,79 @@ run_ssh() systemctl start sshd } -compose_email() +data_success() { - deadline=$(date -d "+$runtime seconds" +"%Y-%m-%d %H:%M:%S") data="To: $email -Subject: [NOTIFY compass-ci] $testbox ready to ssh +Subject: [NOTIFY Compass-ci] $testbox ready to use + +Dear $user_name: + Thanks for your participation in software ecosystem! + According to your application, $testbox has been provisioned. + The datails are as follows: -Dear $email: - Thanks for your participation in Kunpeng and software ecosystem! - According to your application, $testbox has been provisioned. - The datails are as follows: + login: + ssh root(a)api.compass-ci.openeuler.org -p $port + Due time: + $deadline + HW: + nr_cpu: $nr_cpu + memory: $memory + testbox: $testbox + OS: + $os $os_version $os_arch - Login: - ssh root@$PUB_IP - Due time: - $deadline +Regards +Compass-ci +" +} - HW: - nr_cpu: $nr_cpu - memory: $memory - testbox: $testbox - OS: - $os $os_version $os_arch +data_failure() +{ + data="To: $email +Subject: [NOTIFY Compass-ci] Applying $testbox failed + +Dear $user_name: + Sorry to inform you that your application failed. + You may need to wait for a while and then try again. Regards -compass-ci +Compass-ci " } +compose_email() +{ + deadline=$(date -d "+$runtime seconds" +"%Y-%m-%d %H:%M:%S") + if [ -n "$port" ]; then + data_success + else + data_failure + fi +} + +get_port() +{ + [ -n "$uuid" ] || return + for i in $(seq 1 10) + do + port=$(($(date +%s%N)%"$sshr_port_len"+"$sshr_port_base")) + ssh -o StrictHostKeyChecking=no -o ExitOnForwardFailure=yes -o TCPKeepAlive=yes \ + -Nf -R $port:localhost:22 sshr@"$sshr_ip" -p "$sshr_port" &>/dev/null + [ $? -eq 0 ] || continue + return + done + port="" +} + run_email() { - [ -n "$email" ] || return 0 + [ -n "$email" ] && [ -n "$uuid" ] || return 0 + user_name=$(echo "$email" | awk -F '@' '{print $1}') compose_email - curl -XPOST "$MAIL_HOST:$MAIL_PORT/send_mail_text" -d "$data" + curl -XPOST "${MAIL_HOST:-$LKP_SERVER}:${MAIL_PORT:-11312}/send_mail_text" -d "$data" } run_ssh +get_port +report_ssh_port $port run_email -- 2.23.0

2 2

[PATCH compass-ci 4/4] container/send-mail: update start
by Luan Shengde 16 Oct '20

16 Oct '20

add setup default variables update heredoc Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/send-mail/start | 74 +++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/container/send-mail/start b/container/send-mail/start index 80be86a..e287322 100755 --- a/container/send-mail/start +++ b/container/send-mail/start @@ -1,29 +1,53 @@ -#!/bin/bash +#!/usr/bin/env ruby # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true -. $CCI_SRC/container/defconfig.sh +require 'set' +require_relative '../defconfig' -docker_rm send-mail +docker_rm 'send-mail' -cmd=( - docker run - --restart=always - --name=send-mail - -u nobody - -d - -p 11311:11311 - -v /etc/localtime:/etc/localtime:ro - send-mail -) +names = Set.new %w[ + EMAIL_PSWD + SEND_MAIL_PORT +] -"${cmd[@]}" +defaults = relevant_defaults(names) -cat<<EOF +EMAIL_PSWD = defaults['EMAIL_PSWD'] +SEND_MAIL_PORT = defaults['SEND_MAIL_PORT'] + +cmd = %W[ + docker run + --restart=always + --name=send-mail + -u nobody + -d + -e EMAIL_PSWD=#{EMAIL_PSWD} + -e SEND_MAIL_PORT=#{SEND_MAIL_PORT} + -p #{SEND_MAIL_PORT}:#{SEND_MAIL_PORT} + -v /etc/localtime:/etc/localtime:ro + --log-driver json-file + send-mail +] + +cmd += ['bash', '-c', '/usr/local/bin/mail-post.rb'] + +system(*cmd) + +puts <<HEREDOC port: 11311 + +default from email: + when sending crystal.ci email. + team(a)crystal.ci if no from email appiontted + when sending to WAN email + compass-ci(a)qq.com in all cases + --- data='{ -"from": "email_from_addr" # default: team(a)crystal.ci +"from": "email_from_addr" "subject": "email subject", "to": "email_to_addr", "body": "email message" @@ -32,19 +56,17 @@ data='{ or data=" -from: email_from_addr # default: team(a)crystal.ci +from: email_from_addr subject: email subject to: email_to_addr body: email message" -when compass-ci: - curl -XPOST 'localhost:11311/send_mail_yaml' -d "$data" -when other server: - curl -XPOST '124.90.34.227:11311/send_mail_yaml' -d "$data" +Usage: + curl -XPOST '$send_mail_server:$send_mail_port/send_mail_yaml' -d "$data" --- data=" -From: email_from_addr # default: team(a)crystal.ci +From: email_from_addr To: email_to_addr Subject: email_subject @@ -53,8 +75,6 @@ mail_msg_line2 ... " -when compass-ci: - curl -XPOST 'localhost:11311/send_mail_text' -d "$data" -when other server: - curl -XPOST '124.90.34.227:11311/send_mail_text' -d "$data" -EOF +Usage: + curl -XPOST '$send_mail_server:$send_mail_port/send_mail_text' -d "$data" +HEREDOC -- 2.23.0

1 0

[PATCH compass-ci 3/4] container/send-mail: update Dockerfile
by Luan Shengde 16 Oct '20

16 Oct '20

Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/send-mail/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/container/send-mail/Dockerfile b/container/send-mail/Dockerfile index 36b145c..0cf3eeb 100644 --- a/container/send-mail/Dockerfile +++ b/container/send-mail/Dockerfile @@ -11,9 +11,7 @@ RUN apt-get update && \ ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ echo 'Asia/Shanghai' > /etc/timezone -EXPOSE 25 - COPY mail-post.rb /usr/local/bin/ COPY send-mail.rb /usr/local/bin/ -CMD ["/usr/local/bin/mail-post.rb"] +WORKDIR /usr/local/bin/ -- 2.23.0

1 0

[PATCH compass-ci 2/4] container/send-mail: send-mail.rb
by Luan Shengde 16 Oct '20

16 Oct '20

add setup smtp for qq.com enable send mail via smtp.qq.com Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/send-mail/send-mail.rb | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/container/send-mail/send-mail.rb b/container/send-mail/send-mail.rb index f459e78..4fb2b7e 100755 --- a/container/send-mail/send-mail.rb +++ b/container/send-mail/send-mail.rb @@ -5,15 +5,29 @@ require 'mail' -mail_server = `/sbin/ip route |awk '/default/ {print $3}'`.chomp +# mail_server = `/sbin/ip route |awk '/default/ {print $3}'`.chomp # setup smtp config -smtp = { - address: mail_server, - enable_starttls_auto: false -} +def intranet_smtp(mail_server) + smtp = { + address: mail_server, + enable_starttls_auto: false + } + Mail.defaults { delivery_method :smtp, smtp } +end -Mail.defaults { delivery_method :smtp, smtp } +def internet_smtp + smtp = { + address: 'smtp.qq.com', + port: 25, + domain: 'qq.com', + user_name: 'compass-ci(a)qq.com', + password: (ENV['EMAIL_PSWD']).to_s, + openssl_verify_mode: 'none', + enable_starttls_auto: true + } + Mail.defaults { delivery_method :smtp, smtp } +end # send mail def send_mail(mail_info) -- 2.23.0

1 0

[PATCH compass-ci 1/4] container/send-mail:mail-post.rb
by Luan Shengde 16 Oct '20

16 Oct '20

add setup smpt for intranet/internet smtp enable send mail to both curystal.ci and WAN email Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/send-mail/mail-post.rb | 36 +++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/container/send-mail/mail-post.rb b/container/send-mail/mail-post.rb index 4eecf6e..c7c31d5 100755 --- a/container/send-mail/mail-post.rb +++ b/container/send-mail/mail-post.rb @@ -7,21 +7,21 @@ require 'sinatra' require 'json' require 'yaml' require 'open3' -require_relative 'send-mail.rb' +require_relative 'send-mail' set :bind, '0.0.0.0' -set :port, 11311 +set :port, ENV['SEND_MAIL_PORT'] post '/send_mail_yaml' do data = YAML.safe_load request.body.read - raise TypeError, data, 'request data type error' unless data.class.eql? Hash + raise TypeError, data, 'request data type error' unless data.instance_of?(Hash) mail_info = { 'references' => data['references'] || '', 'from' => data['from'] || 'team(a)crystal.ci', - 'subject' => data['subject'], - 'to' => data['to'], - 'body' => data['body'] + 'subject' => data['subject'] || '', + 'to' => data['to'] || '', + 'body' => data['body'] || '' } check_send_mail(mail_info) end @@ -32,17 +32,29 @@ post '/send_mail_text' do mail_info = { 'references' => data.references || '', 'from' => data.from || 'team(a)crystal.ci', - 'subject' => data.subject, - 'to' => data.to, - 'body' => data.body.decoded + 'subject' => data.subject || '', + 'to' => data.to[0] || '', + 'body' => data.body.decoded || '' } check_send_mail(mail_info) end def check_send_mail(mail_info) - raise TypeError, data, 'empty subject.' if mail_info['subject'].empty? - raise TypeError, data, 'empty email address.' if mail_info['to'].empty? - raise TypeError, data, 'empty email content.' if mail_info['body'].empty? + raise 'empty subject.' if mail_info['subject'].empty? + raise 'empty email address.' if mail_info['to'].empty? + raise 'empty email content.' if mail_info['body'].empty? + + check_smtp(mail_info) send_mail(mail_info) end + +def check_smtp(mail_info) + mail_server = `/sbin/ip route |awk '/default/ {print $3}'`.chomp + if [nil, 'crystal.ci'].include? mail_info['to'].split('@')[1] + intranet_smtp(mail_server) + else + internet_smtp + mail_info['from'] = 'compass-ci(a)qq.com' + end +end -- 2.23.0

1 0

[PATCH lkp-tests] tests: add get_ipmi_ip function to host-info
by Zhang Yu 16 Oct '20

16 Oct '20

If ipmitool returns error or empty, the return of "$(ipmitool lan print |grep -E "IP Address\s*:" | awk '{print $4}')" is null, and only add the field Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- tests/host-info | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/host-info b/tests/host-info index a57855dc..6ee8df21 100755 --- a/tests/host-info +++ b/tests/host-info @@ -80,6 +80,11 @@ get_partitions() fi } +get_ipmi_ip() +{ + echo "ipmi_ip: $(ipmitool lan print |grep -E "IP Address\s*:" | awk '{print $4}')" +} + main() { get_node @@ -87,6 +92,7 @@ main() get_partitions get_mac get_model_name + get_ipmi_ip } main -- 2.23.0

4 3

[PATCH compass-ci] feat(sparrow): add kernel param: net.ipv6.bindv6only=1
by Yu Chuan 16 Oct '20

16 Oct '20

Turn on the bindv6only function to avoid the use of tcp6 port for services that do not actually need tcp6. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- sparrow/0-package/common | 1 + 1 file changed, 1 insertion(+) diff --git a/sparrow/0-package/common b/sparrow/0-package/common index 2c7de3754fad..31588f40b602 100755 --- a/sparrow/0-package/common +++ b/sparrow/0-package/common @@ -10,6 +10,7 @@ grep -q '^committer:' /etc/group || groupadd -g 1999 committer cat >> /etc/sysctl.conf <<EOF net.ipv4.ip_forward=1 +net.ipv6.bindv6only=1 vm.max_map_count=262144 EOF -- 2.23.0

1 0

[PATCH compass-ci 1/2] web-backend-nginx: Restrict interface access.
by Zhang Yuhang 16 Oct '20

16 Oct '20

Signed-off-by: Zhang Yuhang <zhangyuhang25(a)huawei.com> --- container/web-backend-nginx/nginx.conf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/container/web-backend-nginx/nginx.conf b/container/web-backend-nginx/nginx.conf index 5ab1a26..b461a2c 100644 --- a/container/web-backend-nginx/nginx.conf +++ b/container/web-backend-nginx/nginx.conf @@ -6,7 +6,7 @@ events { http { include /etc/nginx/mime.types; default_type application/octet-stream; - include /etc/nginx/conf.d/*.conf; + include /etc/nginx/conf.d/*.conf; server { listen 11320 ssl; server_name api.compass-ci.openeuler.org; @@ -15,8 +15,7 @@ http { ssl_session_timeout 5m; ssl_ciphers BCDHE-RSA-AES128-GCM-SHA256:ECDHE:ECDH:AES:HIGH:!NULL:!aNULL:!MD5:!ADH:!RC4; ssl_prefer_server_ciphers on; - - location / { + location ~ ^/(get_repos|get_jobs|compare|compare_candidates)$ { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forward-For $proxy_add_x_forwarded_for; -- 2.23.0

1 0

[PATCH v6 compass-ci 2/6] container/mail-robot: answerback-email.rb
by Luan Shengde 16 Oct '20

16 Oct '20

AssignUuid: read email content check email available check if base url in upstream-repos check if commit url valid check pub_key assign account generate uuid write account/uuid to es send uuid to user Signed-off-by: Luan Shengde <luanshengde2(a)huawei.com> --- container/mail-robot/answerback-email.rb | 181 +++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100755 container/mail-robot/answerback-email.rb diff --git a/container/mail-robot/answerback-email.rb b/container/mail-robot/answerback-email.rb new file mode 100755 index 0000000..2cd15ef --- /dev/null +++ b/container/mail-robot/answerback-email.rb @@ -0,0 +1,181 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'mail' +require_relative '/compass-ci/lib/es_client.rb' + +# Assign uuid/account +class AssignUuid + def initialize(mail_content) + @mail_content = mail_content + end + + def build_message(sum_infos) + message = <<~EMAIL_MESSAGE + To: #{sum_infos['email']} + Message-ID: #{sum_infos['message_id']} + Subject: Account Ready + + Dear #{sum_infos['email'].split('@')[0]} + + Thank you for joining us. + You can use the following info to submit jobs: + + First: add the info to ~/.config/compass-ci/defaults/{USER}.yaml + + uuid: #{sum_infos['account_uuid']} + lab: z9 + + Second: download lkp-tests and dependencies + git clone https://gitee.com_/wu_fengguang/lkp-tests.git + cd lkp-tests + make install + source {HOME}/.bashrc && source {HOME}/.bash_profile + + Third: submit job + reference: https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/tutorial.md + how to write job yaml + submit jobs/netperf.yaml + + Repalce the fields {...} with your local environment variable. + + regards + compass-ci + EMAIL_MESSAGE + + return message + end + + def url_addr + url_line = @mail_content.part[0].body.decoded.split(/\n/).find { |line| line =~ %r{https?://} } + message = 'No commit url found' + + raise message unless url_line + + url = url_line.split[-1] + + return url + end + + def check_email_available(email) + url = url_addr + + base_url = url.gsub(%r{/commit/[^/].*}, '') + + upstream_dir = '/upstream-repos' + check_url_in_upstream_repos(upstream_dir, base_url) + check_url_commit(url, base_url, email) + + return url + end + + def check_url_in_upstream_repos(upstream_dir, base_url) + Dir.chdir(upstream_dir) + match_out = %x(grep -rn #{base_url}) + + message = 'The url is not in upstream-repo list' + + raise StandardError, message, base_url if match_out.empty? + end + + def check_url_commit(url, base_url, email) + hub_name = url.split('/')[2] + check_gitee_commit(url, base_url, email) if hub_name.eql? 'gitee.com' + check_non_gitee_commit(url, email) unless hub_name.eql? 'gitee.com' + end + + def check_non_gitee_commit(url, email) + url_fdback = %x(curl #{url}) + email_index = url_fdback.index email + + message = 'No email matched from the commit_id feedback.' + raise StandardError, message, url if email_index.nil? + end + + def check_gitee_commit(url, base_url, email) + repo_info = { + 'repos_dir' => '/repos_dir', + 'repo_dir' => url.split('/')[-3], + 'repo_url' => [base_url, 'git'].join('.'), + 'repo_path' => File.join('/repos_dir', url.split('/')[-3]), + 'commit_id' => url.split('/')[-1] + } + check_gitee_repo_commit(url, email, repo_info) + end + + def check_gitee_repo_commit(url, email, repo_info) + Dir.chdir repo_info['repos_dir'] + %x(/usr/bin/git clone #{repo_info['repo_url']}) + email_index = %x(/usr/bin/git -C #{repo_info['repo_dir']} show #{repo_info['commit_id']}).index email + + FileUtils.rm_rf repo_info['repo_dir'] + + message = 'No email matched from commit_id feedback.' if email_index.nil? + raise StandardError, message, url if email_index.nil? + end + + def pub_key + pub_key = @mail_content.part[1].body.decoded.split(/\r|\n/).join if @mail_content.part[1].filename == 'id_rsa.pub' + + message = 'No pub_key found' + + raise message unless pub_key + + return pub_key + end + + def account_info(pub_key) + account_info_str = %x(curl -XGET "#{ENV['J_IP']}:#{ENV['J_PORT']}/assign_account" -d "pub_key: #{pub_key}") + account_info = JSON.parse account_info_str + + message = 'No available jumper account returned.' + raise StandardError, message, account_info if account_info['account'].empty? + + return account_info + end + + def build_sum_infos + email = @mail_content.from[0] + account_uuid = %x(uuidgen).chomp + message_id = @mail_content.message_id + sum_infos = { + 'email' => email, + 'message_id' => message_id, + 'url' => check_email_available(email), + 'acct_infos' => account_info(pub_key), + 'account_uuid' => account_uuid, + 'pub_key' => pub_key + } + store_account_info(sum_infos) + sum_infos + end + + def send_uuid(sum_infos) + message = build_message(sum_infos) + + send_mail_server = `/sbin/ip route |awk '/default/ {print $3}'`.chomp + %x(curl -XPOST "#{send_mail_server}:#{ENV['S_M_PORT']}/send_mail_text" -d "#{message}") + end + + def send_account + sum_infos = build_sum_infos + + send_uuid(sum_infos) + end + + def store_account_info(sum_infos) + account_uuid_info = { + 'email' => sum_infos['email'], + 'name' => sum_infos['email'].split('@')[0], + 'uuid' => sum_infos['account_uuid'], + 'user' => sum_infos['acct_infos']['account'], + 'my_commit_url' => sum_infos['url'] + } + + es = ESClient.new(index: 'accounts') + es.put_source_by_id(sum_infos['email'], account_uuid_info) + end +end -- 2.23.0

3 11

[PATCH compass-ci 2/3] git_mirror: put repo info to es
by Li Yuanchao 16 Oct '20

16 Oct '20

move a method to pass rubocop limit: Class xxx has too many lines[101/100] Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 5f7fa6b..fef4aca 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -9,6 +9,8 @@ require 'json' # gem install PriorityQueue require 'priority_queue' require 'English' +require 'elasticsearch' +require_relative 'constants.rb' # worker threads class GitMirror @@ -84,6 +86,7 @@ class MirrorMain @git_info = {} @defaults = {} @git_queue = Queue.new + @es_client = Elasticsearch::Client.new(url: "http://#{ES_HOST}:#{ES_PORT}") load_fork_info connection = Bunny.new('amqp://172.17.0.1:5672') connection.start @@ -108,16 +111,6 @@ class MirrorMain @defaults[project] = YAML.safe_load(File.open(defaults_file)) end - def load_repo_file(repodir, project, fork_name) - git_repo = "#{project}/#{fork_name}" - @git_info[git_repo] = YAML.safe_load(File.open(repodir)) - @git_info[git_repo]['git_repo'] = git_repo - @git_info[git_repo].merge!(@defaults[project]) if @defaults[project] - fork_stat_init(git_repo) - @priority_queue.push git_repo, @priority - @priority += 1 - end - def traverse_repodir(repodir) if File.directory? repodir load_defaults(repodir) @@ -194,6 +187,17 @@ end # main thread class MirrorMain + def load_repo_file(repodir, project, fork_name) + git_repo = "#{project}/#{fork_name}" + @git_info[git_repo] = YAML.safe_load(File.open(repodir)) + @git_info[git_repo]['git_repo'] = git_repo + @git_info[git_repo].merge!(@defaults[project]) if @defaults[project] + es_repo_update(git_repo, @git_info[git_repo]) + fork_stat_init(git_repo) + @priority_queue.push git_repo, @priority + @priority += 1 + end + def compare_refs(cur_refs, old_refs) new_refs = { heads: {} } cur_refs[:heads].each do |ref, commit_id| @@ -246,4 +250,12 @@ class MirrorMain load_repo_file(repo_dir, File.dirname(file), File.basename(file)) if File.file?(repo_dir) end end + + def es_repo_update(git_repo, repo_info) + body = { + "doc": repo_info, + "doc_as_upsert": true + } + @es_client.update(index: 'repo', type: '_doc', id: git_repo, body: body) + end end -- 2.23.0

2 1