- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci 2/5] service/scheduler: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/scheduler/my-start | 6 ++++++ container/scheduler/start | 6 ++++++ src/scheduler/elasticsearch_client.cr | 8 +++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/container/scheduler/my-start b/container/scheduler/my-start index e5855af..5dd871e 100755 --- a/container/scheduler/my-start +++ b/container/scheduler/my-start @@ -13,6 +13,8 @@ names = Set.new %w[ REDIS_PORT ES_HOST ES_PORT + ES_USER + ES_PASSWORD OS_HTTP_HOST OS_HTTP_PORT INITRD_HTTP_HOST @@ -38,6 +40,10 @@ defaults['SCHED_PORT'] ||= '3000' defaults['SCHED_HOST'] ||= '172.17.0.1' defaults['MASTER_FLUENTD_PORT'] ||= '24224' defaults['SRV_HTTP_CCI_PORT'] ||= '11307' + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) CCI_REPOS = ENV['CCI_REPOS'] || '/c' diff --git a/container/scheduler/start b/container/scheduler/start index 598b263..029dda8 100755 --- a/container/scheduler/start +++ b/container/scheduler/start @@ -14,6 +14,8 @@ names = Set.new %w[ REDIS_PORT ES_HOST ES_PORT + ES_USER + ES_PASSWORD OS_HTTP_HOST OS_HTTP_PORT INITRD_HTTP_HOST @@ -44,6 +46,10 @@ SCHED_PORT = defaults['SCHED_PORT'] sched_nodes.each do |node| docker_rm "s001-alpine-#{node['port']}" end + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) CCI_REPOS = ENV['CCI_REPOS'] || '/c' diff --git a/src/scheduler/elasticsearch_client.cr b/src/scheduler/elasticsearch_client.cr index 0e83ff1..1ef79a6 100644 --- a/src/scheduler/elasticsearch_client.cr +++ b/src/scheduler/elasticsearch_client.cr @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +require "uri" require "yaml" require "json" require "any_merge" @@ -30,7 +31,12 @@ class Elasticsearch::Client HOST = (ENV.has_key?("ES_HOST") ? ENV["ES_HOST"] : JOB_ES_HOST) PORT = (ENV.has_key?("ES_PORT") ? ENV["ES_PORT"] : JOB_ES_PORT).to_i32 - def initialize(host = HOST, port = PORT) + def initialize(host = HOST, port = PORT, auth = true) + if auth + user = ENV["ES_USER"]? + password = ENV["ES_PASSWORD"]? + host = "http://#{user}:#{URI.encode_www_form(password)}@#{host}" if user && password + end @client = Elasticsearch::API::Client.new({:host => host, :port => port}) end -- 2.23.0

1 0

[PATCH compass-ci 1/5] service/es: configuring authentication
by Wu Zhende 21 Apr '21

21 Apr '21

To improve security, need to enter the account and password to access the kibana web page. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/defconfig.sh | 19 +++++++++++++++++++ container/es/Dockerfile | 7 +++++++ container/es/build | 9 ++++++++- container/es/start | 2 +- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/container/defconfig.sh b/container/defconfig.sh index 50446d8..64a434f 100755 --- a/container/defconfig.sh +++ b/container/defconfig.sh @@ -20,6 +20,12 @@ load_cci_defaults() done } +load_service_authentication() +{ + shopt -s nullglob + create_yaml_variables '/etc/compass-ci/passwd.yaml' +} + docker_rm() { container=$1 @@ -28,6 +34,19 @@ docker_rm() docker rm -f $container } +check_auth_es_ready() +{ + local port=$1 + load_service_authentication + local i + for i in {1..30} + do + + curl -s localhost:$port -u $ES_USER:$ES_PASSWORD> /dev/null && return + sleep 2 + done +} + check_service_ready() { local port=$1 diff --git a/container/es/Dockerfile b/container/es/Dockerfile index 44d87ee..f29e677 100644 --- a/container/es/Dockerfile +++ b/container/es/Dockerfile @@ -4,11 +4,16 @@ FROM elasticsearch:7.11.1@sha256:d52cda1e73d1b1915ba2d76ca1e426620c7b5d6942d9d2f432259503974ba786 ARG MEMORY +ARG USER +ARG PASSWORD RUN sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a path.data: /srv/es' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a node.name: node-1' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a cluster.initial_master_nodes: ["node-1"]' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.security.enabled: true' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.license.self_generated.type: basic' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.security.transport.ssl.enabled: true' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i "s/-Xms256m/-Xms${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options && \ sed -i "s/-Xmx256m/-Xmx${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options @@ -17,6 +22,8 @@ RUN mkdir /usr/share/elasticsearch/tmp && \ WORKDIR /usr/share/elasticsearch +RUN ./bin/elasticsearch-users useradd ${USER} -p ${PASSWORD} -r superuser + ENV PATH /usr/share/elasticsearch/bin:$PATH ENV ES_TMPDIR /usr/share/elasticsearch/tmp diff --git a/container/es/build b/container/es/build index db5145f..e023227 100755 --- a/container/es/build +++ b/container/es/build @@ -3,10 +3,17 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig.rb' +names = Set.new %w[ + ES_USER + ES_PASSWORD +] + +defaults = relevant_service_authentication(names) docker_skip_rebuild "es:7.11.1" available_memory = get_available_memory -system "docker build -t es:7.11.1 --build-arg MEMORY=#{available_memory} --network=host ." +system "docker build -t es:7.11.1 --build-arg MEMORY=#{available_memory} --build-arg USER=#{defaults['ES_USER']} --build-arg PASSWORD=#{defaults['ES_PASSWORD']} --network=host ." diff --git a/container/es/start b/container/es/start index 3aa9525..cff44ea 100755 --- a/container/es/start +++ b/container/es/start @@ -21,5 +21,5 @@ cmd=( "${cmd[@]}" # set index -check_service_ready 9200 +check_auth_es_ready 9200 find $CCI_SRC/sbin/ -name "es-*-mapping.sh" -exec sh {} \; -- 2.23.0

1 0

[PATCH compass-ci] delimiter: delete es_query in bisect script
by Cao Xueliang 20 Apr '21

20 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/git_bisect.rb | 8 +++----- src/delimiter/find-commit/bisect_run_script.rb | 18 +++++++++++------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/git_bisect.rb b/lib/git_bisect.rb index 6aedda5..c0c0a60 100644 --- a/lib/git_bisect.rb +++ b/lib/git_bisect.rb @@ -3,7 +3,6 @@ # frozen_string_literal: true require 'json' -require_relative 'es_query' require_relative 'error_messages' require_relative 'sched_client' require_relative '../src/delimiter/utils' @@ -12,7 +11,6 @@ require_relative "#{ENV['LKP_SRC']}/lib/monitor" # find the first bad commit class GitBisect def initialize(task) - @es = ESQuery.new @task = task end @@ -54,7 +52,7 @@ class GitBisect puts "upstream_commit: #{@upstream_commit}" raise 'upstream info is null' unless @upstream_repo || @upstream_commit - @upstream_repo_git = "git://#{GIT_MIRROR_HOST}/#{@upstream_repo}" + @upstream_repo_git = "git://#{GIT_MIRROR_HOST}/upstream/#{@upstream_repo}" end def set_upstream_url @@ -66,7 +64,7 @@ class GitBisect def set_work_dir @work_dir = Utils.clone_repo(@upstream_repo_git, @upstream_commit) puts "work_dir: #{@work_dir}" - raise "checkout repo: #{@upstream_repo} to commit: #{@upstream_commit} failed!" unless @work_dir + raise "checkout repo: #{@upstream_repo_git} to commit: #{@upstream_commit} failed!" unless @work_dir end def set_pkgbuild_repo @@ -104,7 +102,7 @@ class GitBisect return nil unless temp[0].include?('is the first bad commit') || temp[-1].include?('bisect run success') first_bad_commit = Utils.parse_first_bad_commit(result) - bisect_error = ErrorMessages.new(@build_pkg_dir).obtain_error_messages_by_error_id(@error_id) + bisect_error = Utils.get_bisect_error(@bad_job_id, @error_id) all_errors = Utils.obt_errors(@work_dir, first_bad_commit) first_bad_commit_result_root = Utils.obt_result_root_by_commit(first_bad_commit) diff --git a/src/delimiter/find-commit/bisect_run_script.rb b/src/delimiter/find-commit/bisect_run_script.rb index 8cdf675..05c529e 100755 --- a/src/delimiter/find-commit/bisect_run_script.rb +++ b/src/delimiter/find-commit/bisect_run_script.rb @@ -5,13 +5,11 @@ # frozen_string_literal: true require 'json' -require_relative "#{ENV['CCI_SRC']}/lib/es_query" require_relative "#{ENV['CCI_SRC']}/src/delimiter/utils" # git bisect run class GitBisectRun def initialize(job_id, error_id, work_dir) - @es = ESQuery.new @job_id = job_id @error_id = error_id @work_dir = work_dir @@ -27,12 +25,17 @@ class GitBisectRun private def get_bisect_status(job) - status = Utils.get_job_status(job, @error_id) - exit 125 unless status + begin + status = Utils.get_job_status(job, @error_id) + exit 125 unless status - exit 1 if status.eql?('bad') + exit 1 if status.eql?('bad') - exit 0 + exit 0 + rescue StandardError => e + puts e.backtrace.inspect + exit -1 + end end end @@ -43,6 +46,7 @@ begin run = GitBisectRun.new job_id, error_id, work_dir run.git_bisect -rescue +rescue StandardError => e + puts e.backtrace.inspect exit -1 end -- 2.23.0

1 0

[PATCH compass-ci] container/assist-result: add new function in assist-result
by Cao Xueliang 20 Apr '21

20 Apr '21

add get_compare_result and get_error_messages inassist-result, use them in delimiter utils. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/assist-result/routes.rb | 23 +++++++++++++++++++ .../assist-result/views/get_compare_result.rb | 10 ++++++++ .../assist-result/views/get_error_messages.rb | 9 ++++++++ lib/assist_result_client.rb | 20 ++++++++++++++++ src/delimiter/utils.rb | 15 ++++++++---- 5 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 container/assist-result/views/get_compare_result.rb create mode 100644 container/assist-result/views/get_error_messages.rb diff --git a/container/assist-result/routes.rb b/container/assist-result/routes.rb index 287db25..c0712a9 100755 --- a/container/assist-result/routes.rb +++ b/container/assist-result/routes.rb @@ -9,6 +9,8 @@ require 'sinatra' require_relative './views/get_job_yaml' require_relative './views/get_job_content' require_relative './views/check_job_credible' +require_relative './views/get_compare_result' +require_relative './views/get_error_messages' configure do set :bind, '0.0.0.0' @@ -45,3 +47,24 @@ get '/get_job_content/:job_id' do return [200, result.to_json] end + +get '/get_compare_errors/*,*' do |pre_id, cur_id| + begin + result = get_compare_errors(pre_id, cur_id) + rescue StandardError => e + return [400, e.backtrace.inspect] + end + + return [200, result.to_json] +end + +post '/get_error_messages' do + begin + data = JSON.parse(Base64.decode64(request.body.read)) + result = get_error_messages(data['job_id'], data['error_id']) + rescue StandardError => e + return [400, e.backtrace.inspect] + end + + return [200, result.to_json] +end diff --git a/container/assist-result/views/get_compare_result.rb b/container/assist-result/views/get_compare_result.rb new file mode 100644 index 0000000..cf9e1de --- /dev/null +++ b/container/assist-result/views/get_compare_result.rb @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative "#{ENV['CCI_SRC']}/lib/compare_error_messages" + +def get_compare_errors(pre_id, cur_id) + _, errors = CEM.get_compare_result(pre_id, cur_id) + return errors +end diff --git a/container/assist-result/views/get_error_messages.rb b/container/assist-result/views/get_error_messages.rb new file mode 100644 index 0000000..e429dfe --- /dev/null +++ b/container/assist-result/views/get_error_messages.rb @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative "#{ENV['CCI_SRC']}/lib/compare_error_messages" + +def get_error_messages(job_id, error_id) + return CEM.get_error_messages(job_id, error_id) +end diff --git a/lib/assist_result_client.rb b/lib/assist_result_client.rb index 9c42b97..592c841 100644 --- a/lib/assist_result_client.rb +++ b/lib/assist_result_client.rb @@ -42,4 +42,24 @@ class AssistResult return response.body end + + def get_compare_errors(pre_id, cur_id) + resource = RestClient::Resource.new("http://#{@host}:#{@port}/get_compare_errors/#{pre_id},#{cur_id}") + response = resource.get() + return nil unless response.code == 200 + + return response.body + end + + def get_error_messages(job_id, error_id) + data = { + 'job_id' => job_id, + 'error_id' => error_id + } + resource = RestClient::Resource.new("http://#{@host}:#{@port}/get_error_messages") + response = resource.post(Base64.encode64(data.to_json)) + return nil unless response.code == 200 + + return response.body + end end diff --git a/src/delimiter/utils.rb b/src/delimiter/utils.rb index f960c7e..c782ea2 100644 --- a/src/delimiter/utils.rb +++ b/src/delimiter/utils.rb @@ -67,7 +67,7 @@ module Utils def submit_job(job) save_job_to_yaml(job, PROCESS_JOB_YAML) - response = %x(#{LKP_SRC}/sbin/submit #{PROCESS_JOB_YAML}) + response = %x(#{ENV['LKP_SRC']}/sbin/submit #{PROCESS_JOB_YAML}) puts "submit job response: #{response}" return nil if response =~ /job id=0/ return $1 if response =~ /job id=(.*)/ @@ -87,6 +87,7 @@ module Utils query = { 'job_id': new_job_id, 'job_state': 'extract_finished' } extract_finished = monitor_run_stop(query) return nil unless extract_finished.zero? + puts "#{bad_job_id}, #{new_job_id}, #{error_id}" check_result = AssistResult.new.check_job_credible(bad_job_id, new_job_id, error_id) raise "check job credible failed: #{bad_job_id}, #{new_job_id}, #{error_id}" if check_result == nil @@ -185,13 +186,17 @@ module Utils def obt_errors_by_commits(cur_commit, pre_commit) cur_id = obt_id_by_commit(cur_commit) pre_id = obt_id_by_commit(pre_commit) - _, errors = get_compare_result(pre_id, cur_id) - return errors + return AssistResult.new.get_compare_errors(pre_id, cur_id) end def obt_result_root_by_commit(commit) - id = obt_id_by_commit(commit) - ESQuery.new.query_by_id(id)['result_root'] + job_id = obt_id_by_commit(commit) + job_content = AssistResult.new.get_job_content(job_id) + return JSON.parse(job_content)['result_root'] + end + + def get_bisect_error(job_id, error_id) + return AssistResult.new.get_error_messages(job_id, error_id) end end end -- 2.23.0

1 0

[PATCH v3 lkp-tests] sbin/submit: support -n
by Li Yuanchao 20 Apr '21

20 Apr '21

If -n $number, and $number is a integer and $number > 1, submit $number times. And 'nr_run' would be added in the job.yaml. Else, submit once. [usage] submit xxx xxx -n 3 xxx.yaml Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/submit | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/sbin/submit b/sbin/submit index edb207cc0..a35ca0dac 100755 --- a/sbin/submit +++ b/sbin/submit @@ -19,6 +19,7 @@ opt_monitor_query = {} opt_my_queue = false actions = ['output', 'stop'] result_roots = [] +nr_run = 1 options = OptionParser.new do |opts| opts.banner = 'Usage: submit [options] job1.yaml job2.yaml ...' @@ -57,6 +58,11 @@ options = OptionParser.new do |opts| actions << 'mirror_result' end + opts.on('-n job_number', '--number job_number', 'number to submit job') do |number| + nr_run = number.to_i + nr_run = 1 if nr_run < 1 + end + opts.on('-m', '--monitor', "monitor job status: use -m 'KEY: VALUE' to add rule") do opt_monitor = true k, v = ARGV[0].sub(' ', '').split(':', 2) if ARGV[0] @@ -195,16 +201,19 @@ ARGV.each do |jobfile| scheduler_client = SchedulerClient.new(job['SCHED_HOST'], job['SCHED_PORT']) # submit job + job_hash['nr_run'] = nr_run job_json = job_hash.to_json - messages = scheduler_client.submit_job(job_json) - JSON.parse(messages).each do |msg| - if msg['message'].empty? - result_roots << msg['result_root'] - job_ids << msg['job_id'].to_s - puts("submit #{jobfile}, got job id=#{msg['job_id']}") - else - opt_monitor = false - puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + nr_run.times do + messages = scheduler_client.submit_job(job_json) + JSON.parse(messages).each do |msg| + if msg['message'].empty? + result_roots << msg['result_root'] + job_ids << msg['job_id'].to_s + puts("submit #{jobfile}, got job id=#{msg['job_id']}") + else + opt_monitor = false + puts("submit #{jobfile} failed, got job id=#{msg['job_id']}, error: #{msg['message']}") + end end end end -- 2.23.0

1 0

[PATCH compass-ci] lib: use module as namespace in compare_error_messages.rb
by Cao Xueliang 20 Apr '21

20 Apr '21

1. use module CEM. 2. add get_error_messages function. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- .../assist-result/views/check_job_credible.rb | 2 +- lib/compare_error_messages.rb | 138 ++++++++++-------- 2 files changed, 75 insertions(+), 65 deletions(-) diff --git a/container/assist-result/views/check_job_credible.rb b/container/assist-result/views/check_job_credible.rb index a8d387f..c5fc453 100644 --- a/container/assist-result/views/check_job_credible.rb +++ b/container/assist-result/views/check_job_credible.rb @@ -5,5 +5,5 @@ require_relative "#{ENV['CCI_SRC']}/lib/compare_error_messages" def check_job_credible(pre_job_id, cur_job_id, error_id) - return credible?(pre_job_id, cur_job_id, error_id) + return CEM.credible?(pre_job_id, cur_job_id, error_id) end diff --git a/lib/compare_error_messages.rb b/lib/compare_error_messages.rb index 9c864a3..93b38ed 100644 --- a/lib/compare_error_messages.rb +++ b/lib/compare_error_messages.rb @@ -8,91 +8,101 @@ require_relative 'es_query' require_relative 'error_messages' require_relative "#{ENV['LKP_SRC']}/lib/common" -# previous_job_id's error_ids include error_id -# If later_job_id's output include error filename that extract from error_id, -# the later_job_id is credible. -def credible?(previous_job_id, later_job_id, error_id) - es = ESQuery.new - later_es_result = es.query_by_id(later_job_id) +module CEM + extend self - return true if later_es_result['job_state'] == 'finished' + # previous_job_id's error_ids include error_id + # If later_job_id's output include error filename that extract from error_id, + # the later_job_id is credible. + def credible?(previous_job_id, later_job_id, error_id) + es = ESQuery.new + later_es_result = es.query_by_id(later_job_id) - previous_result_file = File.join('/srv', es.query_by_id(previous_job_id)['result_root'], 'build-pkg') - later_result_file = File.join('/srv', later_es_result['result_root'], 'build-pkg') + return true if later_es_result['job_state'] == 'finished' - filenames_check = filenames_check(previous_result_file, later_result_file, error_id) + previous_result_file = File.join('/srv', es.query_by_id(previous_job_id)['result_root'], 'build-pkg') + later_result_file = File.join('/srv', later_es_result['result_root'], 'build-pkg') - return false if filenames_check.empty? || filenames_check.value?(false) + filenames_check = filenames_check(previous_result_file, later_result_file, error_id) - return true -end - -def filenames_check(previous_result_file, later_result_file, error_id) - filenames = Set.new - filenames_check = Hash.new { |h, k| h[k] = false } + return false if filenames_check.empty? || filenames_check.value?(false) - error_lines = ErrorMessages.new(previous_result_file).obtain_error_messages_by_error_id(error_id, true) - error_lines.each do |error_line| - # "src/ssl_sock.c:1454:104: warning: unused parameter 'al' [-Wunused-parameter]" => "src/ssl_sock" - filenames << $1.chomp(File.extname($1)) if error_line =~ /(.*)(:\d+){2}: (error|warning):/ + return true end - File.open(later_result_file).each_line do |line| - filenames.each do |filename| - filenames_check[filename] - filenames_check[filename] = true if line.include?(filename) + def filenames_check(previous_result_file, later_result_file, error_id) + filenames = Set.new + filenames_check = Hash.new { |h, k| h[k] = false } + + error_lines = ErrorMessages.new(previous_result_file).obtain_error_messages_by_error_id(error_id, true) + error_lines.each do |error_line| + # "src/ssl_sock.c:1454:104: warning: unused parameter 'al' [-Wunused-parameter]" => "src/ssl_sock" + filenames << $1.chomp(File.extname($1)) if error_line =~ /(.*)(:\d+){2}: (error|warning):/ + end + + File.open(later_result_file).each_line do |line| + filenames.each do |filename| + filenames_check[filename] + filenames_check[filename] = true if line.include?(filename) + end end + + return filenames_check end - return filenames_check -end + def get_compare_result(previous_job_id, later_job_id) + es = ESQuery.new -def get_compare_result(previous_job_id, later_job_id) - es = ESQuery.new + previous_error_ids = es.query_by_id(previous_job_id)['error_ids'] + later_es_result = es.query_by_id(later_job_id) + later_error_ids = later_es_result['error_ids'] + later_result_file = File.join('/srv', later_es_result['result_root'], 'build-pkg') - previous_error_ids = es.query_by_id(previous_job_id)['error_ids'] - later_es_result = es.query_by_id(later_job_id) - later_error_ids = later_es_result['error_ids'] - later_result_file = File.join('/srv', later_es_result['result_root'], 'build-pkg') + new_error_ids = get_new_error_ids(later_error_ids, previous_error_ids) + error_messages = ErrorMessages.new(later_result_file).obtain_error_messages + new_error_number, formatted_error_messages = format_error_messages(error_messages, new_error_ids) - new_error_ids = get_new_error_ids(later_error_ids, previous_error_ids) - error_messages = ErrorMessages.new(later_result_file).obtain_error_messages - new_error_number, formatted_error_messages = format_error_messages(error_messages, new_error_ids) + # new error use ">>" as an identifier in the error header + return new_error_number, formatted_error_messages + end - # new error use ">>" as an identifier in the error header - return new_error_number, formatted_error_messages -end + def get_error_messages(job_id, error_id) + es_result = ESQuery.new.query_by_id(job_id) + result_file = File.join('/srv', es_result['result_root'], 'build-pkg') + return ErrorMessages.new(result_file).obtain_error_messages_by_error_id(error_id) + end -def get_new_error_ids(later_error_ids, previous_error_ids) - return nil_to_empty_array(later_error_ids) - nil_to_empty_array(previous_error_ids) -end + def get_new_error_ids(later_error_ids, previous_error_ids) + return nil_to_empty_array(later_error_ids) - nil_to_empty_array(previous_error_ids) + end -def nil_to_empty_array(array) - array = [] if array.nil? - return array -end + def nil_to_empty_array(array) + array = [] if array.nil? + return array + end -def format_error_messages(error_messages, new_error_ids) - formatted_error_messages = Terminal::Table.new - formatted_error_messages.style = { border_x: '', border_y: '', border_i: '', padding_left: 0 } - new_error_number = 0 - - error_messages.each do |k, v| - if new_error_ids.include?("build-pkg.#{build_pkg_error_id(k)}") - new_error_number += 1 - formatted_error_messages = add_sign(formatted_error_messages, '>>', v) - else - formatted_error_messages = add_sign(formatted_error_messages, ' ', v) + def format_error_messages(error_messages, new_error_ids) + formatted_error_messages = Terminal::Table.new + formatted_error_messages.style = { border_x: '', border_y: '', border_i: '', padding_left: 0 } + new_error_number = 0 + + error_messages.each do |k, v| + if new_error_ids.include?("build-pkg.#{build_pkg_error_id(k)}") + new_error_number += 1 + formatted_error_messages = add_sign(formatted_error_messages, '>>', v) + else + formatted_error_messages = add_sign(formatted_error_messages, ' ', v) + end end + + return new_error_number, formatted_error_messages.to_s end - return new_error_number, formatted_error_messages.to_s -end + def add_sign(formatted_error_messages, sign, set) + set.each do |value| + formatted_error_messages.add_row([sign, value]) + end -def add_sign(formatted_error_messages, sign, set) - set.each do |value| - formatted_error_messages.add_row([sign, value]) + return formatted_error_messages end - - return formatted_error_messages end -- 2.23.0

1 0

[PATCH compass-ci] sbin/auto_submit: support argvs for competition
by Li Yuanchao 20 Apr '21

20 Apr '21

Get true author_email and committer_email for PR. And get base_commit for branch, which is the commit when branch is created. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- sbin/auto_submit | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/sbin/auto_submit b/sbin/auto_submit index 4adce89..28e3715 100755 --- a/sbin/auto_submit +++ b/sbin/auto_submit @@ -51,17 +51,35 @@ class AutoSubmit real_argvs = Array.new(submit_argv) real_argvs.push("upstream_branch=#{branch}") - real_argvs.push("upstream_commit=#{commit_id}") real_argvs.push("upstream_dir=#{belong}") tag = %x(git -C #{mirror_dir} tag --points-at #{commit_id}) real_argvs.push("upstream_tag=#{tag}") unless tag.empty? - author_email = %x(git -C #{mirror_dir} log --format=%ae -1) - committer_email = %x(git -C #{mirror_dir} log --format=%ce -1) - real_argvs.push("author_email=#{author_email} committer_email=#{committer_email}") + if git_repo.include?('my-sql_with_boostkit_patch') + competi_argvs = get_argvs_for_competition(mirror_dir, branch, commit_id) + real_argvs.push(competi_argvs) + end + real_argvs.push("upstream_commit=#{commit_id}") system(real_argvs.join(' ')) end + def get_argvs_for_competition(real_argvs, mirror_dir, branch, commit_id) + argvs_for_base = Array.new(real_argvs) + author_email = %x(git -C #{mirror_dir} log --format=%ae -1 #{commit_id}).chomp + committer_email = %x(git -C #{mirror_dir} log --format=%ce -1 #{commit_id}).chomp + + if committer_email.include?('noreply') + emails = %x(git -C #{mirror_dir} log --format=%ae -2 #{commit_id}) + committer_email = emails.lines[0].chomp + author_email = emails.lines[1].chomp + end + + base_commit = %x(git -C #{mirror_dir} merge-base master #{branch}).chomp + argvs_for_base.push("upstream_commit=#{base_commit}") + system(argvs_for_base.join(' ')) + return "author_email=#{author_email} committer_email=#{committer_email} base_commit=#{base_commit}" + end + def submit_specific_branch(submit_argv, newrefs_info, branches) branches.each do |branch_name| branch = "refs/heads/#{branch_name}" -- 2.23.0

1 0

[PATCH compass-ci] extract-stats: fix expect ":" but got "="
by Cao Xueliang 20 Apr '21

20 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/stats_worker.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index 47c3a59..f385fee 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -99,7 +99,7 @@ class StatsWorker sample_error_id = new_error_ids.sample @log.info("send a delimiter task: job_id is #{job_id}") queue = "#{DELIMITER_TASK_QUEUE}/#{job_id}" - value = {"job_id" => job_id, "error_id" => sample_error_id} + value = %({"job_id": "#{job_id}", "error_id": "#{sample_error_id}"}) @etcd.put(queue, value) msg = %({"job_id": "#{job_id}", "new_error_id": "#{sample_error_id}"}) -- 2.23.0

1 0

[PATCH compass-ci] container/scheduler-http: fix unfriendly error
by Liu Yinsi 20 Apr '21

20 Apr '21

before: /c/compass-ci/container/scheduler-https# ./start Traceback (most recent call last): ./start:14:in `<main>': CA certificate is missing. Ignore if the HTTPS scheduler is not required (RuntimeError) after: /c/compass-ci/container/scheduler-https# ./start CA certificate is missing. Ignore if the HTTPS scheduler is not required Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/scheduler-https/start | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/container/scheduler-https/start b/container/scheduler-https/start index 4ec84a28..374300c0 100755 --- a/container/scheduler-https/start +++ b/container/scheduler-https/start @@ -11,7 +11,10 @@ crt_path = "#{ca_path}/server.crt" key_path = "#{ca_path}/server.key" warn_msg = 'CA certificate is missing. Ignore if the HTTPS scheduler is not required' -raise warn_msg unless File.exist?(crt_path) && File.exist?(key_path) +unless File.exist?(crt_path) && File.exist?(key_path) + puts warn_msg + exit +end names = Set.new %w[ SCHED_HOST -- 2.23.0

1 0

[PATCH v2 compass-ci] lib/rpm-repo.rb: do not move to pub directory until test completed
by Li Ping 20 Apr '21

20 Apr '21

Signed-off-by: Li Ping <1477412247(a)qq.com> --- lib/rpm-repo.rb | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/rpm-repo.rb b/lib/rpm-repo.rb index 658a5270..c8d6b087 100755 --- a/lib/rpm-repo.rb +++ b/lib/rpm-repo.rb @@ -6,6 +6,7 @@ require 'fileutils' require 'json' require './mq_client' +require_relative 'json_logger.rb' MQ_HOST = ENV['MQ_HOST'] || ENV['LKP_SERVER'] || '172.17.0.1' MQ_PORT = ENV['MQ_PORT'] || 5672 @@ -29,31 +30,47 @@ MQ_PORT = ENV['MQ_PORT'] || 5672 # change /srv/rpm/pub/**/repodate # @mq.ack(info) class HandleRepo + @@upload_dir_prefix = "/srv/rpm/upload/" def initialize @mq = MQClient.new(MQ_HOST, MQ_PORT) - @update = [] + @log = JSONLogger.new end def handle_new_rpm queue = @mq.queue("update_repo") queue.subscribe({:block => true, :manual_ack => true}) do |info, _pro, msg| - rpm_info = JSON.parse(msg) - rpm_info["upload_rpms"].each do |rpm| - rpm_path = File.dirname(rpm).sub("upload", "testing") - FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) + begin + rpm_info = JSON.parse(msg) + check_upload_rpms(rpm_info) + rpm_info["upload_rpms"].each do |rpm| + rpm_path = File.dirname(rpm).sub("upload", "testing") + FileUtils.mkdir_p(rpm_path) unless File.directory?(rpm_path) - dest = File.join(rpm_path.to_s, File.basename(rpm)) - @update << dest - FileUtils.mv(rpm, dest) - system("createrepo --update $(dirname #{rpm_path})") + dest = File.join(rpm_path.to_s, File.basename(rpm)) + FileUtils.mv(rpm, dest) + system("createrepo --update $(dirname #{rpm_path})") + end + @mq.ack(info) + rescue StandardError => e + @log.warn({ + "error message": e.message + }.to_json) + @mq.ack(info) end - update_pub_dir - @mq.ack(info) end end - def update_pub_dir - @update.each do |rpm| + def check_upload_rpms(data) + raise JSON.dump({ "errcode" => "200", "errmsg" => "no upload_rpms params" }) unless data.key?("upload_rpms") + raise JSON.dump({ "errcode" => "200", "errmsg" => "upload_rpms params type error" }) if data["upload_rpms"].class != Array + data["upload_rpms"].each do |rpm| + raise JSON.dump({ "errcode" => "200", "errmsg" => "#{rpm} not exist" }) unless File.exist?(rpm) + raise JSON.dump({ "errcode" => "200", "errmsg" => "the upload directory is incorrect" }) unless File.dirname(rpm).start_with?(@@upload_dir_prefix) + end + end + + def update_pub_dir(update) + update.each do |rpm| pub_path = File.dirname(rpm).sub("testing", "pub") FileUtils.mkdir_p(pub_path) unless File.directory?(pub_path) -- 2.23.0

1 0