- Compass-ci - mailweb.openeuler.org

[PATCH v1 compass-ci 3/4] providers/libvirt: request scheduler to consume job
by Xiao Shenwei 21 Jan '21

21 Jan '21

provider a public methods: request_job it will register hostname mac queues and request http://#{@sched_host}:#{@sched_port}/boot.libvirt/mac/#{@info['mac']} Signed-off-by: Xiao Shenwei <xiaoshenwei96(a)163.com> --- providers/libvirt/lib/consumer.rb | 81 +++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 providers/libvirt/lib/consumer.rb diff --git a/providers/libvirt/lib/consumer.rb b/providers/libvirt/lib/consumer.rb new file mode 100644 index 0000000..6cfa448 --- /dev/null +++ b/providers/libvirt/lib/consumer.rb @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require 'yaml' +require 'json' +require_relative 'base' +require_relative "#{ENV['CCI_SRC']}/container/defconfig" + +# register hostname mac queues +# request scheduler +class Consumer < Base + attr_reader :info + + def initialize(hostname, queues) + @info = {} + @info['hostname'] = hostname + @info['queues'] = queues + end + + def close + del_host_info + end + + def request_job + get_mac_from_hostname(@info['hostname']) + config_scheduler + set_host_info + host_exists + url = "http://#{@sched_host}:#{@sched_port}/boot.libvirt/mac/#{@info['mac']}" + @logger.info("Request URL: #{url}") + res = %x(curl #{url}) + if res.empty? + @logger.error('Can not connect scheduler') + raise 'Can not connect scheduler' + end + JSON.parse(res) + end + + private + + def config_scheduler + names = Set.new %w[ + SCHED_HOST + SCHED_PORT + ] + defaults = relevant_defaults(names) + @sched_host = defaults['SCHED_HOST'] || '172.17.0.1' + @sched_port = defaults['SCHED_PORT'] || 3000 + @logger.info("SCHED_HOST: #{@sched_host}") + @logger.info("SCHED_PORT: #{@sched_port}") + end + + def get_mac_from_hostname(hostname) + cmd = %(echo #{hostname} | md5sum | sed "s/^\$..\$\$..\$\$..\$\$..\$\$..\$.*$/0a-\\1-\\2-\\3-\\4-\\5/") + @info['mac'] = %x(#{cmd}).chomp + @logger.info("Mac address: #{@info['mac']}") + end + + def set_host_info + system "curl -X PUT \ + 'http://#{@sched_host}:#{@sched_port}/set_host_mac?hostname=#{@info['hostname']}&mac=#{@info['mac']}'" + system "curl -X PUT \ + 'http://#{@sched_host}:#{@sched_port}/set_host2queues?host=#{@info['hostname']}&queues=#{@info['queues']}'" + end + + def del_host_info + system "curl -X PUT 'http://#{@sched_host}:#{@sched_port}/del_host_mac?mac=#{@info['mac']}'" + system "curl -X PUT 'http://#{@sched_host}:#{@sched_port}/del_host2queues?host=#{@info['hostname']}'" + end + + def host_exists + @info['host'] = @info['hostname'].split('.')[0] + host_file = "#{ENV['LKP_SRC']}/hosts/#{@info['host']}" + return if FileTest.exists?(host_file) + + @logger.error("The file does't exist: #{host_file}") + raise "The file does't exist: #{host_file}" + end +end -- 2.23.0

2 2

[PATCH v2 compass-ci 5/5] upload-libvirt-xml/nginx.conf: nginx.conf in upload-libvirt-xml container
by cuiyili 21 Jan '21

21 Jan '21

Signed-off-by: cuiyili <2268260388(a)qq.com> --- container/upload-libvirt-xml/nginx.conf | 43 +++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 container/upload-libvirt-xml/nginx.conf diff --git a/container/upload-libvirt-xml/nginx.conf b/container/upload-libvirt-xml/nginx.conf new file mode 100644 index 0000000..aef8fb8 --- /dev/null +++ b/container/upload-libvirt-xml/nginx.conf @@ -0,0 +1,43 @@ +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ + +user lkp; +worker_processes auto; +pid /run/nginx.pid; + +events {} + +http { + server { + listen 3081; + server_name "upload-libvirt-xml"; + server_tokens off; + client_max_body_size 50m; + access_log /tmp/access.log; + error_log /tmp/error.log; + + location / { + allow all; + root /srv/cci/libvirt-xml/; + + autoindex on; + create_full_put_path on; + dav_methods PUT MKCOL; + dav_access user:rw group:rw all:rw; + content_by_lua_block { + ngx.req.read_body() + local filename = string.gsub(ngx.var.request_uri, "/", "") + if string.match(filename, "([^/]+).xml$") then + add_path = string.match(filename, "([^/]+).xml$") + local data = ngx.req.get_body_data() + local file = io.open("/srv/cci/libvirt-xml/"..filename, "w") + file:write(data) + file:close() + file_md5 = io.popen("cd /srv/cci/libvirt-xml/ && file_md5=$(md5sum "..filename.." | awk '{print $1}') && mkdir -p "..add_path.." && mv "..filename.." "..add_path.."/${file_md5}.xml && echo ${file_md5}"):read() + ngx.say(add_path.."/"..tostring(file_md5)..".xml") + end + } + } + + } +} -- 2.23.0

1 0

[PATCH v2 compass-ci 4/5] upload-libvirt-xml/openresty.sh: start openresty in upload-libvirt-xml docker image
by cuiyili 21 Jan '21

21 Jan '21

Signed-off-by: cuiyili <2268260388(a)qq.com> --- container/upload-libvirt-xml/openresty.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 container/upload-libvirt-xml/openresty.sh diff --git a/container/upload-libvirt-xml/openresty.sh b/container/upload-libvirt-xml/openresty.sh new file mode 100755 index 0000000..6215d91 --- /dev/null +++ b/container/upload-libvirt-xml/openresty.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +umask 002 +/usr/local/openresty/bin/openresty -g 'daemon off;' -- 2.23.0

1 0

[PATCH v2 compass-ci 3/5] upload-libvirt-xml/start: start upload-libvirt-xml docker container
by cuiyili 21 Jan '21

21 Jan '21

Signed-off-by: cuiyili <2268260388(a)qq.com> --- container/upload-libvirt-xml/start | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 container/upload-libvirt-xml/start diff --git a/container/upload-libvirt-xml/start b/container/upload-libvirt-xml/start new file mode 100755 index 0000000..ddb04e7 --- /dev/null +++ b/container/upload-libvirt-xml/start @@ -0,0 +1,23 @@ +#!/bin/bash +# Refer to: https://hub.docker.com/r/openresty/openresty +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# SPDX-License-Identifier: MulanPSL-2.0+ + +. $CCI_SRC/container/defconfig.sh + +: ${UPLOAD_LIBVIRT_PORT:=3081} + +docker_rm upload-libvirt-xml + +cmd=( + docker run -d + --name upload-libvirt-xml + --restart=always + -p ${UPLOAD_LIBVIRT_PORT}:3081 + -v $PWD/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf + -v /etc/localtime:/etc/localtime:ro + -v /srv/cci/libvirt-xml:/srv/cci/libvirt-xml + upload-libvirt-xml +) + +"${cmd[@]}" -- 2.23.0

1 0

[PATCH v2 compass-ci 2/5] upload-libvirt-xml/Dockerfile: add upload-libvirt-xml docker image
by cuiyili 21 Jan '21

21 Jan '21

Add upload-libvirt-xml docker image in order to upload libvirt xml file. Signed-off-by: cuiyili <2268260388(a)qq.com> --- container/upload-libvirt-xml/Dockerfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 container/upload-libvirt-xml/Dockerfile diff --git a/container/upload-libvirt-xml/Dockerfile b/container/upload-libvirt-xml/Dockerfile new file mode 100644 index 0000000..db48bcb --- /dev/null +++ b/container/upload-libvirt-xml/Dockerfile @@ -0,0 +1,14 @@ +# Origin: https://hub.docker.com/r/openresty/openresty +# Copyright (C) 2016-2020 Eric D. Evan Wies +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +FROM openresty/openresty:alpine + +RUN apk add bash +RUN mkdir -p /var/log/nginx/ +RUN adduser lkp -u 1090 -g 1090 -D + +COPY ./openresty.sh /usr/local/bin/ + +CMD ["/usr/local/bin/openresty.sh"] -- 2.23.0

1 0

[PATCH lkp-tests] submit: upload libvirt config files
by Ren Wen 21 Jan '21

21 Jan '21

An option for 'submit': -f, --upload-file FILE upload an XML template of libvirt File's extension should be '.xml'. Example: case1: submit -f existent_file.xml => existent_file/<md5sum>.xml case2: submit -f nonexistent_file.xml => Find no file: nonexistent_file.xml Signed-off-by: Ren Wen <15991987063(a)163.com> --- lib/job2sh.rb | 10 ++++++++++ sbin/submit | 31 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/lib/job2sh.rb b/lib/job2sh.rb index 7f9cf562f..14c4e3f4b 100755 --- a/lib/job2sh.rb +++ b/lib/job2sh.rb @@ -15,6 +15,16 @@ SHELL_BLOCK_KEYWORDS = { 'function' => ['{', '}'] }.freeze +def cci_defaults + hash = {} + Dir.glob(['/etc/compass-ci/defaults/*.yaml', + '/etc/compass-ci/accounts/*.yaml', + "#{ENV['HOME']}/.config/compass-ci/defaults/*.yaml"]).each do |file| + hash.update YAML.load_file(file) || {} + end + hash +end + def valid_shell_variable?(key) key =~ /^[a-zA-Z_]+[a-zA-Z0-9_]*$/ end diff --git a/sbin/submit b/sbin/submit index 39cdcf576..e9e10d903 100755 --- a/sbin/submit +++ b/sbin/submit @@ -17,6 +17,7 @@ opt_auto_define_files = false opt_monitor = false opt_monitor_query = {} opt_my_queue = false +opt_upload_files = [] actions = ['output', 'stop'] options = OptionParser.new do |opts| @@ -64,10 +65,40 @@ options = OptionParser.new do |opts| opt_my_queue = true end + opts.on('-f FILE', '--upload-file FILE', "upload an XML template of libvirt. File: *.xml") do |file| + unless File.exist? file + puts "Find no file: #{file}" + exit 1 + end + + opt_upload_files << file + end + end options.parse!(ARGV) +def upload_configs(files) + defaults = cci_defaults() + upload_host = defaults['LKP_SERVER'] + upload_port = defaults['UPLOAD_LIBVIRT_PORT'] || 3081 + + files.each do |file| + response = `curl -sSf -T #{file} http://#{upload_host}:#{upload_port}/` + if $? == 0 + puts response + else + # TODO: how to deal with uploaded file when one failing + puts response + exit 1 + end + end + + exit 0 +end + +upload_configs(opt_upload_files) + seen_job_yaml = false ARGV.delete_if do |arg| if arg.index '=' -- 2.23.0

2 3

[PATCH v2 compass-ci] lib/job.cr: add log when check account failed
by Wu Zhende 21 Jan '21

21 Jan '21

[Why] The information about the job that fails the account verification can be traced on kibana. [Log] {"level_num":2,"level":"WARN","time":"2021-01-21T11:28:25.476+0800", "msg":"Invalid account","my_email":"wuzhende666(a)163.com","my_name":"wuzhende","suite":"iperf","testbox":vm-2p8g"} Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/job.cr | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 17a0f1b..9b52d88 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -10,6 +10,7 @@ require "scheduler/jobfile_operate.cr" require "scheduler/kernel_params.cr" require "scheduler/pp_params.cr" require "../scheduler/elasticsearch_client" +require "./json_logger" struct JSON::Any def []=(key : String, value : String) @@ -53,6 +54,7 @@ class Job @hash = job_content.as_h @es = Elasticsearch::Client.new @account_info = Hash(String, JSON::Any).new + @log = JSONLogger.new # init job with "-1", or use the original job_content["id"] id = "-1" if "#{id}" == "" @@ -394,17 +396,30 @@ class Job error_msg += "Please refer to https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/apply-acco…" account_info = @es.get_account(self["my_email"]) - raise error_msg unless account_info.is_a?(JSON::Any) + + flag = is_valid_account?(account_info) + @log.warn({"msg" => "Invalid account", + "my_email" => self["my_email"], + "my_name" => self["my_name"], + "suite" => self["suite"], + "testbox" => self["testbox"] + }.to_json) unless flag + raise error_msg unless flag + + @hash.delete("my_uuid") + @hash.delete("my_token") + end + + private def is_valid_account?(account_info) + return false unless account_info.is_a?(JSON::Any) @account_info = account_info.as_h # my_name can be nil in es # my_token can't be nil in es - raise error_msg unless self["my_name"] == @account_info["my_name"]?.to_s - raise error_msg unless self["my_token"] == @account_info["my_token"]? - - @hash.delete("my_uuid") - @hash.delete("my_token") + return false unless self["my_name"] == @account_info["my_name"]?.to_s + return false unless self["my_token"] == @account_info["my_token"]? + return true end private def get_initialized_keys -- 2.23.0

1 0

[PATCH v2 lkp-tests] sbin/submit: add -f option for uploading libvirt config
by Ren Wen 21 Jan '21

21 Jan '21

An option for 'submit': -f, --upload-file FILE upload an XML template of libvirt FILE's extension should be '.xml'. Example: case1: submit -f existent_file.xml => existent_file/<md5sum>.xml case2: submit -f nonexistent_file.xml => Find no file: nonexistent_file.xml Signed-off-by: Ren Wen <15991987063(a)163.com> --- lib/job.rb | 11 +++++++++++ sbin/submit | 31 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/lib/job.rb b/lib/job.rb index 631ebfc98..812f8c299 100755 --- a/lib/job.rb +++ b/lib/job.rb @@ -16,6 +16,17 @@ require 'set' require 'pp' require 'English' +# copied from compass-ci/container/defconfig.rb +def cci_defaults + hash = {} + Dir.glob(['/etc/compass-ci/defaults/*.yaml', + '/etc/compass-ci/accounts/*.yaml', + "#{ENV['HOME']}/.config/compass-ci/defaults/*.yaml"]).each do |file| + hash.update YAML.load_file(file) || {} + end + hash +end + def restore(ah, copy) if ah.class == Hash ah.clear.merge!(copy) diff --git a/sbin/submit b/sbin/submit index 39cdcf576..e9e10d903 100755 --- a/sbin/submit +++ b/sbin/submit @@ -17,6 +17,7 @@ opt_auto_define_files = false opt_monitor = false opt_monitor_query = {} opt_my_queue = false +opt_upload_files = [] actions = ['output', 'stop'] options = OptionParser.new do |opts| @@ -64,10 +65,40 @@ options = OptionParser.new do |opts| opt_my_queue = true end + opts.on('-f FILE', '--upload-file FILE', "upload an XML template of libvirt. File: *.xml") do |file| + unless File.exist? file + puts "Find no file: #{file}" + exit 1 + end + + opt_upload_files << file + end + end options.parse!(ARGV) +def upload_configs(files) + defaults = cci_defaults() + upload_host = defaults['LKP_SERVER'] + upload_port = defaults['UPLOAD_LIBVIRT_PORT'] || 3081 + + files.each do |file| + response = `curl -sSf -T #{file} http://#{upload_host}:#{upload_port}/` + if $? == 0 + puts response + else + # TODO: how to deal with uploaded file when one failing + puts response + exit 1 + end + end + + exit 0 +end + +upload_configs(opt_upload_files) + seen_job_yaml = false ARGV.delete_if do |arg| if arg.index '=' -- 2.23.0

1 1

[PATCH compass-ci] lib/job.cr: add log when check account failed
by Wu Zhende 21 Jan '21

21 Jan '21

[Why] The information about the job that fails the account verification can be traced on kibana. [Log] {"level_num":2,"level":"WARN","time":"2021-01-21T11:08:19.647+0800","msg":"Invalid account","my_email":"wuzhende666(a)163.com","suite":"iperf","testcase":"iperf"} Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/job.cr | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/lib/job.cr b/src/lib/job.cr index 17a0f1b..2971a77 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -10,6 +10,7 @@ require "scheduler/jobfile_operate.cr" require "scheduler/kernel_params.cr" require "scheduler/pp_params.cr" require "../scheduler/elasticsearch_client" +require "./json_logger" struct JSON::Any def []=(key : String, value : String) @@ -53,6 +54,7 @@ class Job @hash = job_content.as_h @es = Elasticsearch::Client.new @account_info = Hash(String, JSON::Any).new + @log = JSONLogger.new # init job with "-1", or use the original job_content["id"] id = "-1" if "#{id}" == "" @@ -394,17 +396,29 @@ class Job error_msg += "Please refer to https://gitee.com/wu_fengguang/compass-ci/blob/master/doc/manual/apply-acco…" account_info = @es.get_account(self["my_email"]) - raise error_msg unless account_info.is_a?(JSON::Any) + + flag = is_valid_account?(account_info) + @log.warn({"msg" => "Invalid account", + "my_email" => self["my_email"], + "suite" => self["suite"], + "testcase" => self["testcase"] + }.to_json) unless flag + raise error_msg unless flag + + @hash.delete("my_uuid") + @hash.delete("my_token") + end + + private def is_valid_account?(account_info) + return false unless account_info.is_a?(JSON::Any) @account_info = account_info.as_h # my_name can be nil in es # my_token can't be nil in es - raise error_msg unless self["my_name"] == @account_info["my_name"]?.to_s - raise error_msg unless self["my_token"] == @account_info["my_token"]? - - @hash.delete("my_uuid") - @hash.delete("my_token") + return false unless self["my_name"] == @account_info["my_name"]?.to_s + return false unless self["my_token"] == @account_info["my_token"]? + return true end private def get_initialized_keys -- 2.23.0

1 0

[PATCH compass-ci] git_mirror.rb: add new_refs_count by year/month/day
by Li Yuanchao 21 Jan '21

21 Jan '21

Add a hash table new_refs_count in fork_stat for every repo, which will record new refs times of this repo by year/month/day. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 4f74751..6d1624f 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -298,6 +298,7 @@ class MirrorMain offset_new_refs = 0 if offset_new_refs >= 10 @fork_stat[git_repo][:new_refs_time][offset_new_refs] = Time.now.to_s @fork_stat[git_repo][:offset_new_refs] = offset_new_refs + 1 + @fork_stat[git_repo][:new_refs_count] = update_new_refs_count(@fork_stat[git_repo][:new_refs_count]) end def update_fork_stat(git_repo, possible_new_refs) @@ -373,15 +374,53 @@ class MirrorMain fetch_time: [], offset_fetch: 0, new_refs_time: [], - offset_new_refs: 0 + offset_new_refs: 0, + new_refs_count: {} } query = { query: { match: { git_repo: git_repo } } } result = @es_client.search(index: 'repo', body: query)['hits'] return fork_stat unless result['total'].positive? fork_stat.each_key do |key| - fork_stat[key] = result['hits'][0]['_source'][key.to_s] + fork_stat[key] = result['hits'][0]['_source'][key.to_s] || fork_stat[key] end return fork_stat end + + def create_year_hash(new_refs_count, year, month, day) + new_refs_count[year] = { 'year_count' => 1, month => { 'month_count' => 1, day => 1 } } + return new_refs_count + end + + def update_year_hash(new_refs_count, year, month, day) + new_refs_count[year]['year_count'] += 1 + return create_month_hash(new_refs_count, year, month, day) if new_refs_count[year][month].nil? + + return update_month_hash(new_refs_count, year, month, day) + end + + def create_month_hash(new_refs_count, year, month, day) + new_refs_count[year][month] = { 'month_count' => 1, day => 1 } + return new_refs_count + end + + def update_month_hash(new_refs_count, year, month, day) + new_refs_count[year][month]['month_count'] += 1 + if new_refs_count[year][month][day].nil? + new_refs_count[year][month][day] = 1 + else + new_refs_count[year][month][day] += 1 + end + return new_refs_count + end + + def update_new_refs_count(new_refs_count) + t = Time.now + year = t.year.to_s + month = t.month.to_s + day = t.day.to_s + return create_year_hash(new_refs_count, year, month, day) if new_refs_count[year].nil? + + return update_year_hash(new_refs_count, year, month, day) + end end -- 2.23.0

1 0