March 2021 - Compass-ci - mailweb.openeuler.org

[PATCH v3 compass-ci 2/2] git_mirror.rb: reload more internal files when DEFAULTS change
by Li Yuanchao 02 Mar '21

02 Mar '21

DEFAULTS files are used by many files in the inner directory, so when DEFAULTS files change, those inner files should also be reload. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index bad73d5..bb0c9f9 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -285,10 +285,11 @@ class MirrorMain def reload(file_list, belong) system("git -C #{REPO_DIR}/#{belong} pull") + reload_defaults(file_list, belong) file_list.each_line do |file| - next if File.basename(file) == '.ignore' - file = file.chomp + next if File.basename(file) == '.ignore' || File.basename(file) == 'DEFAULTS' + repo_dir = "#{REPO_DIR}/#{belong}/#{file}" load_repo_file(repo_dir, File.dirname(file), File.basename(file), belong) if File.file?(repo_dir) end @@ -497,4 +498,15 @@ class MirrorMain end return false end + + def reload_defaults(file_list, belong) + file_list.each_line do |file| + file = file.chomp + next unless File.basename(file) == 'DEFAULTS' + + repodir = "#{REPO_DIR}/#{belong}/#{File.dirname(file)}" + load_defaults(repodir, belong) + traverse_repodir(repodir, belong) + end + end end -- 2.23.0

1 0

[PATCH v3 compass-ci 1/2] lib/git_mirror.rb: support more than one upstream repos warehouse
by Li Yuanchao 02 Mar '21

02 Mar '21

Support monitor repos in upstream-repos/ansible-repos/dockerfile-repos/pkgbuild-repos, they are just like upstream-repos. And which to monitor can be configured in file upstream-config. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 89 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 29 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 22c4e07..bad73d5 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -81,7 +81,7 @@ class GitMirror def mirror_sync fork_info = @queue.pop - mirror_dir = "/srv/git/#{fork_info['git_repo']}" + mirror_dir = "/srv/git/#{fork_info['belong']}/#{fork_info['git_repo']}" mirror_dir = "#{mirror_dir}.git" unless fork_info['is_submodule'] possible_new_refs = git_repo_download(fork_info['url'], mirror_dir) feedback(fork_info['git_repo'], possible_new_refs) @@ -107,6 +107,7 @@ class MirrorMain @defaults = {} @git_queue = Queue.new @es_client = Elasticsearch::Client.new(url: "http://#{ES_HOST}:#{ES_PORT}") + clone_upstream_repo load_fork_info connection_init handle_webhook @@ -124,35 +125,37 @@ class MirrorMain @fork_stat[git_repo] = get_fork_stat(git_repo) end - def load_defaults(repodir) + def load_defaults(repodir, belong) defaults_file = "#{repodir}/DEFAULTS" return unless File.exist?(defaults_file) - defaults_key = repodir == REPO_DIR ? 'default' : repodir.delete_prefix("#{REPO_DIR}/") + defaults_key = repodir == "#{REPO_DIR}/#{belong}" ? belong : repodir.delete_prefix("#{REPO_DIR}/#{belong}/") @defaults[defaults_key] = YAML.safe_load(File.open(defaults_file)) - @defaults[defaults_key] = merge_defaults(defaults_key, @defaults[defaults_key]) + @defaults[defaults_key] = merge_defaults(defaults_key, @defaults[defaults_key], belong) end - def traverse_repodir(repodir) + def traverse_repodir(repodir, belong) if File.directory? repodir - load_defaults(repodir) + load_defaults(repodir, belong) entry_list = Dir.children(repodir) - ['DEFAULTS'] entry_list.each do |entry| next if entry.start_with? '.' - traverse_repodir("#{repodir}/#{entry}") + traverse_repodir("#{repodir}/#{entry}", belong) end else return if File.dirname(repodir) == REPO_DIR - project = File.dirname(repodir).delete_prefix("#{REPO_DIR}/") + project = File.dirname(repodir).delete_prefix("#{REPO_DIR}/#{belong}/") fork_name = File.basename(repodir) - load_repo_file(repodir, project, fork_name) + load_repo_file(repodir, project, fork_name, belong) end end def load_fork_info - traverse_repodir(REPO_DIR) + @upstreams['upstreams'].each do |repo| + traverse_repodir("#{REPO_DIR}/#{repo['location']}", repo['location']) + end end def create_workers @@ -182,7 +185,7 @@ class MirrorMain update_fork_stat(git_repo, feedback_info[:possible_new_refs]) return unless feedback_info[:possible_new_refs] - return reload_fork_info if git_repo == 'u/upstream-repos/upstream-repos' + return reload_fork_info(git_repo) if is_upstream_repo?(git_repo) new_refs = check_new_refs(git_repo) return if new_refs[:heads].empty? @@ -221,14 +224,16 @@ end # main thread class MirrorMain - def load_repo_file(repodir, project, fork_name) + def load_repo_file(repodir, project, fork_name, belong) git_repo = "#{project}/#{fork_name}" git_info = YAML.safe_load(File.open(repodir)) return if git_info.nil? || git_info['url'].nil? @git_info[git_repo] = git_info @git_info[git_repo]['git_repo'] = git_repo - @git_info[git_repo] = merge_defaults(git_repo, @git_info[git_repo]) + @git_info[git_repo]['belong'] = belong + @git_info[git_repo] = merge_defaults(git_repo, @git_info[git_repo], belong) + fork_stat_init(git_repo) @priority_queue.push git_repo, @priority @priority += 1 @@ -247,7 +252,7 @@ class MirrorMain def get_cur_refs(git_repo) return if @git_info[git_repo]['is_submodule'] - mirror_dir = "/srv/git/#{git_repo}.git" + mirror_dir = "/srv/git/#{@git_info[git_repo]['belong']}/#{git_repo}.git" show_ref_out = %x(git -C #{mirror_dir} show-ref --heads 2>/dev/null) cur_refs = { heads: {} } show_ref_out.each_line do |line| @@ -266,27 +271,26 @@ class MirrorMain return new_refs end - def reload_fork_info - upstream_repos = 'u/upstream-repos/upstream-repos' + def reload_fork_info(upstream_repos) if @git_info[upstream_repos][:cur_refs].empty? @git_info[upstream_repos][:cur_refs] = get_cur_refs(upstream_repos) else old_commit = @git_info[upstream_repos][:cur_refs][:heads]['refs/heads/master'] new_refs = check_new_refs(upstream_repos) new_commit = new_refs[:heads]['refs/heads/master'] - changed_files = %x(git -C /srv/git/#{upstream_repos}.git diff --name-only #{old_commit}...#{new_commit}) - reload(changed_files) + changed_files = %x(git -C /srv/git/#{@git_info[upstream_repos]['belong']}/#{upstream_repos}.git diff --name-only #{old_commit}...#{new_commit}) + reload(changed_files, @git_info[upstream_repos]['belong']) end end - def reload(file_list) - system("git -C #{REPO_DIR} pull") + def reload(file_list, belong) + system("git -C #{REPO_DIR}/#{belong} pull") file_list.each_line do |file| next if File.basename(file) == '.ignore' file = file.chomp - repo_dir = "#{REPO_DIR}/#{file}" - load_repo_file(repo_dir, File.dirname(file), File.basename(file)) if File.file?(repo_dir) + repo_dir = "#{REPO_DIR}/#{belong}/#{file}" + load_repo_file(repo_dir, File.dirname(file), File.basename(file), belong) if File.file?(repo_dir) end end @@ -357,7 +361,7 @@ class MirrorMain end end - def handle_submodule(submodule) + def handle_submodule(submodule, belong) submodule.each_line do |line| next unless line.include?('url = ') @@ -365,7 +369,7 @@ class MirrorMain git_repo = url.split('://')[1] if url.include?('://') break unless git_repo - @git_info[git_repo] = { 'url' => url, 'git_repo' => git_repo, 'is_submodule' => true } + @git_info[git_repo] = { 'url' => url, 'git_repo' => git_repo, 'is_submodule' => true, 'belong' => belong } fork_stat_init(git_repo) @priority_queue.push git_repo, @priority @priority += 1 @@ -378,11 +382,11 @@ class MirrorMain return true end - mirror_dir = "/srv/git/#{git_repo}.git" + mirror_dir = "/srv/git/#{@git_info[git_repo]['belong']}/#{git_repo}.git" submodule = %x(git -C #{mirror_dir} show HEAD:.gitmodules 2>/dev/null) return if submodule.empty? - handle_submodule(submodule) + handle_submodule(submodule, @git_info[git_repo]['belong']) end def get_fork_stat(git_repo) @@ -453,8 +457,8 @@ end # main thread class MirrorMain - def merge_defaults(object_key, object) - return object if object_key == 'default' + def merge_defaults(object_key, object, belong) + return object if object_key == belong defaults_key = File.dirname(object_key) while defaults_key != '.' @@ -462,8 +466,35 @@ class MirrorMain defaults_key = File.dirname(defaults_key) end - return @defaults['default'].merge(object) if @defaults['default'] + return @defaults[belong].merge(object) if @defaults[belong] return object end + + def clone_upstream_repo + if File.exist?('/etc/compass-ci/defaults/upstream-config') + @upstreams = YAML.safe_load(File.open('/etc/compass-ci/defaults/upstream-config')) + @upstreams['upstreams'].each do |repo| + url = get_url(repo['url']) + %x(git clone #{url} #{REPO_DIR}/#{repo['location']} 2>&1) + end + else + puts 'ERROR: No upstream-config file' + return -1 + end + end + + def get_url(url) + if url.include?('gitee.com/') && File.exist?("/srv/git/#{url.delete_prefix('https://')}") + url = "/srv/git/#{url.delete_prefix('https://')}" + end + return url + end + + def is_upstream_repo?(git_repo) + @upstreams['upstreams'].each do |repo| + return true if git_repo == repo['git_repo'] + end + return false + end end -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] modify all SRV_HTTP_HOST/PORT
by Lu Weitao 02 Mar '21

02 Mar '21

[Why] split compass-ci/container/srv-http to 6 service pre dir pre service with a port [How] SRV_HTTP_PORT => SRV_HTTP_RESULT_HOST/PORT, user: compass-ci/lib/mail_job_result.rb compass-ci/container/extract-stats SRV_HTTP_OS_HOST/PORT, user: compass-ci/sbin/download-rootfs SRV_HTTP_GIT_HOST/PORT, user: compass-ci/lib/mail_bisect_result.rb SRV_HTTP_CCI_HOST/PORT, user: compass-ci/lib/compass-ci/src/scheduler/find_job_boot.cr compass-ci/providers/lib/domain.rb add them(HOST/PORT) to: compass-ci/lib/constants.rb compass-ci/sparrow/3-code/dev-env Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/extract-stats/start | 12 ++++++------ container/scheduler/start | 6 +++--- lib/constants.rb | 8 ++++++-- lib/mail_job_result.rb | 2 +- providers/lib/domain.rb | 4 ++-- sbin/download-rootfs | 2 +- sparrow/3-code/dev-env | 24 ++++++++++++++++++------ src/scheduler/constants.cr | 4 ++-- src/scheduler/find_job_boot.cr | 4 ++-- 9 files changed, 41 insertions(+), 25 deletions(-) diff --git a/container/extract-stats/start b/container/extract-stats/start index c158977..5033754 100755 --- a/container/extract-stats/start +++ b/container/extract-stats/start @@ -14,8 +14,8 @@ names = Set.new %w[ ES_PORT SEND_MAIL_HOST SEND_MAIL_PORT - SRV_HTTP_HOST - SRV_HTTP_PORT + SRV_HTTP_RESULT_HOST + SRV_HTTP_RESULT_PORT lab ] @@ -24,8 +24,8 @@ env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' DEFAULT_CCI = '/c/compass-ci' -SRV_HTTP_HOST = defaults['SRV_HTTP_HOST'] || Open3.capture3('curl ip.sb')[0].chomp -SRV_HTTP_PORT = defaults['SRV_HTTP_PORT'] || '11300' +SRV_HTTP_RESULT_HOST = defaults['SRV_HTTP_RESULT_HOST'] || Open3.capture3('curl ip.sb')[0].chomp +SRV_HTTP_RESULT_PORT = defaults['SRV_HTTP_RESULT_PORT'] || '11300' docker_rm "extract-stats" cmd = %w[ @@ -36,8 +36,8 @@ cmd = %w[ ] + env + %W[ -e LKP_SRC=#{DEFAULT_LKP} -e CCI_SRC=#{DEFAULT_CCI} - -e SRV_HTTP_HOST=#{SRV_HTTP_HOST} - -e SRV_HTTP_PORT=#{SRV_HTTP_PORT} + -e SRV_HTTP_RESULT_HOST=#{SRV_HTTP_RESULT_HOST} + -e SRV_HTTP_RESULT_PORT=#{SRV_HTTP_RESULT_PORT} -v #{ENV['LKP_SRC']}:#{DEFAULT_LKP} -v /etc/localtime:/etc/localtime:ro -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} diff --git a/container/scheduler/start b/container/scheduler/start index 4aaa71d..d83fcc2 100755 --- a/container/scheduler/start +++ b/container/scheduler/start @@ -25,8 +25,8 @@ names = Set.new %w[ SSHR_PORT SSHR_PORT_BASE SSHR_PORT_LEN - SRV_HTTP_HOST - SRV_HTTP_PORT + SRV_HTTP_CCI_HOST + SRV_HTTP_CCI_PORT lab ] @@ -37,7 +37,7 @@ defaults['SSHR_PORT_LEN'] ||= 2000 defaults['SCHED_PORT'] ||= '3000' defaults['SCHED_HOST'] ||= '172.17.0.1' defaults['MASTER_FLUENTD_PORT'] ||= '24224' -defaults['SRV_HTTP_PORT'] ||= '11300' +defaults['SRV_HTTP_CCI_PORT'] ||= '11306' env = docker_env(defaults) CCI_REPOS = ENV['CCI_REPOS'] || '/c' diff --git a/lib/constants.rb b/lib/constants.rb index 5ed4d57..8d8e488 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -11,8 +11,12 @@ ES_PORT ||= config['ES_PORT'] || 9200 SEND_MAIL_HOST ||= config['SEND_MAIL_HOST'] || '172.17.0.1' SEND_MAIL_PORT ||= config['SEND_MAIL_PORT'] || 49000 -SRV_HTTP_HOST ||= config['SRV_HTTP_HOST'] || ENV['SRV_HTTP_HOST'] || '172.17.0.1' -SRV_HTTP_PORT ||= config['SRV_HTTP_PORT'] || ENV['SRV_HTTP_PORT'] || 11300 +SRV_HTTP_RESULT_HOST ||= config['SRV_HTTP_RESULT_HOST'] || ENV['SRV_HTTP_RESULT_HOST'] || '172.17.0.1' +SRV_HTTP_OS_HOST ||= config['SRV_HTTP_OS_HOST'] || ENV['SRV_HTTP_OS_HOST'] || '172.17.0.1' +SRV_HTTP_GIT_HOST ||= config['SRV_HTTP_GIT_HOST'] || ENV['SRV_HTTP_GIT_HOST'] || '172.17.0.1' +SRV_HTTP_RESULT_PORT ||= config['SRV_HTTP_RESULT_PORT'] || ENV['SRV_HTTP_RESULT_PORT'] || 11300 +SRV_HTTP_OS_PORT ||= config['SRV_HTTP_OS_PORT'] || ENV['SRV_HTTP_OS_PORT'] || 11304 +SRV_HTTP_GIT_PORT ||= config['SRV_HTTP_GIT_PORT'] || ENV['SRV_HTTP_GIT_PORT'] || 11305 SRV_HTTP_DOMAIN ||= config['SRV_HTTP_DOMAIN'] || ENV['SRV_HTTP_DOMAIN'] || 'api.compass-ci.openeuler.org' diff --git a/lib/mail_job_result.rb b/lib/mail_job_result.rb index 796a4f3..e588470 100644 --- a/lib/mail_job_result.rb +++ b/lib/mail_job_result.rb @@ -9,7 +9,7 @@ require 'json' # compose and send email for job result class MailJobResult - def initialize(job_id, result_host = SRV_HTTP_HOST, result_port = SRV_HTTP_PORT) + def initialize(job_id, result_host = SRV_HTTP_RESULT_HOST, result_port = SRV_HTTP_RESULT_PORT) @job_id = job_id @result_host = result_host @result_port = result_port diff --git a/providers/lib/domain.rb b/providers/lib/domain.rb index 8df9c06..318e68d 100644 --- a/providers/lib/domain.rb +++ b/providers/lib/domain.rb @@ -46,8 +46,8 @@ class Domain end def load_xml_file(filepath) - host = @context.info['SRV_HTTP_HOST'] - port = @context.info['SRV_HTTP_PORT'] + host = @context.info['SRV_HTTP_CCI_HOST'] + port = @context.info['SRV_HTTP_CCI_PORT'] system "wget --timestamping --progress=bar:force http://#{host}:#{port}/cci/libvirt-xml/#{filepath}" %x(basename #{filepath}).chomp end diff --git a/sbin/download-rootfs b/sbin/download-rootfs index e1d4091..74afbf6 100755 --- a/sbin/download-rootfs +++ b/sbin/download-rootfs @@ -25,7 +25,7 @@ download_os_rootfs() mkdir -p $rootfs_dir cd $rootfs_dir - download_url="http://api.compass-ci.openeuler.org:${SRV_HTTP_PORT}/os/${os}/${os_arch}/${…" + download_url="http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" wget $download_url [ "$?" -ne 0 ] && { diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index cc94e01..698a4a2 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -16,8 +16,14 @@ server_name=$(hostname | cut -f1 -d.) : ${OS_HTTP_HOST:=$server_ip} : ${SEND_MAIL_HOST:=$server_ip} : ${SEND_MAIL_PORT:=49000} -: ${SRV_HTTP_HOST:=$server_ip} -: ${SRV_HTTP_PORT:=11300} +: ${SRV_HTTP_RESULT_HOST:=$server_ip} +: ${SRV_HTTP_OS_HOST:=$server_ip} +: ${SRV_HTTP_GIT_HOST:=$server_ip} +: ${SRV_HTTP_CCI_HOST:=$server_ip} +: ${SRV_HTTP_RESULT_PORT:=11300} +: ${SRV_HTTP_OS_PORT:=11304} +: ${SRV_HTTP_GIT_PORT:=11305} +: ${SRV_HTTP_CCI_PORT:=11306} : ${LOGGING_ES_HOST:=$server_ip} : ${LOGGING_ES_PORT:=9202} : ${INITRD_HTTP_HOST:=$server_ip} @@ -49,8 +55,14 @@ GIT_SERVER: $GIT_SERVER OS_HTTP_HOST: $OS_HTTP_HOST GITCACHE_HOST: $gitcache_host GITCACHE_PORT: $gitcache_port -SRV_HTTP_HOST: $SRV_HTTP_HOST -SRV_HTTP_PORT: $SRV_HTTP_PORT +SRV_HTTP_RESULT_HOST: $SRV_HTTP_RESULT_HOST +SRV_HTTP_OS_HOST: $SRV_HTTP_OS_HOST +SRV_HTTP_GIT_HOST: $SRV_HTTP_GIT_HOST +SRV_HTTP_CCI_HOST: $SRV_HTTP_CCI_HOST +SRV_HTTP_RESULT_PORT: $SRV_HTTP_RESULT_PORT +SRV_HTTP_OS_PORT: $SRV_HTTP_OS_PORT +SRV_HTTP_GIT_PORT: $SRV_HTTP_GIT_PORT +SRV_HTTP_CCI_PORT: $SRV_HTTP_CCI_PORT TASKQUEUE_HOST: $sched_host SEND_MAIL_HOST: $SEND_MAIL_HOST SEND_MAIL_PORT: $SEND_MAIL_PORT @@ -77,8 +89,8 @@ LKP_SERVER: GIT_SERVER: GITCACHE_HOST: GITCACHE_PORT: -SRV_HTTP_HOST: -SRV_HTTP_PORT: +SRV_HTTP_CCI_HOST: +SRV_HTTP_CCI_PORT: SEND_MAIL_HOST: SEND_MAIL_PORT: INITRD_HTTP_HOST: diff --git a/src/scheduler/constants.cr b/src/scheduler/constants.cr index 668da6d..d7d7a9a 100644 --- a/src/scheduler/constants.cr +++ b/src/scheduler/constants.cr @@ -23,8 +23,8 @@ INITRD_HTTP_PORT = (ENV.has_key?("INITRD_HTTP_PORT") ? ENV["INITRD_HTTP_PORT"] : OS_HTTP_HOST = (ENV.has_key?("OS_HTTP_HOST") ? ENV["OS_HTTP_HOST"] : "172.17.0.1") OS_HTTP_PORT = (ENV.has_key?("OS_HTTP_PORT") ? ENV["OS_HTTP_PORT"] : 8000).to_i32 -SRV_HTTP_HOST = (ENV.has_key?("SRV_HTTP_HOST") ? ENV["SRV_HTTP_HOST"] : "172.17.0.1") -SRV_HTTP_PORT = (ENV.has_key?("SRV_HTTP_PORT") ? ENV["SRV_HTTP_PORT"] : "11300") +SRV_HTTP_CCI_HOST = (ENV.has_key?("SRV_HTTP_CCI_HOST") ? ENV["SRV_HTTP_CCI_HOST"] : "172.17.0.1") +SRV_HTTP_CCI_PORT = (ENV.has_key?("SRV_HTTP_CCI_PORT") ? ENV["SRV_HTTP_CCI_PORT"] : "11306") CCI_REPOS = (ENV.has_key?("CCI_REPOS") ? ENV["CCI_REPOS"] : "/c") LAB_REPO = "lab-z9" diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 8dd22a8..3323d4d 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -191,8 +191,8 @@ class Sched "LKP_SERVER" => job["LKP_SERVER"]?, "vt" => _vt, "RESULT_WEBDAV_PORT" => job["RESULT_WEBDAV_PORT"]? || "3080", - "SRV_HTTP_HOST" => SRV_HTTP_HOST, - "SRV_HTTP_PORT" => SRV_HTTP_PORT, + "SRV_HTTP_CCI_HOST" => SRV_HTTP_CCI_HOST, + "SRV_HTTP_CCI_PORT" => SRV_HTTP_CCI_PORT, }.to_json end -- 2.23.0

1 0

[PATCH v2 compass-ci 1/2] container/srv-http: split up service
by Lu Weitao 02 Mar '21

02 Mar '21

[Why] This service ties up different dirs to the same service. However we'll need the flexibility to hold the dirs in different file servers. [How] one service per dir, and access-like: result: http://$host:11300/ pub: http://$host:11302/ initrd: http://$host:11303/ os: http://$host:11304/ git: http://$host:11305/ cci: http://$host:11306/ Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/srv-http/start | 27 ++++++--------------------- container/srv-http/start-cci | 19 +++++++++++++++++++ container/srv-http/start-git | 19 +++++++++++++++++++ container/srv-http/start-initrd | 20 ++++++++++++++++++++ container/srv-http/start-os | 19 +++++++++++++++++++ container/srv-http/start-pub | 19 +++++++++++++++++++ container/srv-http/start-result | 19 +++++++++++++++++++ 7 files changed, 121 insertions(+), 21 deletions(-) create mode 100755 container/srv-http/start-cci create mode 100755 container/srv-http/start-git create mode 100755 container/srv-http/start-initrd create mode 100755 container/srv-http/start-os create mode 100755 container/srv-http/start-pub create mode 100755 container/srv-http/start-result diff --git a/container/srv-http/start b/container/srv-http/start index fa56fca..f262d81 100755 --- a/container/srv-http/start +++ b/container/srv-http/start @@ -2,24 +2,9 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -. $CCI_SRC/container/defconfig.sh -docker_rm srv-http - -cmd=( - docker run - --restart=always - --name srv-http - -p 11300:11300 - -v /srv/pub:/srv/pub:ro - -v /etc/localtime:/etc/localtime:ro - -v /srv/result:/srv/result:ro - -v /srv/initrd/qemu-image:/srv/initrd/qemu-image:ro - -v /srv/initrd/dockerimage:/srv/initrd/dockerimage:ro - -v /srv/os:/srv/os:ro - -v /srv/git/archlinux:/srv/git/archlinux:ro - -v /srv/cci/libvirt-xml:/srv/cci/libvirt-xml:ro - -d - srv-http -) - -"${cmd[@]}" +./start-pub +./start-result +./start-initrd +./start-os +./start-git +./start-cci diff --git a/container/srv-http/start-cci b/container/srv-http/start-cci new file mode 100755 index 0000000..78ab149 --- /dev/null +++ b/container/srv-http/start-cci @@ -0,0 +1,19 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. $CCI_SRC/container/defconfig.sh +docker_rm srv-http-cci + +cmd=( + docker run + --restart=always + --name srv-http-cci + -p 11306:11300 + -v /srv/cci/libvirt-xml:/srv/cci/libvirt-xml:ro + -v /etc/localtime:/etc/localtime:ro + -d + srv-http +) + +"${cmd[@]}" diff --git a/container/srv-http/start-git b/container/srv-http/start-git new file mode 100755 index 0000000..f2292bf --- /dev/null +++ b/container/srv-http/start-git @@ -0,0 +1,19 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. $CCI_SRC/container/defconfig.sh +docker_rm srv-http-git + +cmd=( + docker run + --restart=always + --name srv-http-git + -p 11305:11300 + -v /srv/git/archlinux:/srv/git/archlinux:ro + -v /etc/localtime:/etc/localtime:ro + -d + srv-http +) + +"${cmd[@]}" diff --git a/container/srv-http/start-initrd b/container/srv-http/start-initrd new file mode 100755 index 0000000..7c73ea1 --- /dev/null +++ b/container/srv-http/start-initrd @@ -0,0 +1,20 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. $CCI_SRC/container/defconfig.sh +docker_rm srv-http-initrd + +cmd=( + docker run + --restart=always + --name srv-http-initrd + -p 11303:11300 + -v /srv/initrd/qemu-image:/srv/initrd/qemu-image:ro + -v /srv/initrd/dockerimage:/srv/initrd/dockerimage:ro + -v /etc/localtime:/etc/localtime:ro + -d + srv-http +) + +"${cmd[@]}" diff --git a/container/srv-http/start-os b/container/srv-http/start-os new file mode 100755 index 0000000..bb344cf --- /dev/null +++ b/container/srv-http/start-os @@ -0,0 +1,19 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. $CCI_SRC/container/defconfig.sh +docker_rm srv-http-os + +cmd=( + docker run + --restart=always + --name srv-http-os + -p 11304:11300 + -v /srv/os:/srv/os:ro + -v /etc/localtime:/etc/localtime:ro + -d + srv-http +) + +"${cmd[@]}" diff --git a/container/srv-http/start-pub b/container/srv-http/start-pub new file mode 100755 index 0000000..dc1864c --- /dev/null +++ b/container/srv-http/start-pub @@ -0,0 +1,19 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. $CCI_SRC/container/defconfig.sh +docker_rm srv-http-pub + +cmd=( + docker run + --restart=always + --name srv-http-pub + -p 11302:11300 + -v /srv/pub:/srv/pub:ro + -v /etc/localtime:/etc/localtime:ro + -d + srv-http +) + +"${cmd[@]}" diff --git a/container/srv-http/start-result b/container/srv-http/start-result new file mode 100755 index 0000000..f9ecc97 --- /dev/null +++ b/container/srv-http/start-result @@ -0,0 +1,19 @@ +#!/bin/bash +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +. $CCI_SRC/container/defconfig.sh +docker_rm srv-http-result + +cmd=( + docker run + --restart=always + --name srv-http-result + -p 11300:11300 + -v /srv/result:/srv/result:ro + -v /etc/localtime:/etc/localtime:ro + -d + srv-http +) + +"${cmd[@]}" -- 2.23.0

1 0

[PATCH compass-ci] sbin/compare won't remove job if job['stats'] no metric start with job['suite']
by Lu Weitao 02 Mar '21

02 Mar '21

[Why] before convert job_list => matrix we remove job from job_list unless: job['stats'].any? { |metric| metric.start_with?(job['suite'])} but some job(such as build-pkg) have no metric start_with suite(build-pkg.xxx) so this logic is unreasonable, we should keep the job in job_list Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/params_group.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/params_group.rb b/lib/params_group.rb index 2082c5d..37cf03a 100644 --- a/lib/params_group.rb +++ b/lib/params_group.rb @@ -50,7 +50,7 @@ def job_is_useful?(job) return unless stats suite = job['_source']['suite'] - return unless suite && stats.keys.any? { |stat| stat.start_with?(suite) } + return unless suite true end -- 2.23.0

1 0

[PATCH v2 compass-ci 2/2] git_mirror.rb: reload more internal files when DEFAULTS change
by Li Yuanchao 02 Mar '21

02 Mar '21

DEFAULTS files are used by many files in the inner directory, so when DEFAULTS files change, those inner files should also be reload. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 6417b52..017153f 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -285,10 +285,11 @@ class MirrorMain def reload(file_list, belong) system("git -C #{REPO_DIR}/#{belong} pull") + reload_defaults(file_list, belong) file_list.each_line do |file| - next if File.basename(file) == '.ignore' - file = file.chomp + next if File.basename(file) == '.ignore' || File.basename(file) == 'DEFAULTS' + repo_dir = "#{REPO_DIR}/#{belong}/#{file}" load_repo_file(repo_dir, File.dirname(file), File.basename(file), belong) if File.file?(repo_dir) end @@ -497,4 +498,15 @@ class MirrorMain end return false end + + def reload_defaults(file_list, belong) + file_list.each_line do |file| + file = file.chomp + next unless File.basename(file) == 'DEFAULTS' + + repodir = "#{REPO_DIR}/#{belong}/#{File.dirname(file)}" + load_defaults(repodir, belong) + traverse_repodir(repodir, belong) + end + end end -- 2.23.0

2 2

[PATCH v3 compass-ci 1/2] sched: response ipxe_response value if ipxe_response field in job yaml
by Cao Xueliang 02 Mar '21

02 Mar '21

The install-iso job need scheduler response the ipxe_response value, when consume the install-iso job. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/scheduler/find_job_boot.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 9e2898e..88db69e 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -160,6 +160,8 @@ class Sched end private def get_boot_ipxe(job : Job) + return job["ipxe_response"] if job["suite"] == "install-iso" && job["ipxe_response"]? + response = "#!ipxe\n\n" _initrds_uri = Array(String).from_json(job.initrds_uri).map { |uri| "initrd #{uri}" } -- 2.23.0

2 3

[PATCH compass-ci] modify all SRV_HTTP_PORT
by Lu Weitao 02 Mar '21

02 Mar '21

[Why] split compass-ci/container/srv-http to 6 service pre dir pre service with a port [How] SRV_HTTP_PORT => SRV_HTTP_RESULT_PORT, user: compass-ci/lib/mail_job_result.rb compass-ci/container/extract-stats SRV_HTTP_OS_PORT, user: compass-ci/sbin/download-rootfs SRV_HTTP_GIT_PORT, user: compass-ci/lib/mail_bisect_result.rb SRV_HTTP_CCI_PORT, user: compass-ci/lib/compass-ci/src/scheduler/find_job_boot.cr compass-ci/providers/lib/domain.rb add them(PORT) to: compass-ci/lib/constants.rb compass-ci/sparrow/3-code/dev-env Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- container/extract-stats/start | 6 +++--- container/scheduler/start | 4 ++-- lib/constants.rb | 4 +++- lib/mail_job_result.rb | 2 +- providers/lib/domain.rb | 2 +- sbin/download-rootfs | 2 +- sparrow/3-code/dev-env | 12 +++++++++--- src/scheduler/constants.cr | 2 +- src/scheduler/find_job_boot.cr | 2 +- 9 files changed, 22 insertions(+), 14 deletions(-) diff --git a/container/extract-stats/start b/container/extract-stats/start index c158977..ba259af 100755 --- a/container/extract-stats/start +++ b/container/extract-stats/start @@ -15,7 +15,7 @@ names = Set.new %w[ SEND_MAIL_HOST SEND_MAIL_PORT SRV_HTTP_HOST - SRV_HTTP_PORT + SRV_HTTP_RESULT_PORT lab ] @@ -25,7 +25,7 @@ env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' DEFAULT_CCI = '/c/compass-ci' SRV_HTTP_HOST = defaults['SRV_HTTP_HOST'] || Open3.capture3('curl ip.sb')[0].chomp -SRV_HTTP_PORT = defaults['SRV_HTTP_PORT'] || '11300' +SRV_HTTP_RESULT_PORT = defaults['SRV_HTTP_RESULT_PORT'] || '11300' docker_rm "extract-stats" cmd = %w[ @@ -37,7 +37,7 @@ cmd = %w[ -e LKP_SRC=#{DEFAULT_LKP} -e CCI_SRC=#{DEFAULT_CCI} -e SRV_HTTP_HOST=#{SRV_HTTP_HOST} - -e SRV_HTTP_PORT=#{SRV_HTTP_PORT} + -e SRV_HTTP_RESULT_PORT=#{SRV_HTTP_RESULT_PORT} -v #{ENV['LKP_SRC']}:#{DEFAULT_LKP} -v /etc/localtime:/etc/localtime:ro -v #{ENV['CCI_SRC']}:#{DEFAULT_CCI} diff --git a/container/scheduler/start b/container/scheduler/start index 4aaa71d..d218626 100755 --- a/container/scheduler/start +++ b/container/scheduler/start @@ -26,7 +26,7 @@ names = Set.new %w[ SSHR_PORT_BASE SSHR_PORT_LEN SRV_HTTP_HOST - SRV_HTTP_PORT + SRV_HTTP_CCI_PORT lab ] @@ -37,7 +37,7 @@ defaults['SSHR_PORT_LEN'] ||= 2000 defaults['SCHED_PORT'] ||= '3000' defaults['SCHED_HOST'] ||= '172.17.0.1' defaults['MASTER_FLUENTD_PORT'] ||= '24224' -defaults['SRV_HTTP_PORT'] ||= '11300' +defaults['SRV_HTTP_CCI_PORT'] ||= '11306' env = docker_env(defaults) CCI_REPOS = ENV['CCI_REPOS'] || '/c' diff --git a/lib/constants.rb b/lib/constants.rb index 5ed4d57..431c8d6 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -12,7 +12,9 @@ SEND_MAIL_HOST ||= config['SEND_MAIL_HOST'] || '172.17.0.1' SEND_MAIL_PORT ||= config['SEND_MAIL_PORT'] || 49000 SRV_HTTP_HOST ||= config['SRV_HTTP_HOST'] || ENV['SRV_HTTP_HOST'] || '172.17.0.1' -SRV_HTTP_PORT ||= config['SRV_HTTP_PORT'] || ENV['SRV_HTTP_PORT'] || 11300 +SRV_HTTP_RESULT_PORT ||= config['SRV_HTTP_RESULT_PORT'] || ENV['SRV_HTTP_RESULT_PORT'] || 11300 +SRV_HTTP_OS_PORT ||= config['SRV_HTTP_OS_PORT'] || ENV['SRV_HTTP_OS_PORT'] || 11304 +SRV_HTTP_GIT_PORT ||= config['SRV_HTTP_GIT_PORT'] || ENV['SRV_HTTP_GIT_PORT'] || 11305 SRV_HTTP_DOMAIN ||= config['SRV_HTTP_DOMAIN'] || ENV['SRV_HTTP_DOMAIN'] || 'api.compass-ci.openeuler.org' diff --git a/lib/mail_job_result.rb b/lib/mail_job_result.rb index 796a4f3..e93c6de 100644 --- a/lib/mail_job_result.rb +++ b/lib/mail_job_result.rb @@ -9,7 +9,7 @@ require 'json' # compose and send email for job result class MailJobResult - def initialize(job_id, result_host = SRV_HTTP_HOST, result_port = SRV_HTTP_PORT) + def initialize(job_id, result_host = SRV_HTTP_HOST, result_port = SRV_HTTP_RESULT_PORT) @job_id = job_id @result_host = result_host @result_port = result_port diff --git a/providers/lib/domain.rb b/providers/lib/domain.rb index 8df9c06..b3de90a 100644 --- a/providers/lib/domain.rb +++ b/providers/lib/domain.rb @@ -47,7 +47,7 @@ class Domain def load_xml_file(filepath) host = @context.info['SRV_HTTP_HOST'] - port = @context.info['SRV_HTTP_PORT'] + port = @context.info['SRV_HTTP_CCI_PORT'] system "wget --timestamping --progress=bar:force http://#{host}:#{port}/cci/libvirt-xml/#{filepath}" %x(basename #{filepath}).chomp end diff --git a/sbin/download-rootfs b/sbin/download-rootfs index e1d4091..74afbf6 100755 --- a/sbin/download-rootfs +++ b/sbin/download-rootfs @@ -25,7 +25,7 @@ download_os_rootfs() mkdir -p $rootfs_dir cd $rootfs_dir - download_url="http://api.compass-ci.openeuler.org:${SRV_HTTP_PORT}/os/${os}/${os_arch}/${…" + download_url="http://api.compass-ci.openeuler.org:${SRV_HTTP_OS_PORT}/os/${os}/${os_arch}…" wget $download_url [ "$?" -ne 0 ] && { diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index cc94e01..4e6a4cd 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -17,7 +17,10 @@ server_name=$(hostname | cut -f1 -d.) : ${SEND_MAIL_HOST:=$server_ip} : ${SEND_MAIL_PORT:=49000} : ${SRV_HTTP_HOST:=$server_ip} -: ${SRV_HTTP_PORT:=11300} +: ${SRV_HTTP_RESULT_PORT:=11300} +: ${SRV_HTTP_OS_PORT:=11304} +: ${SRV_HTTP_GIT_PORT:=11305} +: ${SRV_HTTP_CCI_PORT:=11306} : ${LOGGING_ES_HOST:=$server_ip} : ${LOGGING_ES_PORT:=9202} : ${INITRD_HTTP_HOST:=$server_ip} @@ -50,7 +53,10 @@ OS_HTTP_HOST: $OS_HTTP_HOST GITCACHE_HOST: $gitcache_host GITCACHE_PORT: $gitcache_port SRV_HTTP_HOST: $SRV_HTTP_HOST -SRV_HTTP_PORT: $SRV_HTTP_PORT +SRV_HTTP_RESULT_PORT: $SRV_HTTP_RESULT_PORT +SRV_HTTP_OS_PORT: $SRV_HTTP_OS_PORT +SRV_HTTP_GIT_PORT: $SRV_HTTP_GIT_PORT +SRV_HTTP_CCI_PORT: $SRV_HTTP_CCI_PORT TASKQUEUE_HOST: $sched_host SEND_MAIL_HOST: $SEND_MAIL_HOST SEND_MAIL_PORT: $SEND_MAIL_PORT @@ -78,7 +84,7 @@ GIT_SERVER: GITCACHE_HOST: GITCACHE_PORT: SRV_HTTP_HOST: -SRV_HTTP_PORT: +SRV_HTTP_CCI_PORT: SEND_MAIL_HOST: SEND_MAIL_PORT: INITRD_HTTP_HOST: diff --git a/src/scheduler/constants.cr b/src/scheduler/constants.cr index 668da6d..2cb4588 100644 --- a/src/scheduler/constants.cr +++ b/src/scheduler/constants.cr @@ -24,7 +24,7 @@ OS_HTTP_HOST = (ENV.has_key?("OS_HTTP_HOST") ? ENV["OS_HTTP_HOST"] : "172.17.0.1 OS_HTTP_PORT = (ENV.has_key?("OS_HTTP_PORT") ? ENV["OS_HTTP_PORT"] : 8000).to_i32 SRV_HTTP_HOST = (ENV.has_key?("SRV_HTTP_HOST") ? ENV["SRV_HTTP_HOST"] : "172.17.0.1") -SRV_HTTP_PORT = (ENV.has_key?("SRV_HTTP_PORT") ? ENV["SRV_HTTP_PORT"] : "11300") +SRV_HTTP_CCI_PORT = (ENV.has_key?("SRV_HTTP_CCI_PORT") ? ENV["SRV_HTTP_CCI_PORT"] : "11306") CCI_REPOS = (ENV.has_key?("CCI_REPOS") ? ENV["CCI_REPOS"] : "/c") LAB_REPO = "lab-z9" diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 8dd22a8..9d46d72 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -192,7 +192,7 @@ class Sched "vt" => _vt, "RESULT_WEBDAV_PORT" => job["RESULT_WEBDAV_PORT"]? || "3080", "SRV_HTTP_HOST" => SRV_HTTP_HOST, - "SRV_HTTP_PORT" => SRV_HTTP_PORT, + "SRV_HTTP_CCI_PORT" => SRV_HTTP_CCI_PORT, }.to_json end -- 2.23.0

2 2

[PATCH v2 compass-ci 1/2] container/dracut-initrd: add module 90lkp/module-setup.sh
by Xu Xijian 01 Mar '21

01 Mar '21

90lkp module a module to deploy lkp module into rootfs. module-setup.sh is a base setup script for every kernel module. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- .../modules.d/90lkp/module-setup.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 container/dracut-initrd/modules.d/90lkp/module-setup.sh diff --git a/container/dracut-initrd/modules.d/90lkp/module-setup.sh b/container/dracut-initrd/modules.d/90lkp/module-setup.sh new file mode 100755 index 0000000..c188600 --- /dev/null +++ b/container/dracut-initrd/modules.d/90lkp/module-setup.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +check() { + return 0 +} + +depends() { + # We do not depend on any modules - just some root + return 0 +} + +# called by dracut +installkernel() { + return 0 +} + +install() { + inst_hook pre-pivot 10 "$moddir/lkp-deploy.sh" +} -- 2.23.0

1 0

[PATCH compass-ci] container/dracut-initrd: add module 90lkp/module-setup.sh
by Xu Xijian 01 Mar '21

01 Mar '21

90lkp module a module to deploy lkp module into rootfs. module-setup.sh is a base setup script for every kernel module. Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- .../modules.d/90lkp/module-setup.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 container/dracut-initrd/modules.d/90lkp/module-setup.sh diff --git a/container/dracut-initrd/modules.d/90lkp/module-setup.sh b/container/dracut-initrd/modules.d/90lkp/module-setup.sh new file mode 100755 index 0000000..c188600 --- /dev/null +++ b/container/dracut-initrd/modules.d/90lkp/module-setup.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +check() { + return 0 +} + +depends() { + # We do not depend on any modules - just some root + return 0 +} + +# called by dracut +installkernel() { + return 0 +} + +install() { + inst_hook pre-pivot 10 "$moddir/lkp-deploy.sh" +} -- 2.23.0

1 0