[Compass-ci] Re: [PATCH compass-ci 2/2] lib/git_mirror.rb: support more than one upstream repos warehouse

Would you give a bit comment/example on the newly introduced 'belong' field? It looks not obvious enough to me.

Thanks, Fengguang

On Fri, Feb 26, 2021 at 05:33:42PM +0800, Li Yuanchao wrote:

Support monitor repos in upstream-repos/ansible-repos/dockerfile-repos/pkgbuild-repos, they are just like upstream-repos.

And which to monitor can be configured in file upstream-config.

Signed-off-by: Li Yuanchao lyc163mail@163.com

lib/git_mirror.rb | 55 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 15 deletions(-)

diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 22c4e07..4652921 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -81,7 +81,7 @@ class GitMirror

def mirror_sync fork_info = @queue.pop

• mirror_dir = "/srv/git/#{fork_info['git_repo']}"

• mirror_dir = "/srv/git/#{fork_info['belong']}/#{fork_info['git_repo']}" mirror_dir = "#{mirror_dir}.git" unless fork_info['is_submodule'] possible_new_refs = git_repo_download(fork_info['url'], mirror_dir) feedback(fork_info['git_repo'], possible_new_refs)

@@ -107,6 +107,7 @@ class MirrorMain @defaults = {} @git_queue = Queue.new @es_client = Elasticsearch::Client.new(url: "http://#%7BES_HOST%7D:#%7BES_PORT%7D")

• clone_upstream_repo load_fork_info connection_init handle_webhook

@@ -124,35 +125,37 @@ class MirrorMain @fork_stat[git_repo] = get_fork_stat(git_repo) end

• def load_defaults(repodir)

• def load_defaults(repodir, belong) defaults_file = "#{repodir}/DEFAULTS" return unless File.exist?(defaults_file)

• defaults_key = repodir == REPO_DIR ? 'default' : repodir.delete_prefix("#{REPO_DIR}/")

• defaults_key = repodir == "#{REPO_DIR}/#{belong}" ? belong : repodir.delete_prefix("#{REPO_DIR}/#{belong}/") @defaults[defaults_key] = YAML.safe_load(File.open(defaults_file))

• @defaults[defaults_key] = merge_defaults(defaults_key, @defaults[defaults_key])

• @defaults[defaults_key] = merge_defaults(defaults_key, @defaults[defaults_key], belong) end

• def traverse_repodir(repodir)

• def traverse_repodir(repodir, belong) if File.directory? repodir

•  load_defaults(repodir)
  

•  load_defaults(repodir, belong)
  entry_list = Dir.children(repodir) - ['DEFAULTS']
  entry_list.each do |entry|
    next if entry.start_with? '.'
  
  

•    traverse_repodir("#{repodir}/#{entry}")
  

•    traverse_repodir("#{repodir}/#{entry}", belong)
  end
  

  else return if File.dirname(repodir) == REPO_DIR

•  project = File.dirname(repodir).delete_prefix("#{REPO_DIR}/")
  

•  project = File.dirname(repodir).delete_prefix("#{REPO_DIR}/#{belong}/")
  fork_name = File.basename(repodir)
  

•  load_repo_file(repodir, project, fork_name)
  

•  load_repo_file(repodir, project, fork_name, belong)
  

  end end

  def load_fork_info

• traverse_repodir(REPO_DIR)

• @upstreams['upstreams'].each do |repo|

•  traverse_repodir("#{REPO_DIR}/#{repo['location']}", repo['location'])
  

• end end

  def create_workers

@@ -221,14 +224,16 @@ end

# main thread class MirrorMain

• def load_repo_file(repodir, project, fork_name)

• def load_repo_file(repodir, project, fork_name, belong) git_repo = "#{project}/#{fork_name}" git_info = YAML.safe_load(File.open(repodir)) return if git_info.nil? || git_info['url'].nil?

  @git_info[git_repo] = git_info @git_info[git_repo]['git_repo'] = git_repo

• @git_info[git_repo] = merge_defaults(git_repo, @git_info[git_repo])

• @git_info[git_repo]['belong'] = belong
• @git_info[git_repo] = merge_defaults(git_repo, @git_info[git_repo], belong)
• fork_stat_init(git_repo) @priority_queue.push git_repo, @priority @priority += 1

@@ -453,8 +458,8 @@ end

# main thread class MirrorMain

• def merge_defaults(object_key, object)
• return object if object_key == 'default'

• def merge_defaults(object_key, object, belong)

• return object if object_key == belong

  defaults_key = File.dirname(object_key) while defaults_key != '.'

@@ -462,8 +467,28 @@ class MirrorMain

  defaults_key = File.dirname(defaults_key)
end

• return @defaults['default'].merge(object) if @defaults['default']

• return @defaults[belong].merge(object) if @defaults[belong]

  return object end

• def clone_upstream_repo

• if File.exist?("#{ENV['CCI_SRC']}/sbin/upstream-config")

•  @upstreams = YAML.safe_load(File.open('../sbin/upstream-config'))
  

•  @upstreams['upstreams'].each do |repo|
  

•    url = get_url(repo['url'])
  

•    %x(git clone #{url} #{REPO_DIR}/#{repo['location']} 2>&1)
  

•  end
  

• else

•  puts 'ERROR: No upstream-config file'
  

•  return -1
  

• end

• end

• def get_url(url)

• if url.include?('gitee.com/') && File.exist?("/srv/git/#{url.delete_prefix('https://%27)%7D")

•  url = "/srv/git/#{url.delete_prefix('https://')}"
  

• end

• return url

• end

end

2.23.0