I'm very sorry! However it looks cleaner to use ES -- otherwise the git-mirror service will depend on the web-backend service, which looks awkward relationship. Thanks, Fengguang On Tue, Oct 13, 2020 at 05:48:43PM +0800, Zhang Yuhang wrote:

"/update_repos" will be called by git mirror service.

Signed-off-by: Zhang Yuhang <zhangyuhang25@huawei.com> --- container/web-backend-nginx/nginx.conf | 5 +- container/web-backend/web-backend | 9 ++ lib/constants.rb | 3 + src/lib/web_backend.rb | 138 ++++++++++++++----------- 4 files changed, 94 insertions(+), 61 deletions(-)

diff --git a/container/web-backend-nginx/nginx.conf b/container/web-backend-nginx/nginx.conf index 5ab1a26..b461a2c 100644 --- a/container/web-backend-nginx/nginx.conf +++ b/container/web-backend-nginx/nginx.conf @@ -6,7 +6,7 @@ events { http { include /etc/nginx/mime.types; default_type application/octet-stream; - include /etc/nginx/conf.d/*.conf; + include /etc/nginx/conf.d/*.conf; server { listen 11320 ssl; server_name api.compass-ci.openeuler.org; @@ -15,8 +15,7 @@ http { ssl_session_timeout 5m; ssl_ciphers BCDHE-RSA-AES128-GCM-SHA256:ECDHE:ECDH:AES:HIGH:!NULL:!aNULL:!MD5:!ADH:!RC4; ssl_prefer_server_ciphers on; - - location / { + location ~ ^/(get_repos|get_jobs|compare|compare_candidates)$ { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forward-For $proxy_add_x_forwarded_for; diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 60c7827..05f7f6a 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -9,6 +9,7 @@ require "#{CCI_SRC}/src/lib/web_backend.rb" set :bind, '0.0.0.0' set :port, 32767 set :show_exceptions, false +set :environment, :production

# GET /compare_candidates # return: {"query_conditions": {...}, "dimension": [...]} @@ -62,3 +63,11 @@ end get '/get_repos' do get_repos(params) end + +# POST /update_repos +# Body: repos hash json +# Content-Type: "Application/json" +# Description: update repos by git mirror service +post '/update_repos' do + update_repos(request) +end diff --git a/lib/constants.rb b/lib/constants.rb index 3a4f083..efecaef 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -10,3 +10,6 @@ ES_PORT = config['ES_PORT'] || 9200

MAIL_HOST = config['MAIL_HOST'] || '172.17.0.1' MAIL_PORT = config['MAIL_PORT'] || 11_311 + +# web-backend service port that makes git mirror service updates "repos" of web-backend service +WEB_BACKEND_PORT = 32767 diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 4544690..da1da25 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -12,7 +12,6 @@ require "#{CCI_SRC}/lib/es_query.rb" require "#{CCI_SRC}/lib/matrix2.rb"

UPSTREAM_REPOS_PATH = ENV['UPSTREAM_REPOS_PATH'] || '/c/upstream-repos' - FIELDS = %w[ upstream_repo os @@ -33,6 +32,66 @@ PREFIX_SEARCH_FIELDS = ['tbox_group'].freeze ES_CLIENT = Elasticsearch::Client.new(url: "http://#{ES_HOST}:#{ES_PORT}") COMPARE_RECORDS_NUMBER = 50

+# Repos operation class +class Repos + attr_reader :repos, :repos_list + + def initialize + @repos = {} + @repos_list = [] + + Dir["#{UPSTREAM_REPOS_PATH}/*/*/*"].sort.each do |repo_file| + git_repo = repo_file.slice(UPSTREAM_REPOS_PATH.size + 1) + urls = YAML.load_file(repo_file)['url'] if File.file? repo_file + repo = { + git_repo: git_repo, + git_url: get_repo_url(git_repo, urls: urls) + } + @repos_list << repo + end + end + + def get_repo_url(git_repo, urls: nil) + urls ||= @repos[git_repo]['url'] + + return if urls.nil? || urls.empty? + return urls[1] if urls[1] && urls[1][0, 4] == 'http' + + return urls[0] + end + + def find_repos(git_repo, page_num, page_size) + from = page_num * page_size + return [] if wrong_size?(page_size, from) + + repos = [] + if git_repo + @repos_list.each do |repo| + repos << repo if repo[:git_repo].include?(git_repo) + end + end + repos = @repos_list if repos.empty? + repos[from, page_size] + end + + def get_repo(git_repo) + { + git_repo: git_repo, + git_url: get_repo_url(git_repo) + } + end + + def repos=(repos) + repos_list = [] + @repos = repos + @repos.keys.sort.each do |git_repo| + repos_list << get_repo(git_repo) + end + @repos_list = repos_list + end +end + +REPOS = Repos.new def es_query(query) ES_CLIENT.search index: 'jobs*', body: query end @@ -95,7 +154,7 @@ def compare_candidates_body OS: [ { os: 'openeuler', os_version: ['1.0', '20.03'] }, { os: 'centos', os_version: ['7.6', '7.8', '8.1', 'sid'] }, - { os: 'debian', os_version: ['10', 'sid'] }, + { os: 'debian', os_version: %w[10 sid] }, { os: 'archlinux', os_version: ['5.5.0-1'] } ], os_arch: %w[aarch64 x86_64], @@ -270,7 +329,7 @@ end def get_banner(git_repo, branches) { repo: git_repo, - git_url: get_repo(git_repo)[:git_url], + git_url: REPOS.get_repo_url(git_repo), upstream_branch: branches } end @@ -309,79 +368,42 @@ end

def get_jobs(params) begin - body = get_jobs_body(params) + body = get_jobs_body(params).to_json rescue StandardError => e warn e.message return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'get jobs error'] end - [200, headers.merge('Access-Control-Allow-Origin' => '*'), JSON.dump(body)] -end - -def get_repo_url(repo_file) - return unless File.file? repo_file - - urls = YAML.load_file(repo_file)['url'] - urls.each do |url| - return url if url[0, 4] == 'http' - end - urls[0] -end - -def get_repo(git_repo, repo_file = nil) - repo = { - git_repo: git_repo, - git_url: nil - } - return repo if !git_repo && !repo_file - - if !git_repo - git_repo = repo_file[UPSTREAM_REPOS_PATH.size + 1, repo_file.size - 1] - repo[:git_repo] = git_repo - elsif !repo_file - repo_file = File.join(UPSTREAM_REPOS_PATH, git_repo) - end - - repo[:git_url] = get_repo_url(repo_file) - repo -end - -def repo_files_list - Dir["#{UPSTREAM_REPOS_PATH}/*/*/*"].sort -end - -def get_repos_list(repo_files, from, finish, total) - repos_list = [] - - total.times do |index| - next if index < from - break if index >= finish - - repos_list << get_repo(nil, repo_files[index]) - end - repos_list + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] end

def get_repos_body(params) - repo_files = repo_files_list + git_repo = space_to_nil(params[:repo]) page_size = get_positive_number(params[:page_size], 20) page_num = get_positive_number(params[:page_num], 1) - 1

- from = page_num * page_size - finish = from + page_size - total = repo_files.size - { - total: total, - repos: get_repos_list(repo_files, from, finish, total) + total: REPOS.repos_list.size, + repos: REPOS.find_repos(git_repo, page_num, page_size) } end

def get_repos(params) begin - body = get_repos_body(params) + body = get_repos_body(params).to_json rescue StandardError => e warn e.message return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'get repos error'] end - [200, headers.merge('Access-Control-Allow-Origin' => '*'), JSON.dump(body)] + [200, headers.merge('Access-Control-Allow-Origin' => '*'), body] +end + +def update_repos(request) + begin + request.body.rewind # in case someone already read it + REPOS.repos = JSON.parse(request.body.read) + rescue StandardError => e + warn e.message + return [500, headers, 'error'] + end + [200, headers, 'success'] end -- 2.23.0