[Compass-ci] Re: [PATCH compass-ci] web-backend: add "/update_repos" api and search feature to "/get_repos"

I'm very sorry! However it looks cleaner to use ES -- otherwise the git-mirror service will depend on the web-backend service, which looks awkward relationship.

Thanks, Fengguang

On Tue, Oct 13, 2020 at 05:48:43PM +0800, Zhang Yuhang wrote:

"/update_repos" will be called by git mirror service.

Signed-off-by: Zhang Yuhang zhangyuhang25@huawei.com

container/web-backend-nginx/nginx.conf | 5 +- container/web-backend/web-backend | 9 ++ lib/constants.rb | 3 + src/lib/web_backend.rb | 138 ++++++++++++++----------- 4 files changed, 94 insertions(+), 61 deletions(-)

diff --git a/container/web-backend-nginx/nginx.conf b/container/web-backend-nginx/nginx.conf index 5ab1a26..b461a2c 100644 --- a/container/web-backend-nginx/nginx.conf +++ b/container/web-backend-nginx/nginx.conf @@ -6,7 +6,7 @@ events { http { include /etc/nginx/mime.types; default_type application/octet-stream;

• include /etc/nginx/conf.d/*.conf;

• include /etc/nginx/conf.d/*.conf; server { listen 11320 ssl; server_name api.compass-ci.openeuler.org;

@@ -15,8 +15,7 @@ http { ssl_session_timeout 5m; ssl_ciphers BCDHE-RSA-AES128-GCM-SHA256:ECDHE:ECDH:AES:HIGH:!NULL:!aNULL:!MD5:!ADH:!RC4; ssl_prefer_server_ciphers on;

• location / {
  

• location ~ ^/(get_repos|get_jobs|compare|compare_candidates)$ {
  proxy_set_header Host $host;
  proxy_set_header X-Real-IP $remote_addr;
  proxy_set_header X-Forward-For $proxy_add_x_forwarded_for;
  

diff --git a/container/web-backend/web-backend b/container/web-backend/web-backend index 60c7827..05f7f6a 100755 --- a/container/web-backend/web-backend +++ b/container/web-backend/web-backend @@ -9,6 +9,7 @@ require "#{CCI_SRC}/src/lib/web_backend.rb" set :bind, '0.0.0.0' set :port, 32767 set :show_exceptions, false +set :environment, :production

# GET /compare_candidates # return: {"query_conditions": {...}, "dimension": [...]} @@ -62,3 +63,11 @@ end get '/get_repos' do get_repos(params) end

+# POST /update_repos +# Body: repos hash json +# Content-Type: "Application/json" +# Description: update repos by git mirror service +post '/update_repos' do

• update_repos(request)

+end diff --git a/lib/constants.rb b/lib/constants.rb index 3a4f083..efecaef 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -10,3 +10,6 @@ ES_PORT = config['ES_PORT'] || 9200

MAIL_HOST = config['MAIL_HOST'] || '172.17.0.1' MAIL_PORT = config['MAIL_PORT'] || 11_311

+# web-backend service port that makes git mirror service updates "repos" of web-backend service +WEB_BACKEND_PORT = 32767 diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 4544690..da1da25 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -12,7 +12,6 @@ require "#{CCI_SRC}/lib/es_query.rb" require "#{CCI_SRC}/lib/matrix2.rb"

UPSTREAM_REPOS_PATH = ENV['UPSTREAM_REPOS_PATH'] || '/c/upstream-repos'

FIELDS = %w[ upstream_repo os @@ -33,6 +32,66 @@ PREFIX_SEARCH_FIELDS = ['tbox_group'].freeze ES_CLIENT = Elasticsearch::Client.new(url: "http://#%7BES_HOST%7D:#%7BES_PORT%7D") COMPARE_RECORDS_NUMBER = 50

+# Repos operation class +class Repos

• attr_reader :repos, :repos_list
• def initialize
• @repos = {}
• @repos_list = []
• Dir["#{UPSTREAM_REPOS_PATH}/*/*/*"].sort.each do |repo_file|

•  git_repo = repo_file.slice(UPSTREAM_REPOS_PATH.size + 1)
  

•  urls = YAML.load_file(repo_file)['url'] if File.file? repo_file
  

•  repo = {
  

•    git_repo: git_repo,
  

•    git_url: get_repo_url(git_repo, urls: urls)
  

•  }
  

•  @repos_list << repo
  

• end
• end
• def get_repo_url(git_repo, urls: nil)
• urls ||= @repos[git_repo]['url']
• return if urls.nil? || urls.empty?
• return urls[1] if urls[1] && urls[1][0, 4] == 'http'
• return urls[0]
• end
• def find_repos(git_repo, page_num, page_size)
• from = page_num * page_size
• return [] if wrong_size?(page_size, from)
• repos = []
• if git_repo

•  @repos_list.each do |repo|
  

•    repos << repo if repo[:git_repo].include?(git_repo)
  

•  end
  

• end
• repos = @repos_list if repos.empty?
• repos[from, page_size]
• end
• def get_repo(git_repo)
• {

•  git_repo: git_repo,
  

•  git_url: get_repo_url(git_repo)
  

• }
• end
• def repos=(repos)
• repos_list = []
• @repos = repos
• @repos.keys.sort.each do |git_repo|

•  repos_list << get_repo(git_repo)
  

• end
• @repos_list = repos_list
• end

+end

+REPOS = Repos.new def es_query(query) ES_CLIENT.search index: 'jobs*', body: query end @@ -95,7 +154,7 @@ def compare_candidates_body OS: [ { os: 'openeuler', os_version: ['1.0', '20.03'] }, { os: 'centos', os_version: ['7.6', '7.8', '8.1', 'sid'] },

•    { os: 'debian', os_version: ['10', 'sid'] },
  

•    { os: 'debian', os_version: %w[10 sid] },
    { os: 'archlinux', os_version: ['5.5.0-1'] }
  ],
  os_arch: %w[aarch64 x86_64],
  

@@ -270,7 +329,7 @@ end def get_banner(git_repo, branches) { repo: git_repo,

• git_url: get_repo(git_repo)[:git_url],

• git_url: REPOS.get_repo_url(git_repo), upstream_branch: branches }

end @@ -309,79 +368,42 @@ end

def get_jobs(params) begin

• body = get_jobs_body(params)

• body = get_jobs_body(params).to_json rescue StandardError => e warn e.message return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'get jobs error'] end

• [200, headers.merge('Access-Control-Allow-Origin' => '*'), JSON.dump(body)]

-end

-def get_repo_url(repo_file)

• return unless File.file? repo_file
• urls = YAML.load_file(repo_file)['url']
• urls.each do |url|
• return url if url[0, 4] == 'http'
• end
• urls[0]

-end

-def get_repo(git_repo, repo_file = nil)

• repo = {
• git_repo: git_repo,
• git_url: nil
• }
• return repo if !git_repo && !repo_file
• if !git_repo
• git_repo = repo_file[UPSTREAM_REPOS_PATH.size + 1, repo_file.size - 1]
• repo[:git_repo] = git_repo
• elsif !repo_file
• repo_file = File.join(UPSTREAM_REPOS_PATH, git_repo)
• end
• repo[:git_url] = get_repo_url(repo_file)
• repo

-end

-def repo_files_list

• Dir["#{UPSTREAM_REPOS_PATH}/*/*/*"].sort

-end

-def get_repos_list(repo_files, from, finish, total)

• repos_list = []
• total.times do |index|
• next if index < from
• break if index >= finish
• repos_list << get_repo(nil, repo_files[index])
• end
• repos_list

• [200, headers.merge('Access-Control-Allow-Origin' => '*'), body]

end

def get_repos_body(params)

• repo_files = repo_files_list

• git_repo = space_to_nil(params[:repo]) page_size = get_positive_number(params[:page_size], 20) page_num = get_positive_number(params[:page_num], 1) - 1

• from = page_num * page_size
• finish = from + page_size
• total = repo_files.size
• {
• total: total,
• repos: get_repos_list(repo_files, from, finish, total)

• total: REPOS.repos_list.size,
• repos: REPOS.find_repos(git_repo, page_num, page_size) }

end

def get_repos(params) begin

• body = get_repos_body(params)

• body = get_repos_body(params).to_json rescue StandardError => e warn e.message return [500, headers.merge('Access-Control-Allow-Origin' => '*'), 'get repos error']

end

• [200, headers.merge('Access-Control-Allow-Origin' => '*'), JSON.dump(body)]

• [200, headers.merge('Access-Control-Allow-Origin' => '*'), body]

+end

+def update_repos(request)

• begin
• request.body.rewind # in case someone already read it
• REPOS.repos = JSON.parse(request.body.read)
• rescue StandardError => e
• warn e.message
• return [500, headers, 'error']
• end
• [200, headers, 'success']

end

2.23.0