[Compass-ci] Re: [PATCH compass-ci 2/2] /src/lib/web_backend.rb: remove common es methods

On Mon, Oct 19, 2020 at 03:14:20PM +0800, Sun Yukui wrote:

On Mon, Oct 19, 2020 at 02:48:25PM +0800, Zhang Yuhang wrote:

...

Signed-off-by: Zhang Yuhang zhangyuhang25@huawei.com

Please add your description Thanks sunyukui

OK Thanks, Zhang Yuhang

...

---

src/lib/web_backend.rb | 59 +++++++++--------------------------------- 1 file changed, 12 insertions(+), 47 deletions(-)

diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 48cfadb..721ad77 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -30,16 +30,9 @@ NOT_SHOW_FIELDS = %w[result_root].freeze ALL_FIELDS = FIELDS + NOT_SHOW_FIELDS NOT_NEED_EXIST_FIELDS = %w[error_ids upstream_repo].freeze PREFIX_SEARCH_FIELDS = ['tbox_group'].freeze -ES_CLIENT = Elasticsearch::Client.new(url: "http://#%7BES_HOST%7D:#%7BES_PORT%7D") COMPARE_RECORDS_NUMBER = 50

-def es_query(query)

• ES_CLIENT.search index: 'jobs*', body: query

-end

-def es_count(query)

• ES_CLIENT.count(index: 'jobs*', body: query)['count']

-end +JOBS = ESQuery.new(index: 'jobs*') +REPO = ESQuery.new(index: 'repo')

# "vm-2p8g--212" remove "--212" # "vm-2p8g--zzz" remove "--zzz" @@ -47,7 +40,7 @@ end def filter_tbox_group(es_result) result = Set.new es_result.each do |r|

• if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-([a-zA-Z]+)|(\d+)$)/

• if r =~ /(^.+--.+$)|(^vm-.*-\d\w*-(([a-zA-Z]+)|(\d+))$)/ index = r.index('--') || r.rindex('-') r = r[0, index] end

@@ -56,42 +49,15 @@ def filter_tbox_group(es_result) result.to_a end

-def all_suite

• query = {
• aggs: {

•  all_suite: {
  

•    terms: { field: 'suite', size: 1000 }
  

•  }
  

• },
• size: 0
• }
• es_result = es_query(query)['aggregations']['all_suite']['buckets']
• es_result.sort_by! { |h| h['doc_count'] }
• es_result.reverse!.map! { |x| x['key'] }
• es_result

-end

def all_tbox_group

• query = {
• aggs: {

•  all_tbox_group: {
  

•    terms: { field: 'tbox_group', size: 1000 }
  

•  }
  

• },
• size: 0
• }
• es_result = es_query(query)['aggregations']['all_tbox_group']['buckets']
• es_result.sort_by! { |h| h['doc_count'] }
• es_result.reverse!.map! { |x| x['key'] }

• es_result = JOBS.all_values('tbox_group') filter_tbox_group(es_result)

end

def compare_candidates_body body = { query_conditions: {

•  suite: all_suite,
  

•  suite: JOBS.all_values('suite'),
  OS: [
    { os: 'openeuler', os_version: ['1.0', '20.03'] },
    { os: 'centos', os_version: ['7.6', '7.8', '8.1', 'sid'] },
  

@@ -148,7 +114,7 @@ end def get_groups_matrices(conditions, dimension, must, size, from) must += build_mutli_field_subquery_body(conditions) count_query = { query: { bool: { must: must } } }

• total = es_count(count_query)

• total = JOBS.es_count(count_query) return {} if total < 1

  query = {

@@ -164,14 +130,14 @@ def get_groups_matrices(conditions, dimension, must, size, from) }] }

• result = es_query(query)

• result = JOBS.es_query(query) matrices = combine_group_query_data(result, dimension) while matrices.empty? from += size break if from > total

  query[:from] = from

• result = es_query(query)

• result = JOBS.es_query(query) matrices = combine_group_query_data(result, dimension) end matrices

@@ -229,7 +195,7 @@ def es_search(must, size, from) must << { exists: { field: f } } end count_query = { query: { bool: { must: must } } }

• total = es_count(count_query)

• total = JOBS.es_count(count_query) unless size size = total from = 0

@@ -245,7 +211,7 @@ def es_search(must, size, from) } return {}, total if wrong_size?(size, from)

• return es_query(query)['hits']['hits'], total

• return JOBS.es_query(query)['hits']['hits'], total

end

def get_job(result) @@ -327,7 +293,6 @@ def get_repo_url(urls) end

def get_repo(git_repo)

• repo = nil if git_repo must = [{ regexp: { git_repo: ".*#{git_repo}.*" } }] repo = query_repos(must, from: 0, size: 1)[0]

@@ -344,7 +309,7 @@ def query_repos(must, from: 0, size: 1) git_repo: { order: 'asc' } }] }

• result = ES_CLIENT.search index: 'repo', body: query

• result = REPO.es_query(query) repos = [] result['hits']['hits'].each do |r| r = r['_source']

@@ -361,7 +326,7 @@ def search_repos(git_repo, page_size, page_num) from = size * page_num must = git_repo ? [{ regexp: { git_repo: ".*#{git_repo}.*" } }] : [] count_query = { query: { bool: { must: must } } }

• total = ES_CLIENT.count(index: 'repo', body: count_query)['count']

• total = REPO.es_count(count_query) return [], total if wrong_size?(size, from)

  return query_repos(must, from: from, size: size), total

-- 2.23.0