[PATCH v2 compass-ci] lib/es_query.rb: get multi-field aggregation count result

30 Dec 2020

[why]
get multi-field aggregation count for overview of report page.

[input]:
  fields, query_items
  (optional for query_items, default no scope limitation)

[example1]
  input:
    fields = ['suite']
    query_items = {
      'os' => 'openeuler'
    }
  output:
    [{"key"=>"build-pkg", "doc_count"=>186214},
     {"key"=>"cci-depends", "doc_count"=>1230},
     {"key"=>"iperf", "doc_count"=>1103},
     {"key"=>"spinlock", "doc_count"=>1000},
     {"key"=>"cci-makepkg", "doc_count"=>949},
     {"key"=>"iperf-walk-os-test", "doc_count"=>474},
     {"key"=>"bisect", "doc_count"=>381},
     ...

[example2]
  input:
    fields = ['suite', 'job_state']
    query_items = {
      'os' => 'openeuler'
    }
  output:
    [{"key"=>"build-pkg",
      "doc_count"=>186227,
      "all_job_state"=>
       {"doc_count_error_upper_bound"=>0,
        "sum_other_doc_count"=>0,
        "buckets"=>
         [{"key"=>"failed", "doc_count"=>3830},
          {"key"=>"finished", "doc_count"=>803},
          {"key"=>"incomplete", "doc_count"=>196},
          {"key"=>"submit", "doc_count"=>136},
	  ...

[example3]
  input:
    fields = ['os', 'os_version', 'job_state']
    query_items = {}
  output:
    [{"key"=>"openeuler",
      "doc_count"=>193287,
      "all_os_version"=>
       {"doc_count_error_upper_bound"=>0,
        "sum_other_doc_count"=>0,
        "buckets"=>
         [{"key"=>"20.03",
           "doc_count"=>191789,
           "all_job_state"=>
            {"doc_count_error_upper_bound"=>0,
             "sum_other_doc_count"=>0,
             "buckets"=>
              [{"key"=>"failed", "doc_count"=>3909},
               {"key"=>"finished", "doc_count"=>2401},
               {"key"=>"incomplete", "doc_count"=>300},
               {"key"=>"submit", "doc_count"=>136},
               {"key"=>"OOM", "doc_count"=>23},
               {"key"=>"post_run", "doc_count"=>1}]}},
          {"key"=>"20.03-SP1",
           "doc_count"=>904,
           "all_job_state"=>
            {"doc_count_error_upper_bound"=>0,
             "sum_other_doc_count"=>0,
             "buckets"=>
              [{"key"=>"failed", "doc_count"=>641},
	      ...

Signed-off-by: Lu Kaiyi <2392863668@qq.com>
---
 lib/es_query.rb | 52 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/lib/es_query.rb b/lib/es_query.rb
index 88a420f..a9b651e 100644
--- a/lib/es_query.rb
+++ b/lib/es_query.rb
@@ -87,6 +87,45 @@ class ESQuery
 
     result
   end
+
+  # select field1, field2, count(*) from index where query_items group by field1, field2
+  # example:
+  #   fields = ['suite', 'job_state']
+  #   query_items = {
+  #     'os' => 'openeuler'
+  #   }
+  # input:
+  #   fields, query_items
+  #   (optional for query_items, default no scope limitation)
+  # output:
+  #   [{"key"=>"build-pkg",
+  #     "doc_count"=>186175,
+  #     "all_job_state"=>
+  #      {"doc_count_error_upper_bound"=>0,
+  #       "sum_other_doc_count"=>0,
+  #       "buckets"=>
+  #        [{"key"=>"failed", "doc_count"=>3830},
+  #         {"key"=>"finished", "doc_count"=>803},
+  #         {"key"=>"incomplete", "doc_count"=>196},
+  #         ...
+  def query_fields(fields, query_items = {})
+    field1 = fields.first
+    aggs_hash = {}
+    build_aggs_from_fields(fields, aggs_hash)
+    query = {
+      query: {
+        bool: {
+          must: build_multi_field_subquery_body(query_items)
+        }
+      },
+      aggs: aggs_hash['aggs'],
+      size: 0
+    }
+    result = @client.search(index: @index + '*', body: query)['aggregations']["all_#{field1}"]['buckets']
+    return nil if result.empty?
+
+    result
+  end
 end
 
 # Range Query Example:
@@ -146,3 +185,16 @@ def assign_desc_body(keyword)
     }]
   }
 end
+
+def build_aggs_from_fields(fields, aggs_hash)
+  return if fields.empty?
+
+  field = fields.shift
+  aggs_hash['aggs'] = {
+    "all_#{field}" => {
+      'terms' => { field: field, size: 1000 }
+    }
+  }
+
+  build_aggs_from_fields(fields, aggs_hash['aggs']["all_#{field}"])
+end
-- 
2.23.0

    

Lu Kaiyi

Lu Weitao

tags