- Compass-ci - mailweb.openeuler.org

[PATCH lkp-tests 2/2] add script for fix dockerfile file
by Bai Jing 06 Apr '21

06 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- lib/handle_dockerfile.sh | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 lib/handle_dockerfile.sh diff --git a/lib/handle_dockerfile.sh b/lib/handle_dockerfile.sh new file mode 100644 index 000000000..b71f9865d --- /dev/null +++ b/lib/handle_dockerfile.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" +} + +handle_epel() +{ + sed -i "s/epel-release/bash/g" "$dockerfile_name" +} + +handle_epelrpm() +{ + sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" +} + +handle_rpm_gpg() +{ + sed -i "s|RPM-GPG-KEY-CentOS-7|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" + sed -i "s|RPM-GPG-KEY-centosofficial|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" +} + +add_base_commands() +{ + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" +} + +handle_epel_repo() +{ + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" +} + +handle_error_exit() +{ + sed '/^RUN/RUN set -e;/g' "$dockerfile_name" +} + +handle_dockerfile() +{ + dockerfile_name=$1 + handle_FROM + handle_epel + handle_epelrpm + handle_rpm_gpg + add_base_commands + handle_epel_repo + handle_error_exit +} -- 2.23.0

1 0

[PATCH lkp-tests 1/2] add test script for dockerfile test
by Bai Jing 06 Apr '21

06 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 143 +++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 85 deletions(-) diff --git a/tests/openeuler_docker b/tests/openeuler_docker index 483f40135..3ea57e2c9 100755 --- a/tests/openeuler_docker +++ b/tests/openeuler_docker @@ -8,12 +8,12 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/log.sh +. $LKP_SRC/lib/handle_dockerfile.sh [ -n "$dockerfile_url" ] || die "dockerfile_url is empty" declare -i nr_openeuler_fail=0 declare -i nr_openeuler_success=0 -declare -i is_openeuler_build_fail=0 declare -i dockerfile_count=0 fit_repo() @@ -33,110 +33,51 @@ openeuler_ready() docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar } -centos_docker() -{ - is_openeuler_build_fail=0 - index=1 - for file in $dockerfile_path - do - [[ $file =~ ^centos ]] && continue - file=$(echo "$file" | awk -F: '{print $1}') - dockerfile_name="${file##*/}" - cd "${file%/*}" || continue - image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') - [ -n "$image" ] && image=${image}_$index - if [ -f "build.sh" ] && [ -x "build.sh" ]; then - ./build.sh > /dev/null 2> /dev/null - else - docker_build > /dev/null 2> /dev/null - fi - - [ $? == 0 ] && { - handle_dockerfile - } - - let index++ - done -} - -handle_FROM() -{ - sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" -} - -handle_epel() -{ - grep -qw "epel-release" "$dockerfile_name" && { - sed -i "s/epel-release/bash/g" "$dockerfile_name" - } -} - -handle_epelrpm() -{ - grep -qw "https://.*epel-release.*rpm" "$dockerfile_name" && { - sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" - } -} - -handle_rpm_gpg() -{ - sed -i "s|RPM-GPG-KEY-CentOS-7|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" -} - -add_base_commands() -{ - # grep -qw "groupadd*" "$file" && { - # fix missing useradd, groupadd, chpasswd, etc. commands - sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" -} - -handle_epel_repo() -{ - sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" -} - -handle_dockerfile() +build_centos() { - is_openeuler_build_fail=1 - handle_FROM - handle_epel - handle_rpm_gpg - add_base_commands - if docker_build "-openeuler"; then - docker_run "-openeuler" - let nr_openeuler_success++ + cd ${file%/*} + if [ -f "build.sh" ] && [ -x "build.sh" ]; then + ./build.sh || return 1 else - let nr_openeuler_fail++ + docker build -f "$dockerfile_name" -t "${image}_$1_$index" . || return 1 fi } -docker_build() +build_openeuler() { - docker build -f "$dockerfile_name" -t "${image}$1" . + docker build -f "$dockerfile_name" -t "${image}_$1_$index" . >> $TMP_RESULT_ROOT/${image}_$1_$index 2>&1 [ $? != 0 ] && { - [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.fail: 1" + let nr_openeuler_fail++ + echo "${image}_$index.build.fail: 1" return 1 } - [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.success: 1" - is_openeuler_build_fail=0 + echo "${image}_$index.build.success: 1" + let nr_openeuler_success++ + return 0 } docker_run() -{ - docker run -d --name "${image}" "${image}$1" - docker_stats=$(docker inspect -f '{{.State.Running}}' ${image}) - [ $docker_stats == false ] && { - echo "${image}.run.fail: 1" +{ + docker_run_cmd="docker run -d --name ${image}_$1_$index ${image}_$1_$index" + [ $1 == "openeuler" ] && { + docker_run_cmd="${docker_run_cmd} >> $TMP_RESULT_ROOT/${image}_$1_$index 2>&1" + } + + eval $docker_run_cmd + + docker_stats=$(docker inspect -f '{{.State.Running}}' $image_$1_$index) + [ $docker_stats == "false" ] && { + [ $1 == "openeuler" ] && echo "${image}_$index.run.fail: 1" return 1 } - echo "${image}.run.success: 1" + [ $1 == "openeuler"] && echo "${image}_$index.run.success: 1" } handle_build_result() { if [ "$nr_openeuler_fail" != 0 ]; then - echo "openeuler build failed times: $nr_openeuler_fail" + echo "openeuler_build.nr_failed: $nr_openeuler_fail" echo "openeuler_valid_url: $dockerfile_url" exit 1 # centos all success, openeuler all success @@ -146,6 +87,38 @@ handle_build_result() fi } +build() +{ + dockerfile_name="${file##*/}" + dockerfile_name="${dockerfile_name%:*}" + image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') + + build_centos "centos" > /dev/null 2>&1 || return 1 + handle_dockerfile "$dockerfile_name" + build_openeuler "openeuler" + [ $? != 0 ] && return 1 +} + +run() +{ + docker_run "centos" > /dev/null 2>&1 + [ $? != 0 ] && return 1 + docker_run "openeuler" +} + +centos_docker() +{ + index=1 + for file in $dockerfile_path + do + [[ $file =~ ^centos ]] && continue + + build + [ $? = 0 ] && run + let index++ + done +} + docker_test() { fit_repo -- 2.23.0

1 0

[PATCH compass-ci] sbin/es-find: support check query keyword
by Lu Weitao 06 Apr '21

06 Apr '21

[How] 1. add --keyword option 2. use like: es-find --keyword category dequeue_time end_time enqueue_time group_id hw.arch hw.memory hw.model hw.nr_cpu ... Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/es_query.rb | 25 +++++++++++++++++++++++++ sbin/es-find | 11 ++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 928bd68..22d68a9 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -130,6 +130,11 @@ class ESQuery parse_fields(result) end + + def query_mapping + mapping = @client.indices.get_mapping(index: @index) + get_mapping_key(mapping) + end end # Range Query Example: @@ -260,3 +265,23 @@ def parse_fields(es_result) result_hash end + +def get_mapping_key(mapping) + mapping_key = [] + properties = mapping.values[0]['mappings']['properties'] + properties.each do |key, value| + parse_mapping_properties(key, value, mapping_key) + end + + mapping_key +end + +def parse_mapping_properties(key, value, mapping_key) + if value['type'] + mapping_key << key + else + value['properties'].each do |k, v| + parse_mapping_properties("#{key}.#{k}", v, mapping_key) + end + end +end diff --git a/sbin/es-find b/sbin/es-find index 2c3b3dc..f548fc1 100755 --- a/sbin/es-find +++ b/sbin/es-find @@ -77,6 +77,8 @@ options = { nr_jobs: 10, desc_keyword: 'start_time' } range = nil count = nil +es = ESQuery.new(ES_HOST, ES_PORT) + opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: es-find [options] search_key1=val1[,val2..] ..' @@ -103,6 +105,14 @@ opt_parser = OptionParser.new do |opts| count = c end + opts.on_tail('--keyword', 'check all available search_keys') do + available_search_key = es.query_mapping + available_search_key.each do |key| + puts " #{key}" + end + exit + end + opts.on_tail('-h', '--help', 'show this message') do puts opts exit @@ -113,7 +123,6 @@ opt_parser.parse!(ARGV) items = parse_argv items[:range] = prase_range(range) if range -es = ESQuery.new(ES_HOST, ES_PORT) if count count_keywords = count.split(',') query_result = es.query_fields(count_keywords, items) -- 2.23.0

1 0

[PATCH v2 compass-ci] delimiter: refactor delimiter with crystal
by Cao Xueliang 04 Apr '21

04 Apr '21

use etcd list watch to submit bisect job Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter.cr | 22 ++++++++++ src/delimiter/delimiter.cr | 83 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 src/delimiter.cr create mode 100644 src/delimiter/delimiter.cr diff --git a/src/delimiter.cr b/src/delimiter.cr new file mode 100644 index 0000000..e8de306 --- /dev/null +++ b/src/delimiter.cr @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "yaml" +require "./delimiter/delimiter" + +def config_secrets_yaml + %x(#{ENV["CCI_SRC"]}/sbin/config_account_yaml.rb delimiter) + account = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/defaults/account.yaml")) + lab = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/include/lab/#{account["lab"]}.yaml")) + secrets = {"secrets" => lab} + File.open("#{ENV["HOME"]}/.config/compass-ci/defaults/secrets.yaml", "w") { |f| YAML.dump(secrets, f) } +end + +begin + config_secrets_yaml + delimiter = Delimiter.new + delimiter.consume_delimiter("delimiter") +rescue ex + puts ex +end diff --git a/src/delimiter/delimiter.cr b/src/delimiter/delimiter.cr new file mode 100644 index 0000000..c98ded9 --- /dev/null +++ b/src/delimiter/delimiter.cr @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "../lib/etcd_client" + +class Delimiter + def initialize + @ec = EtcdClient.new + end + + def consume_delimiter(queue) + channel = Channel(Etcd::Model::Kv).new + revision = consume_by_list(queue, channel) + consume_by_watch(queue, revision, channel) + end + + def consume_by_list(queue, channel) + tasks, revision = get_history_tasks(queue) + handle_history_tasks(tasks, channel) + + return revision + end + + def consume_by_watch(queue, revision, channel) + watch_queue(queue, revision, channel) + handle_events(channel) + end + + def get_history_tasks(queue) + tasks = [] of Etcd::Model::Kv + range = @ec.range_prefix(queue) + revision = range.header.not_nil!.revision + tasks += range.kvs + + return tasks, revision + end + + def handle_history_tasks(tasks, channel) + loop do + return if tasks.empty? + + task = tasks.delete_at(0) + spawn { submit_bisect_job(channel, task) } + end + end + + def watch_queue(queue, revision, channel) + watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64 + 1, filters: [Etcd::Watch::Filter::NODELETE]) do |events| + events.each do |event| + channel.send(event.kv) + end + end + + spawn { watcher.start } + Fiber.yield + end + + def handle_events(channel) + loop do + task = channel.receive + spawn { submit_bisect_job(channel, task) } + end + end + + def submit_bisect_job(channel, task) + key = task.key + value = Hash(String, String).from_json(task.value.not_nil!) + begin + response = %x(#{ENV["LKP_SRC"]}/sbin/submit bad_job_id=#{value["job_id"]} error_id=#{value["error_id"].inspect} bisect.yaml queue=dc-bisect) + puts response + if /id=0/ =~ response + channel.send(task) + return + end + + @ec.delete(key) + rescue ex + puts ex + channel.send(task) + end + end +end -- 2.23.0

1 0

[PATCH v1 compass-ci 3/3] delimiter: refactor delimiter whit crystal
by Cao Xueliang 04 Apr '21

04 Apr '21

use etcd list watch to submit bisect job Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter.cr | 22 ++++++++++ src/delimiter/delimiter.cr | 83 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 src/delimiter.cr create mode 100644 src/delimiter/delimiter.cr diff --git a/src/delimiter.cr b/src/delimiter.cr new file mode 100644 index 0000000..e8de306 --- /dev/null +++ b/src/delimiter.cr @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "yaml" +require "./delimiter/delimiter" + +def config_secrets_yaml + %x(#{ENV["CCI_SRC"]}/sbin/config_account_yaml.rb delimiter) + account = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/defaults/account.yaml")) + lab = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/include/lab/#{account["lab"]}.yaml")) + secrets = {"secrets" => lab} + File.open("#{ENV["HOME"]}/.config/compass-ci/defaults/secrets.yaml", "w") { |f| YAML.dump(secrets, f) } +end + +begin + config_secrets_yaml + delimiter = Delimiter.new + delimiter.consume_delimiter("delimiter") +rescue ex + puts ex +end diff --git a/src/delimiter/delimiter.cr b/src/delimiter/delimiter.cr new file mode 100644 index 0000000..c98ded9 --- /dev/null +++ b/src/delimiter/delimiter.cr @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "../lib/etcd_client" + +class Delimiter + def initialize + @ec = EtcdClient.new + end + + def consume_delimiter(queue) + channel = Channel(Etcd::Model::Kv).new + revision = consume_by_list(queue, channel) + consume_by_watch(queue, revision, channel) + end + + def consume_by_list(queue, channel) + tasks, revision = get_history_tasks(queue) + handle_history_tasks(tasks, channel) + + return revision + end + + def consume_by_watch(queue, revision, channel) + watch_queue(queue, revision, channel) + handle_events(channel) + end + + def get_history_tasks(queue) + tasks = [] of Etcd::Model::Kv + range = @ec.range_prefix(queue) + revision = range.header.not_nil!.revision + tasks += range.kvs + + return tasks, revision + end + + def handle_history_tasks(tasks, channel) + loop do + return if tasks.empty? + + task = tasks.delete_at(0) + spawn { submit_bisect_job(channel, task) } + end + end + + def watch_queue(queue, revision, channel) + watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64 + 1, filters: [Etcd::Watch::Filter::NODELETE]) do |events| + events.each do |event| + channel.send(event.kv) + end + end + + spawn { watcher.start } + Fiber.yield + end + + def handle_events(channel) + loop do + task = channel.receive + spawn { submit_bisect_job(channel, task) } + end + end + + def submit_bisect_job(channel, task) + key = task.key + value = Hash(String, String).from_json(task.value.not_nil!) + begin + response = %x(#{ENV["LKP_SRC"]}/sbin/submit bad_job_id=#{value["job_id"]} error_id=#{value["error_id"].inspect} bisect.yaml queue=dc-bisect) + puts response + if /id=0/ =~ response + channel.send(task) + return + end + + @ec.delete(key) + rescue ex + puts ex + channel.send(task) + end + end +end -- 2.23.0

1 0

[PATCH v1 compass-ci 2/3] container/delimiter: compile delimiter with crystal
by Cao Xueliang 04 Apr '21

04 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/delimiter/Dockerfile | 11 ++++++++++- container/delimiter/build | 4 ++++ container/delimiter/start | 3 +-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/container/delimiter/Dockerfile b/container/delimiter/Dockerfile index cc57320..a9ff279 100644 --- a/container/delimiter/Dockerfile +++ b/container/delimiter/Dockerfile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM alpine:3.13 +FROM alpine:scheduler-dev RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories && \ apk update && \ @@ -11,3 +11,12 @@ RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweiclou RUN umask 002 && \ gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ gem install rest-client activesupport git json yaml elasticsearch faye-websocket terminal-table mail io-console + +ENV RUNTIME_DIR /c/cci/delimiter + +RUN mkdir -p $RUNTIME_DIR && \ + chown -R 1090:1090 /c + +WORKDIR $RUNTIME_DIR + +COPY --chown=1090:1090 delimiter . diff --git a/container/delimiter/build b/container/delimiter/build index 606149d..7f3108b 100755 --- a/container/delimiter/build +++ b/container/delimiter/build @@ -8,6 +8,10 @@ docker_skip_rebuild "delimiter" load_cci_defaults +bash $CCI_SRC/container/compile . + docker build -t delimiter . push_image delimiter:latest + +rm delimiter diff --git a/container/delimiter/start b/container/delimiter/start index 3cff23a..9485750 100755 --- a/container/delimiter/start +++ b/container/delimiter/start @@ -32,10 +32,9 @@ cmd = %w[ -v #{DEFAULT_CONFIG_DIR}:#{DEFAULT_CONFIG_DIR}:ro -v /etc/localtime:/etc/localtime:ro -v /etc/compass-ci/register:/etc/compass-ci/register:ro - -w #{DEFAULT_CCI}/src delimiter ] -cmd += ['sh', '-c', 'umask 002 && ruby ./delimiter.rb'] +cmd += ['sh', '-c', 'umask 002 && ./delimiter'] system(*cmd) -- 2.23.0

1 0

[PATCH v1 compass-ci 1/3] extract-stats: close etcd client
by Cao Xueliang 04 Apr '21

04 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/extract_stats.cr | 11 +++++++---- src/extract-stats/stats_worker.cr | 5 +++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/extract-stats/extract_stats.cr b/src/extract-stats/extract_stats.cr index 4ae3e71..c35d3ec 100644 --- a/src/extract-stats/extract_stats.cr +++ b/src/extract-stats/extract_stats.cr @@ -6,10 +6,13 @@ require "./constants" require "./stats_worker" module ExtractStats + @@ec = EtcdClient.new + def self.consume_tasks channel = Channel(String).new - revision = self.consume_by_list(EXTRACT_STATS_QUEUE_PATH, channel) - self.consume_by_watch(EXTRACT_STATS_QUEUE_PATH, revision, channel) + queue = EXTRACT_STATS_QUEUE_PATH + revision = self.consume_by_list(queue, channel) + self.consume_by_watch(queue, revision, channel) end def self.consume_by_list(queue, channel) @@ -26,7 +29,7 @@ module ExtractStats def self.get_history_tasks(queue) tasks = [] of Etcd::Model::Kv - range = EtcdClient.new.range_prefix(queue) + range = @@ec.range_prefix(queue) revision = range.header.not_nil!.revision tasks += range.kvs @@ -45,7 +48,7 @@ module ExtractStats end def self.watch_queue(queue, revision, channel) - watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64, filters: [Etcd::Watch::Filter::NODELETE]) do |events| + watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64 + 1, filters: [Etcd::Watch::Filter::NODELETE]) do |events| events.each do |event| puts event channel.send(event.kv.key) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index 8910951..45b75a3 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -1,6 +1,5 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. - require "../lib/etcd_client" require "../scheduler/elasticsearch_client" require "../scheduler/redis_client" @@ -30,6 +29,8 @@ class StatsWorker STDERR.puts e.message # incase of many error message when task-queue, ES does not work sleep(10) + ensure + @etcd.close end end @@ -71,7 +72,7 @@ class StatsWorker sample_error_id = new_error_ids.sample STDOUT.puts "send a delimiter task: job_id is #{job_id}" queue = "#{DELIMITER_TASK_QUEUE}/#{job_id}" - value = {"error_id" => sample_error_id} + value = {"job_id" => job_id, "error_id" => sample_error_id} @etcd.put(queue, value) msg = %({"job_id": "#{job_id}", "new_error_id": "#{sample_error_id}"}) -- 2.23.0

1 0

[PATCH compass-ci] extract-stats: close etcd client
by Cao Xueliang 03 Apr '21

03 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/extract_stats.cr | 9 ++++++--- src/extract-stats/stats_worker.cr | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/extract-stats/extract_stats.cr b/src/extract-stats/extract_stats.cr index 4ae3e71..3346d12 100644 --- a/src/extract-stats/extract_stats.cr +++ b/src/extract-stats/extract_stats.cr @@ -6,10 +6,13 @@ require "./constants" require "./stats_worker" module ExtractStats + @@ec = EtcdClient.new + def self.consume_tasks channel = Channel(String).new - revision = self.consume_by_list(EXTRACT_STATS_QUEUE_PATH, channel) - self.consume_by_watch(EXTRACT_STATS_QUEUE_PATH, revision, channel) + queue = EXTRACT_STATS_QUEUE_PATH + revision = self.consume_by_list(queue, channel) + self.consume_by_watch(queue, revision, channel) end def self.consume_by_list(queue, channel) @@ -26,7 +29,7 @@ module ExtractStats def self.get_history_tasks(queue) tasks = [] of Etcd::Model::Kv - range = EtcdClient.new.range_prefix(queue) + range = @@ec.range_prefix(queue) revision = range.header.not_nil!.revision tasks += range.kvs diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index 8910951..9dea267 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -1,6 +1,5 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. - require "../lib/etcd_client" require "../scheduler/elasticsearch_client" require "../scheduler/redis_client" @@ -30,6 +29,8 @@ class StatsWorker STDERR.puts e.message # incase of many error message when task-queue, ES does not work sleep(10) + ensure + @etcd.close end end -- 2.23.0

1 0

review star
by Wu Fengguang 02 Apr '21

02 Apr '21

Cao Xueliang is our review star, thanks Cao Xueliang! baijing@crystal ~/todo/bin% ./review-star Review email records in the last 7 days total review paragraph point people 81 11 11 25 Cao Xueliang 59 12 19 12 Wei Jihui 24 13 28 5 Luan Shengde 22 7 6 3 Cui Yili 19 3 4 3 Wu Zhende 16 3 5 2 Yu Chuan 12 5 5 2 Xu Xijian 2 1 1 1 Wu Fengguang 25 0 0 1 Li Ping 17 1 1 0 Wang Chenglong 20 0 0 0 Liu Yinsi 6 0 0 0 Lu Weitao 1 0 0 0 Zhang Dewan 20 1 1 0 Wang Yong 6 0 0 0 Li Yuanchao 1 0 0 0 Bai Jing 9 4 4 0 Lin Jiaxin 18 5 4 0 Liu Shaofei

1 0

review star
by Wu Fengguang 02 Apr '21

02 Apr '21

Luan Shengde is our review star, thanks Luan Shengde! baijing@crystal ~/todo/bin% ./review-star Review email records in the last 7 days total review paragraph point people 81 11 11 25 Cao Xueliang 59 12 19 12 Wei Jihui 24 13 28 5 Luan Shengde 22 7 6 3 Cui Yili 19 3 4 3 Wu Zhende 16 3 5 2 Yu Chuan 12 5 5 2 Xu Xijian 2 1 1 1 Wu Fengguang 25 0 0 1 Li Ping 17 1 1 0 Wang Chenglong 20 0 0 0 Liu Yinsi 6 0 0 0 Lu Weitao 1 0 0 0 Zhang Dewan 20 1 1 0 Wang Yong 6 0 0 0 Li Yuanchao 1 0 0 0 Bai Jing 9 4 4 0 Lin Jiaxin 18 5 4 0 Liu Shaofei

2 1