- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci] service/scheduler: update data before get job
by Wu Zhende 06 Apr '21

06 Apr '21

If no job, the session is suspended until the job is obtained. Therefore, testbox message and job events need to be updated before get job to ensure the timeliness. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- src/lib/sched.cr | 11 +++++++++++ src/scheduler/find_job_boot.cr | 16 ++++++++-------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/lib/sched.cr b/src/lib/sched.cr index 2e4fd30..f62f83e 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -147,6 +147,17 @@ class Sched @log.warn(e) end + def send_mq_msg(job_state) + mq_msg = { + "job_id" => @env.get?("job_id").to_s, + "testbox" => @env.get?("testbox").to_s, + "deadline" => @env.get?("deadline").to_s, + "time" => get_time, + "job_state" => job_state + } + spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) + end + def set_lifecycle(job, testbox, queues) if job deadline = job.get_deadline diff --git a/src/scheduler/find_job_boot.cr b/src/scheduler/find_job_boot.cr index 01b6f1d..2fca531 100644 --- a/src/scheduler/find_job_boot.cr +++ b/src/scheduler/find_job_boot.cr @@ -25,14 +25,7 @@ class Sched rescue e @log.warn(e) ensure - mq_msg = { - "job_id" => @env.get?("job_id").to_s, - "testbox" => @env.get?("testbox").to_s, - "deadline" => @env.get?("deadline").to_s, - "time" => get_time, - "job_state" => "boot" - } - spawn mq_publish_confirm(JOB_MQ, mq_msg.to_json) + send_mq_msg("boot") end # auto submit a job to collect the host information. @@ -196,6 +189,13 @@ class Sched def get_job_boot(host, boot_type) queues = get_queues(host) + + # do before get job from etcd + # because if no job will hang + # need to update information in a timely manner + set_lifecycle(nil, host, queues) + send_mq_msg("boot") + job = get_job_from_queues(queues, host) set_lifecycle(job, host, queues) -- 2.23.0

1 0

[PATCH lkp-tests 2/2] add script for fix dockerfile file
by Bai Jing 06 Apr '21

06 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- lib/handle_dockerfile.sh | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 lib/handle_dockerfile.sh diff --git a/lib/handle_dockerfile.sh b/lib/handle_dockerfile.sh new file mode 100644 index 000000000..b71f9865d --- /dev/null +++ b/lib/handle_dockerfile.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +handle_FROM() +{ + sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" +} + +handle_epel() +{ + sed -i "s/epel-release/bash/g" "$dockerfile_name" +} + +handle_epelrpm() +{ + sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" +} + +handle_rpm_gpg() +{ + sed -i "s|RPM-GPG-KEY-CentOS-7|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" + sed -i "s|RPM-GPG-KEY-centosofficial|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" +} + +add_base_commands() +{ + # grep -qw "groupadd*" "$file" && { + # fix missing useradd, groupadd, chpasswd, etc. commands + sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" +} + +handle_epel_repo() +{ + sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" +} + +handle_error_exit() +{ + sed '/^RUN/RUN set -e;/g' "$dockerfile_name" +} + +handle_dockerfile() +{ + dockerfile_name=$1 + handle_FROM + handle_epel + handle_epelrpm + handle_rpm_gpg + add_base_commands + handle_epel_repo + handle_error_exit +} -- 2.23.0

1 0

[PATCH lkp-tests 1/2] add test script for dockerfile test
by Bai Jing 06 Apr '21

06 Apr '21

Signed-off-by: Bai Jing <799286817(a)qq.com> --- tests/openeuler_docker | 143 +++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 85 deletions(-) diff --git a/tests/openeuler_docker b/tests/openeuler_docker index 483f40135..3ea57e2c9 100755 --- a/tests/openeuler_docker +++ b/tests/openeuler_docker @@ -8,12 +8,12 @@ . $LKP_SRC/lib/debug.sh . $LKP_SRC/lib/log.sh +. $LKP_SRC/lib/handle_dockerfile.sh [ -n "$dockerfile_url" ] || die "dockerfile_url is empty" declare -i nr_openeuler_fail=0 declare -i nr_openeuler_success=0 -declare -i is_openeuler_build_fail=0 declare -i dockerfile_count=0 fit_repo() @@ -33,110 +33,51 @@ openeuler_ready() docker load < $BENCHMARK_ROOT/openeuler_docker/openeuler-20.03-lts-v1.tar } -centos_docker() -{ - is_openeuler_build_fail=0 - index=1 - for file in $dockerfile_path - do - [[ $file =~ ^centos ]] && continue - file=$(echo "$file" | awk -F: '{print $1}') - dockerfile_name="${file##*/}" - cd "${file%/*}" || continue - image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') - [ -n "$image" ] && image=${image}_$index - if [ -f "build.sh" ] && [ -x "build.sh" ]; then - ./build.sh > /dev/null 2> /dev/null - else - docker_build > /dev/null 2> /dev/null - fi - - [ $? == 0 ] && { - handle_dockerfile - } - - let index++ - done -} - -handle_FROM() -{ - sed -i "s/^FROM .*/FROM openeuler-20.03-lts:v1/g" "$dockerfile_name" -} - -handle_epel() -{ - grep -qw "epel-release" "$dockerfile_name" && { - sed -i "s/epel-release/bash/g" "$dockerfile_name" - } -} - -handle_epelrpm() -{ - grep -qw "https://.*epel-release.*rpm" "$dockerfile_name" && { - sed -i "s|https://.*epel-release.*rpm|bash|g" "$dockerfile_name" - } -} - -handle_rpm_gpg() -{ - sed -i "s|RPM-GPG-KEY-CentOS-7|RPM-GPG-KEY-openEuler|g" "$dockerfile_name" -} - -add_base_commands() -{ - # grep -qw "groupadd*" "$file" && { - # fix missing useradd, groupadd, chpasswd, etc. commands - sed -i '/FROM /a\RUN yum -y install shadow tar' "$dockerfile_name" -} - -handle_epel_repo() -{ - sed -i '/.* wget .*(epel|CentOS-Base)\.repo/ s|^|#|g' "$dockerfile_name" -} - -handle_dockerfile() +build_centos() { - is_openeuler_build_fail=1 - handle_FROM - handle_epel - handle_rpm_gpg - add_base_commands - if docker_build "-openeuler"; then - docker_run "-openeuler" - let nr_openeuler_success++ + cd ${file%/*} + if [ -f "build.sh" ] && [ -x "build.sh" ]; then + ./build.sh || return 1 else - let nr_openeuler_fail++ + docker build -f "$dockerfile_name" -t "${image}_$1_$index" . || return 1 fi } -docker_build() +build_openeuler() { - docker build -f "$dockerfile_name" -t "${image}$1" . + docker build -f "$dockerfile_name" -t "${image}_$1_$index" . >> $TMP_RESULT_ROOT/${image}_$1_$index 2>&1 [ $? != 0 ] && { - [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.fail: 1" + let nr_openeuler_fail++ + echo "${image}_$index.build.fail: 1" return 1 } - [ "$is_openeuler_build_fail" == "1" ] && echo "${image}$1.build.success: 1" - is_openeuler_build_fail=0 + echo "${image}_$index.build.success: 1" + let nr_openeuler_success++ + return 0 } docker_run() -{ - docker run -d --name "${image}" "${image}$1" - docker_stats=$(docker inspect -f '{{.State.Running}}' ${image}) - [ $docker_stats == false ] && { - echo "${image}.run.fail: 1" +{ + docker_run_cmd="docker run -d --name ${image}_$1_$index ${image}_$1_$index" + [ $1 == "openeuler" ] && { + docker_run_cmd="${docker_run_cmd} >> $TMP_RESULT_ROOT/${image}_$1_$index 2>&1" + } + + eval $docker_run_cmd + + docker_stats=$(docker inspect -f '{{.State.Running}}' $image_$1_$index) + [ $docker_stats == "false" ] && { + [ $1 == "openeuler" ] && echo "${image}_$index.run.fail: 1" return 1 } - echo "${image}.run.success: 1" + [ $1 == "openeuler"] && echo "${image}_$index.run.success: 1" } handle_build_result() { if [ "$nr_openeuler_fail" != 0 ]; then - echo "openeuler build failed times: $nr_openeuler_fail" + echo "openeuler_build.nr_failed: $nr_openeuler_fail" echo "openeuler_valid_url: $dockerfile_url" exit 1 # centos all success, openeuler all success @@ -146,6 +87,38 @@ handle_build_result() fi } +build() +{ + dockerfile_name="${file##*/}" + dockerfile_name="${dockerfile_name%:*}" + image=$(echo "$file" | awk -F '/' '{print $5}' | tr 'A-Z' 'a-z') + + build_centos "centos" > /dev/null 2>&1 || return 1 + handle_dockerfile "$dockerfile_name" + build_openeuler "openeuler" + [ $? != 0 ] && return 1 +} + +run() +{ + docker_run "centos" > /dev/null 2>&1 + [ $? != 0 ] && return 1 + docker_run "openeuler" +} + +centos_docker() +{ + index=1 + for file in $dockerfile_path + do + [[ $file =~ ^centos ]] && continue + + build + [ $? = 0 ] && run + let index++ + done +} + docker_test() { fit_repo -- 2.23.0

1 0

[PATCH compass-ci] sbin/es-find: support check query keyword
by Lu Weitao 06 Apr '21

06 Apr '21

[How] 1. add --keyword option 2. use like: es-find --keyword category dequeue_time end_time enqueue_time group_id hw.arch hw.memory hw.model hw.nr_cpu ... Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/es_query.rb | 25 +++++++++++++++++++++++++ sbin/es-find | 11 ++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/lib/es_query.rb b/lib/es_query.rb index 928bd68..22d68a9 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -130,6 +130,11 @@ class ESQuery parse_fields(result) end + + def query_mapping + mapping = @client.indices.get_mapping(index: @index) + get_mapping_key(mapping) + end end # Range Query Example: @@ -260,3 +265,23 @@ def parse_fields(es_result) result_hash end + +def get_mapping_key(mapping) + mapping_key = [] + properties = mapping.values[0]['mappings']['properties'] + properties.each do |key, value| + parse_mapping_properties(key, value, mapping_key) + end + + mapping_key +end + +def parse_mapping_properties(key, value, mapping_key) + if value['type'] + mapping_key << key + else + value['properties'].each do |k, v| + parse_mapping_properties("#{key}.#{k}", v, mapping_key) + end + end +end diff --git a/sbin/es-find b/sbin/es-find index 2c3b3dc..f548fc1 100755 --- a/sbin/es-find +++ b/sbin/es-find @@ -77,6 +77,8 @@ options = { nr_jobs: 10, desc_keyword: 'start_time' } range = nil count = nil +es = ESQuery.new(ES_HOST, ES_PORT) + opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: es-find [options] search_key1=val1[,val2..] ..' @@ -103,6 +105,14 @@ opt_parser = OptionParser.new do |opts| count = c end + opts.on_tail('--keyword', 'check all available search_keys') do + available_search_key = es.query_mapping + available_search_key.each do |key| + puts " #{key}" + end + exit + end + opts.on_tail('-h', '--help', 'show this message') do puts opts exit @@ -113,7 +123,6 @@ opt_parser.parse!(ARGV) items = parse_argv items[:range] = prase_range(range) if range -es = ESQuery.new(ES_HOST, ES_PORT) if count count_keywords = count.split(',') query_result = es.query_fields(count_keywords, items) -- 2.23.0

1 0

[PATCH v2 compass-ci] delimiter: refactor delimiter with crystal
by Cao Xueliang 04 Apr '21

04 Apr '21

use etcd list watch to submit bisect job Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter.cr | 22 ++++++++++ src/delimiter/delimiter.cr | 83 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 src/delimiter.cr create mode 100644 src/delimiter/delimiter.cr diff --git a/src/delimiter.cr b/src/delimiter.cr new file mode 100644 index 0000000..e8de306 --- /dev/null +++ b/src/delimiter.cr @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "yaml" +require "./delimiter/delimiter" + +def config_secrets_yaml + %x(#{ENV["CCI_SRC"]}/sbin/config_account_yaml.rb delimiter) + account = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/defaults/account.yaml")) + lab = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/include/lab/#{account["lab"]}.yaml")) + secrets = {"secrets" => lab} + File.open("#{ENV["HOME"]}/.config/compass-ci/defaults/secrets.yaml", "w") { |f| YAML.dump(secrets, f) } +end + +begin + config_secrets_yaml + delimiter = Delimiter.new + delimiter.consume_delimiter("delimiter") +rescue ex + puts ex +end diff --git a/src/delimiter/delimiter.cr b/src/delimiter/delimiter.cr new file mode 100644 index 0000000..c98ded9 --- /dev/null +++ b/src/delimiter/delimiter.cr @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "../lib/etcd_client" + +class Delimiter + def initialize + @ec = EtcdClient.new + end + + def consume_delimiter(queue) + channel = Channel(Etcd::Model::Kv).new + revision = consume_by_list(queue, channel) + consume_by_watch(queue, revision, channel) + end + + def consume_by_list(queue, channel) + tasks, revision = get_history_tasks(queue) + handle_history_tasks(tasks, channel) + + return revision + end + + def consume_by_watch(queue, revision, channel) + watch_queue(queue, revision, channel) + handle_events(channel) + end + + def get_history_tasks(queue) + tasks = [] of Etcd::Model::Kv + range = @ec.range_prefix(queue) + revision = range.header.not_nil!.revision + tasks += range.kvs + + return tasks, revision + end + + def handle_history_tasks(tasks, channel) + loop do + return if tasks.empty? + + task = tasks.delete_at(0) + spawn { submit_bisect_job(channel, task) } + end + end + + def watch_queue(queue, revision, channel) + watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64 + 1, filters: [Etcd::Watch::Filter::NODELETE]) do |events| + events.each do |event| + channel.send(event.kv) + end + end + + spawn { watcher.start } + Fiber.yield + end + + def handle_events(channel) + loop do + task = channel.receive + spawn { submit_bisect_job(channel, task) } + end + end + + def submit_bisect_job(channel, task) + key = task.key + value = Hash(String, String).from_json(task.value.not_nil!) + begin + response = %x(#{ENV["LKP_SRC"]}/sbin/submit bad_job_id=#{value["job_id"]} error_id=#{value["error_id"].inspect} bisect.yaml queue=dc-bisect) + puts response + if /id=0/ =~ response + channel.send(task) + return + end + + @ec.delete(key) + rescue ex + puts ex + channel.send(task) + end + end +end -- 2.23.0

1 0

[PATCH v1 compass-ci 3/3] delimiter: refactor delimiter whit crystal
by Cao Xueliang 04 Apr '21

04 Apr '21

use etcd list watch to submit bisect job Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/delimiter.cr | 22 ++++++++++ src/delimiter/delimiter.cr | 83 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 src/delimiter.cr create mode 100644 src/delimiter/delimiter.cr diff --git a/src/delimiter.cr b/src/delimiter.cr new file mode 100644 index 0000000..e8de306 --- /dev/null +++ b/src/delimiter.cr @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "yaml" +require "./delimiter/delimiter" + +def config_secrets_yaml + %x(#{ENV["CCI_SRC"]}/sbin/config_account_yaml.rb delimiter) + account = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/defaults/account.yaml")) + lab = YAML.parse(File.read("#{ENV["HOME"]}/.config/compass-ci/include/lab/#{account["lab"]}.yaml")) + secrets = {"secrets" => lab} + File.open("#{ENV["HOME"]}/.config/compass-ci/defaults/secrets.yaml", "w") { |f| YAML.dump(secrets, f) } +end + +begin + config_secrets_yaml + delimiter = Delimiter.new + delimiter.consume_delimiter("delimiter") +rescue ex + puts ex +end diff --git a/src/delimiter/delimiter.cr b/src/delimiter/delimiter.cr new file mode 100644 index 0000000..c98ded9 --- /dev/null +++ b/src/delimiter/delimiter.cr @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require "../lib/etcd_client" + +class Delimiter + def initialize + @ec = EtcdClient.new + end + + def consume_delimiter(queue) + channel = Channel(Etcd::Model::Kv).new + revision = consume_by_list(queue, channel) + consume_by_watch(queue, revision, channel) + end + + def consume_by_list(queue, channel) + tasks, revision = get_history_tasks(queue) + handle_history_tasks(tasks, channel) + + return revision + end + + def consume_by_watch(queue, revision, channel) + watch_queue(queue, revision, channel) + handle_events(channel) + end + + def get_history_tasks(queue) + tasks = [] of Etcd::Model::Kv + range = @ec.range_prefix(queue) + revision = range.header.not_nil!.revision + tasks += range.kvs + + return tasks, revision + end + + def handle_history_tasks(tasks, channel) + loop do + return if tasks.empty? + + task = tasks.delete_at(0) + spawn { submit_bisect_job(channel, task) } + end + end + + def watch_queue(queue, revision, channel) + watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64 + 1, filters: [Etcd::Watch::Filter::NODELETE]) do |events| + events.each do |event| + channel.send(event.kv) + end + end + + spawn { watcher.start } + Fiber.yield + end + + def handle_events(channel) + loop do + task = channel.receive + spawn { submit_bisect_job(channel, task) } + end + end + + def submit_bisect_job(channel, task) + key = task.key + value = Hash(String, String).from_json(task.value.not_nil!) + begin + response = %x(#{ENV["LKP_SRC"]}/sbin/submit bad_job_id=#{value["job_id"]} error_id=#{value["error_id"].inspect} bisect.yaml queue=dc-bisect) + puts response + if /id=0/ =~ response + channel.send(task) + return + end + + @ec.delete(key) + rescue ex + puts ex + channel.send(task) + end + end +end -- 2.23.0

1 0

[PATCH v1 compass-ci 2/3] container/delimiter: compile delimiter with crystal
by Cao Xueliang 04 Apr '21

04 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- container/delimiter/Dockerfile | 11 ++++++++++- container/delimiter/build | 4 ++++ container/delimiter/start | 3 +-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/container/delimiter/Dockerfile b/container/delimiter/Dockerfile index cc57320..a9ff279 100644 --- a/container/delimiter/Dockerfile +++ b/container/delimiter/Dockerfile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM alpine:3.13 +FROM alpine:scheduler-dev RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories && \ apk update && \ @@ -11,3 +11,12 @@ RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweiclou RUN umask 002 && \ gem sources -r https://rubygems.org/ -a https://gems.ruby-china.com/ && \ gem install rest-client activesupport git json yaml elasticsearch faye-websocket terminal-table mail io-console + +ENV RUNTIME_DIR /c/cci/delimiter + +RUN mkdir -p $RUNTIME_DIR && \ + chown -R 1090:1090 /c + +WORKDIR $RUNTIME_DIR + +COPY --chown=1090:1090 delimiter . diff --git a/container/delimiter/build b/container/delimiter/build index 606149d..7f3108b 100755 --- a/container/delimiter/build +++ b/container/delimiter/build @@ -8,6 +8,10 @@ docker_skip_rebuild "delimiter" load_cci_defaults +bash $CCI_SRC/container/compile . + docker build -t delimiter . push_image delimiter:latest + +rm delimiter diff --git a/container/delimiter/start b/container/delimiter/start index 3cff23a..9485750 100755 --- a/container/delimiter/start +++ b/container/delimiter/start @@ -32,10 +32,9 @@ cmd = %w[ -v #{DEFAULT_CONFIG_DIR}:#{DEFAULT_CONFIG_DIR}:ro -v /etc/localtime:/etc/localtime:ro -v /etc/compass-ci/register:/etc/compass-ci/register:ro - -w #{DEFAULT_CCI}/src delimiter ] -cmd += ['sh', '-c', 'umask 002 && ruby ./delimiter.rb'] +cmd += ['sh', '-c', 'umask 002 && ./delimiter'] system(*cmd) -- 2.23.0

1 0

[PATCH v1 compass-ci 1/3] extract-stats: close etcd client
by Cao Xueliang 04 Apr '21

04 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/extract_stats.cr | 11 +++++++---- src/extract-stats/stats_worker.cr | 5 +++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/extract-stats/extract_stats.cr b/src/extract-stats/extract_stats.cr index 4ae3e71..c35d3ec 100644 --- a/src/extract-stats/extract_stats.cr +++ b/src/extract-stats/extract_stats.cr @@ -6,10 +6,13 @@ require "./constants" require "./stats_worker" module ExtractStats + @@ec = EtcdClient.new + def self.consume_tasks channel = Channel(String).new - revision = self.consume_by_list(EXTRACT_STATS_QUEUE_PATH, channel) - self.consume_by_watch(EXTRACT_STATS_QUEUE_PATH, revision, channel) + queue = EXTRACT_STATS_QUEUE_PATH + revision = self.consume_by_list(queue, channel) + self.consume_by_watch(queue, revision, channel) end def self.consume_by_list(queue, channel) @@ -26,7 +29,7 @@ module ExtractStats def self.get_history_tasks(queue) tasks = [] of Etcd::Model::Kv - range = EtcdClient.new.range_prefix(queue) + range = @@ec.range_prefix(queue) revision = range.header.not_nil!.revision tasks += range.kvs @@ -45,7 +48,7 @@ module ExtractStats end def self.watch_queue(queue, revision, channel) - watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64, filters: [Etcd::Watch::Filter::NODELETE]) do |events| + watcher = EtcdClient.new.watch_prefix(queue, start_revision: revision.to_i64 + 1, filters: [Etcd::Watch::Filter::NODELETE]) do |events| events.each do |event| puts event channel.send(event.kv.key) diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index 8910951..45b75a3 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -1,6 +1,5 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. - require "../lib/etcd_client" require "../scheduler/elasticsearch_client" require "../scheduler/redis_client" @@ -30,6 +29,8 @@ class StatsWorker STDERR.puts e.message # incase of many error message when task-queue, ES does not work sleep(10) + ensure + @etcd.close end end @@ -71,7 +72,7 @@ class StatsWorker sample_error_id = new_error_ids.sample STDOUT.puts "send a delimiter task: job_id is #{job_id}" queue = "#{DELIMITER_TASK_QUEUE}/#{job_id}" - value = {"error_id" => sample_error_id} + value = {"job_id" => job_id, "error_id" => sample_error_id} @etcd.put(queue, value) msg = %({"job_id": "#{job_id}", "new_error_id": "#{sample_error_id}"}) -- 2.23.0

1 0

[PATCH compass-ci] extract-stats: close etcd client
by Cao Xueliang 03 Apr '21

03 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- src/extract-stats/extract_stats.cr | 9 ++++++--- src/extract-stats/stats_worker.cr | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/extract-stats/extract_stats.cr b/src/extract-stats/extract_stats.cr index 4ae3e71..3346d12 100644 --- a/src/extract-stats/extract_stats.cr +++ b/src/extract-stats/extract_stats.cr @@ -6,10 +6,13 @@ require "./constants" require "./stats_worker" module ExtractStats + @@ec = EtcdClient.new + def self.consume_tasks channel = Channel(String).new - revision = self.consume_by_list(EXTRACT_STATS_QUEUE_PATH, channel) - self.consume_by_watch(EXTRACT_STATS_QUEUE_PATH, revision, channel) + queue = EXTRACT_STATS_QUEUE_PATH + revision = self.consume_by_list(queue, channel) + self.consume_by_watch(queue, revision, channel) end def self.consume_by_list(queue, channel) @@ -26,7 +29,7 @@ module ExtractStats def self.get_history_tasks(queue) tasks = [] of Etcd::Model::Kv - range = EtcdClient.new.range_prefix(queue) + range = @@ec.range_prefix(queue) revision = range.header.not_nil!.revision tasks += range.kvs diff --git a/src/extract-stats/stats_worker.cr b/src/extract-stats/stats_worker.cr index 8910951..9dea267 100644 --- a/src/extract-stats/stats_worker.cr +++ b/src/extract-stats/stats_worker.cr @@ -1,6 +1,5 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. - require "../lib/etcd_client" require "../scheduler/elasticsearch_client" require "../scheduler/redis_client" @@ -30,6 +29,8 @@ class StatsWorker STDERR.puts e.message # incase of many error message when task-queue, ES does not work sleep(10) + ensure + @etcd.close end end -- 2.23.0

1 0

review star
by Wu Fengguang 02 Apr '21

02 Apr '21

Cao Xueliang is our review star, thanks Cao Xueliang! baijing@crystal ~/todo/bin% ./review-star Review email records in the last 7 days total review paragraph point people 81 11 11 25 Cao Xueliang 59 12 19 12 Wei Jihui 24 13 28 5 Luan Shengde 22 7 6 3 Cui Yili 19 3 4 3 Wu Zhende 16 3 5 2 Yu Chuan 12 5 5 2 Xu Xijian 2 1 1 1 Wu Fengguang 25 0 0 1 Li Ping 17 1 1 0 Wang Chenglong 20 0 0 0 Liu Yinsi 6 0 0 0 Lu Weitao 1 0 0 0 Zhang Dewan 20 1 1 0 Wang Yong 6 0 0 0 Li Yuanchao 1 0 0 0 Bai Jing 9 4 4 0 Lin Jiaxin 18 5 4 0 Liu Shaofei

1 0