- Compass-ci - mailweb.openeuler.org

[PATCH compass-ci 1/3] service/logging-es: configuring the service account and password
by Wu Zhende 21 Apr '21

21 Apr '21

Improve security Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/defconfig.rb | 8 ++++++++ container/logging-es/Dockerfile | 7 +++++++ container/logging-es/build | 9 ++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/container/defconfig.rb b/container/defconfig.rb index b121de8..ad94940 100755 --- a/container/defconfig.rb +++ b/container/defconfig.rb @@ -21,6 +21,14 @@ def relevant_defaults(names) cci_defaults.select { |k, _| names.include? k } end +def relevant_service_account(names) + hash = {} + Dir.glob(['/etc/compass-ci/passwd.yaml']).each do |file| + hash.update YAML.load_file(file) || {} + end + hash.select { |k, _| names.include? k } +end + def set_local_env hash = cci_defaults hash.map { |k, v| system "export #{k}=#{v}" } diff --git a/container/logging-es/Dockerfile b/container/logging-es/Dockerfile index a0acf94..f66dc35 100644 --- a/container/logging-es/Dockerfile +++ b/container/logging-es/Dockerfile @@ -5,6 +5,8 @@ ARG BASE_IMAGE FROM $BASE_IMAGE ARG MEMORY +ARG USER +ARG PASSWORD # docker image borrowed from hub.docker.com/r/gagara/elasticsearch-oss-arm64 @@ -16,6 +18,9 @@ RUN sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/elastic sed -i '$a cluster.initial_master_nodes: ["node-1"]' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a indices.memory.index_buffer_size: 20%' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i '$a thread_pool.write.queue_size: 2000' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.security.enabled: true' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.license.self_generated.type: basic' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a xpack.security.transport.ssl.enabled: true' /usr/share/elasticsearch/config/elasticsearch.yml && \ sed -i "s/-Xms1g/-Xms${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options && \ sed -i "s/-Xmx1g/-Xmx${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options @@ -24,6 +29,8 @@ RUN mkdir /usr/share/elasticsearch/tmp && \ WORKDIR /usr/share/elasticsearch +RUN ./bin/elasticsearch-users useradd ${USER} -p ${PASSWORD} -r superuser + ENV PATH /usr/share/elasticsearch/bin:$PATH ENV ES_TMPDIR /usr/share/elasticsearch/tmp diff --git a/container/logging-es/build b/container/logging-es/build index b50830e..ff1f102 100755 --- a/container/logging-es/build +++ b/container/logging-es/build @@ -3,8 +3,15 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig.rb' +names = Set.new %w[ + LOGGING_ES_USER + LOGGING_ES_PASSWORD +] + +defaults = relevant_service_account(names) docker_skip_rebuild "logging-es:7.11.1" BASE_IMAGE_DICT = { @@ -16,4 +23,4 @@ BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] available_memory = get_available_memory -system "docker build -t logging-es:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg MEMORY=#{available_memory} ." +system "docker build -t logging-es:7.11.1 --progress=plain --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg MEMORY=#{available_memory} --build-arg USER=#{defaults['LOGGING_ES_USER']} --build-arg PASSWORD=#{defaults['LOGGING_ES_PASSWORD']} ." -- 2.23.0

2 1

[PATCH compass-ci] assist-result-client: fix git bisect email subject is the "["
by Cao Xueliang 21 Apr '21

21 Apr '21

Need use json parse the data, before return the response. Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- lib/assist_result_client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/assist_result_client.rb b/lib/assist_result_client.rb index 592c841..2d2e60d 100644 --- a/lib/assist_result_client.rb +++ b/lib/assist_result_client.rb @@ -60,6 +60,6 @@ class AssistResult response = resource.post(Base64.encode64(data.to_json)) return nil unless response.code == 200 - return response.body + return JSON.parse(response.body) end end -- 2.23.0

1 0

[PATCH compass-ci] sparrow/dev-env: add the host/port to the job by scheduler
by Cao Xueliang 21 Apr '21

21 Apr '21

Signed-off-by: Cao Xueliang <caoxl78320(a)163.com> --- sparrow/3-code/dev-env | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sparrow/3-code/dev-env b/sparrow/3-code/dev-env index e02de35..a478fff 100755 --- a/sparrow/3-code/dev-env +++ b/sparrow/3-code/dev-env @@ -163,6 +163,11 @@ RESULT_WEBDAV_HOST: RESULT_WEBDAV_PORT: DOCKER_REGISTRY_HOST: DOCKER_REGISTRY_PORT: +GIT_MIRROR_HOST: +JUMPER_HOST: +JUMPER_PORT: +MONITOR_HOST: +MONITOR_PORT: EOF cat > /etc/profile.d/compass.sh <<'EOF' -- 2.23.0

1 0

[PATCH lkp-tests 2/4] add config file for mysql-8.0.20 sysbench test
by Luan Shengde 21 Apr '21

21 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- pkg/mysql-8-0-20/my.cnf | 67 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 pkg/mysql-8-0-20/my.cnf diff --git a/pkg/mysql-8-0-20/my.cnf b/pkg/mysql-8-0-20/my.cnf new file mode 100644 index 000000000..c657234df --- /dev/null +++ b/pkg/mysql-8-0-20/my.cnf @@ -0,0 +1,67 @@ +[mysqld_safe] +log-error=/data/mysql/log/mysql.log +pid-file=/data/mysql/run/mysqld.pid +[mysqldump] +quick +[mysql] +[client] +default-character-set=utf8mb4 +socket=/data/mysql/run/mysql.sock + +[mysqld] +socket=/data/mysql/run/mysql.sock +tmpdir=/data/mysql/tmp +datadir=/data/mysql/data +default_authentication_plugin=mysql_native_password +user=mysql +memlock=on +bind-address=0.0.0.0 +skip-name-resolve +skip-grant-tables + +lc-messages=en_US +lc-messages-dir=/usr/local/mysql/share + +innodb_page_size=64k +max_connections=4000 +back_log=2048 +performance_schema=OFF +max_prepared_stmt_count=128000 + +#file +innodb_file_per_table +innodb_log_file_size=2048M +innodb_log_files_in_group=32 +innodb_open_files=10000 +table_open_cache_instances=64 + +#buffers +innodb_buffer_pool_size=128G +innodb_buffer_pool_instances=64 +innodb_log_buffer_size=2048M + +#tune +sync_binlog=0 +innodb_flush_log_at_trx_commit=0 +innodb_use_native_aio=1 +innodb_spin_wait_delay=160 +innodb_sync_spin_loops=20 +innodb_flush_method=O_DIRECT +innodb_io_capacity=30000 +innodb_io_capacity_max=40000 +innodb_lru_scan_depth=9000 +innodb_page_cleaners=1 +innodb_spin_wait_pause_multiplier=25 + +#perf special +innodb_flush_neighbors=0 +innodb_write_io_threads=1 +innodb_read_io_threads=1 +innodb_purge_threads=1 + +sql_mode=STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION,NO_AUTO_VALUE_ON_ZERO,STRICT_ALL_TABLES + +#skop_log_bin +ssl=0 + +table_open_cache=15000 -- 2.23.0

1 0

[PATCH lkp-tests 1/4] tests: script to start mysql
by Luan Shengde 21 Apr '21

21 Apr '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- tests/mysql-8-0-20 | 73 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 tests/mysql-8-0-20 diff --git a/tests/mysql-8-0-20 b/tests/mysql-8-0-20 new file mode 100755 index 000000000..9b00766f0 --- /dev/null +++ b/tests/mysql-8-0-20 @@ -0,0 +1,73 @@ +#!/bin/sh +# - data_dev +# mysql_user +# mysql_host +# mysql_port +# mysql_db +# mysql_password +. $LKP_SRC/lib/common.sh + +: "${mysql_user:=root}" +: "${mysql_host:=localhost}" +: "${mysql_port:=3306}" +: "${mysql_db:=sysbench_test}" +: "${mysql_password:=mysql_password}" + +create_basedir() +{ + mkdir -p /data + mkfs.xfs -f /dev/"$data_dev" + mount /dev/"$data_dev" /data + + mkdir -p /data/mysql/{data,share,tmp,run,log} + + useradd mysql + touch /data/mysql/log/mysql.log + chown -R mysql:mysql /data + chown -R mysql:mysql /data/mysql/log/mysql.log +} + +configure_mysql() +{ + if [ ! -f $mysqld ] || [ ! -f $mysql ] || [ ! -f $mysql_server ]; then + echo "mysql is not installed!" && exit 1 + fi + + "$mysqld" --user=mysql --basedir=/data/mysql --datadir=/data/mysql/data/ --initialize --port="$mysql_port" + sleep 10 + + "$mysql_server" start + sleep 10 + + "$mysql" -u "$mysql_user" -e "flush privileges; use mysql; alter user 'root'@'localhost' identified by '$mysql_password'; flush privileges;" + sed -i '/^skip-grant-tables/d' /etc/my.cnf + "$mysql_server" restart + sleep 10 +} + +create_database() +{ + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "use mysql; update user set host ='%' where user='root'; flush privileges;" + "$mysql" -u "$mysql_user" -p"$mysql_password" -e "create database $mysql_db;" + "$mysql_server" stop + sleep 10 +} + +start_mysql() +{ + "$mysqld" --defaults-file=/etc/my.cnf --port="$mysql_port" + sleep 20 +} + +main() +{ + local mysql="/usr/local/mysql/bin/mysql" + local mysqld="/usr/local/mysql/bin/mysqld" + local mysql_server="/usr/local/mysql/support-files/mysql.server" + + create_basedir + configure_mysql + create_database + start_mysql +} +main -- 2.23.0

1 0

[PATCH compass-ci] lib/dump_stat.rb: optimize warning info
by Lu Weitao 21 Apr '21

21 Apr '21

[Why] when a key - value is invalid, should show the key and value Signed-off-by: Lu Weitao <luweitaobe(a)163.com> --- lib/dump_stat.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/dump_stat.rb b/lib/dump_stat.rb index a497d461..1b56cc35 100644 --- a/lib/dump_stat.rb +++ b/lib/dump_stat.rb @@ -56,7 +56,7 @@ module DumpStat if value.is_a?(String) value = check_string_value(k, value, @monitor) next unless value - return nil unless number?(value, @invalid_records, @record_index, @monitor) + return nil unless number?(k, value, @invalid_records, @record_index, @monitor) value = value.index('.') ? value.to_f : value.to_i elsif value.is_a?(Array) @@ -65,7 +65,7 @@ module DumpStat value[i] = check_string_value(k, value[i], @monitor) next unless value[i] - return nil unless number?(value[i], @invalid_records, @record_index, @monitor) + return nil unless number?(k, value[i], @invalid_records, @record_index, @monitor) value[i] = value[i].index('.') ? value[i].to_f : value[i].to_i valid_stats_verification(k, value[i]) @@ -164,10 +164,10 @@ def check_string_value(key, value, monitor) end # only number is valid -def number?(value, invalid_records, record_index, monitor) +def number?(key, value, invalid_records, record_index, monitor) unless value.numeric? invalid_records.push record_index - warn_stat "invalid stats value: #{value}", monitor + warn_stat "invalid stats key-value: \n key: #{key} \n value: #{value}", monitor return nil end -- 2.23.0

1 0

[PATCH compass-ci 5/5] es_query: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

Everying where "ESQuery.new" is used needs to be adjusted Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/assign-account/start | 4 ++++ container/assist-result/start | 6 ++++++ container/delimiter/start | 6 ++++++ container/mail-robot/start | 5 +++++ lib/commit_status.rb | 2 +- lib/compare.rb | 4 ++-- lib/es_jobs.rb | 2 +- lib/es_query.rb | 6 ++---- sbin/es-find | 2 +- src/lib/web_backend.rb | 2 +- 10 files changed, 29 insertions(+), 10 deletions(-) diff --git a/container/assign-account/start b/container/assign-account/start index b926853..c8c55da 100755 --- a/container/assign-account/start +++ b/container/assign-account/start @@ -4,6 +4,8 @@ . ../defconfig.sh +load_service_authentication + docker_rm assign-account cmd=( @@ -11,6 +13,8 @@ cmd=( --restart=always --name=assign-account -d + -e ES_USER=$ES_USER + -e ES_PASSWORD=$ES_PASSWORD -v /etc:/etc:rw -v /srv/account-info:/opt/account_data/:rw -v /home:/home:rw diff --git a/container/assist-result/start b/container/assist-result/start index af812aa..9998588 100755 --- a/container/assist-result/start +++ b/container/assist-result/start @@ -9,11 +9,17 @@ require_relative '../defconfig.rb' names = Set.new %w[ ES_HOST ES_PORT + ES_USER + ES_PASSWORD ASSIST_RESULT_HOST ASSIST_RESULT_PORT ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' diff --git a/container/delimiter/start b/container/delimiter/start index 9485750..53adc03 100755 --- a/container/delimiter/start +++ b/container/delimiter/start @@ -9,9 +9,15 @@ require_relative '../defconfig.rb' names = Set.new %w[ TASKQUEUE_HOST TASKQUEUE_PORT + ES_USER + ES_PASSWORD ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' diff --git a/container/mail-robot/start b/container/mail-robot/start index ce18498..07aea39 100755 --- a/container/mail-robot/start +++ b/container/mail-robot/start @@ -15,6 +15,8 @@ names = Set.new %w[ SCHED_HTTPS_HOST SCHED_HTTPS_PORT ES_PORT + ES_USER + ES_PASSWORD MAILDIR ACCOUNT_ISSUER lab @@ -22,6 +24,9 @@ names = Set.new %w[ defaults = relevant_defaults(names) +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) cmd = %w[ diff --git a/lib/commit_status.rb b/lib/commit_status.rb index 81aeb29..baae530 100644 --- a/lib/commit_status.rb +++ b/lib/commit_status.rb @@ -26,7 +26,7 @@ def query_latest_good_commit(job, error_id) end def query_jobs_from_es(items) - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new result = es.multi_field_query items jobs_list = result['hits']['hits'] jobs_list.map! { |job| job['_source'] } diff --git a/lib/compare.rb b/lib/compare.rb index 7c182c4..7386cae 100644 --- a/lib/compare.rb +++ b/lib/compare.rb @@ -38,7 +38,7 @@ end def create_matrices_list(conditions) matrices_list = [] suite_list = [] - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new conditions.each do |condition| query_results = es.multi_field_query(condition, desc_keyword: 'start_time') matrix, suites = combine_query_data(query_results) @@ -71,7 +71,7 @@ def compare_group(argv, dimensions, options) end def create_groups_matrices_list(conditions, dims) - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new query_results = es.multi_field_query(conditions, desc_keyword: 'start_time') combine_group_query_data(query_results['hits']['hits'], dims) end diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index 25e54b4..3c29b66 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -23,7 +23,7 @@ class ESJobs end def query_jobs_from_es - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new result = es.multi_field_query @es_query jobs = result['hits']['hits'] jobs.map! { |job| job['_source'] } diff --git a/lib/es_query.rb b/lib/es_query.rb index 22d68a9..dd6d97d 100644 --- a/lib/es_query.rb +++ b/lib/es_query.rb @@ -7,12 +7,10 @@ require_relative 'constants.rb' # build multiple query request body class ESQuery - HOST = (ENV.key?('ES_HOST') ? ENV['ES_HOST'] : ES_HOST) - PORT = (ENV.key?('ES_PORT') ? ENV['ES_PORT'] : ES_PORT).to_i - def initialize(host = HOST, port = PORT, index: 'jobs') + def initialize(hosts = ES_HOSTS, index: 'jobs') @index = index @scroll_id = '' - @client = Elasticsearch::Client.new url: "http://#{host}:#{port}" + @client = Elasticsearch::Client.new hosts: hosts raise 'Connect Elasticsearch error!' unless @client.ping end diff --git a/sbin/es-find b/sbin/es-find index f548fc1..78e30a8 100755 --- a/sbin/es-find +++ b/sbin/es-find @@ -77,7 +77,7 @@ options = { nr_jobs: 10, desc_keyword: 'start_time' } range = nil count = nil -es = ESQuery.new(ES_HOST, ES_PORT) +es = ESQuery.new(ES_HOSTS) opt_parser = OptionParser.new do |opts| opts.banner = 'Usage: es-find [options] search_key1=val1[,val2..] ..' diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index c4d4edd..91287b4 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -657,7 +657,7 @@ def get_error_objects(filter_items) end def get_job_list(items) - es = ESQuery.new(ES_HOST, ES_PORT) + es = ESQuery.new query_results = es.multi_field_query(items) extract_jobs_list(query_results['hits']['hits']) -- 2.23.0

1 0

[PATCH compass-ci 4/5] ruby-services: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

Two ruby services: web-backend git_mirror Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/git-mirror/start | 12 +++++++++++- container/web-backend/start | 4 ++++ lib/constants.rb | 22 ++++++++++++++++++++++ lib/git_mirror.rb | 2 +- src/lib/web_backend.rb | 2 +- 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/container/git-mirror/start b/container/git-mirror/start index f01789c..9c9e3e2 100755 --- a/container/git-mirror/start +++ b/container/git-mirror/start @@ -3,8 +3,17 @@ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. # frozen_string_literal: true +require 'set' require_relative '../defconfig.rb' +names = Set.new %w[ + ES_USER + ES_PASSWORD +] + +service_authentication = relevant_service_authentication(names) +env = docker_env(service_authentication) + DOCKER_CCI = '/c/compass-ci' DOCKER_REPO = '/c/git-repos' docker_rm "git-mirror" @@ -12,10 +21,11 @@ docker_rm "git-mirror" lkp = %x(id -u lkp) group_id = %x(cat /etc/group | grep '^committer' | awk -F ':' '{print $3}') -cmd = %W[ +cmd = %w[ docker run --restart=always --name git-mirror +] + env + %W[ -u #{lkp.to_i}:#{group_id.to_i} -d -e CCI_SRC=#{DOCKER_CCI} diff --git a/container/web-backend/start b/container/web-backend/start index cebf949..250c524 100755 --- a/container/web-backend/start +++ b/container/web-backend/start @@ -13,6 +13,8 @@ load_cci_defaults [[ $SRV_HTTP_RESULT_HOST ]] || SRV_HTTP_RESULT_HOST=$(curl ip.sb) [[ $SRV_HTTP_RESULT_PORT ]] || SRV_HTTP_RESULT_PORT=20007 +load_service_authentication + cmd=( docker run --name web-backend @@ -26,6 +28,8 @@ cmd=( -e CCI_SRC=/c/compass-ci -e LKP_SRC=/c/lkp-tests -e UPSTREAM_REPOS_PATH=/c/upstream-repos + -e ES_USER=$ES_USER + -e ES_PASSWORD=$ES_PASSWORD -v $CCI_SRC:/c/compass-ci -v $LKP_SRC:/c/lkp-tests diff --git a/lib/constants.rb b/lib/constants.rb index 11927f9..c71c3de 100644 --- a/lib/constants.rb +++ b/lib/constants.rb @@ -5,9 +5,31 @@ require_relative '../container/defconfig.rb' config = cci_defaults + +names = Set.new %w[ + ES_USER + ES_PASSWORD +] + ES_HOST ||= config['ES_HOST'] || '172.17.0.1' ES_PORT ||= config['ES_PORT'] || 9200 +if ENV['ES_USER'] + ES_USER = ENV['ES_USER'] + ES_PASSWORD = ENV['ES_PASSWORD'] +else + service_authentication = relevant_service_authentication(names) + ES_USER = service_authentication['ES_USER'] + ES_PASSWORD = service_authentication['ES_PASSWORD'] +end + +ES_HOSTS = [{ + host: ES_HOST, + port: ES_PORT, + user: ES_USER, + password: ES_PASSWORD +}] + SEND_MAIL_HOST ||= config['SEND_MAIL_HOST'] || '172.17.0.1' SEND_MAIL_PORT ||= config['SEND_MAIL_PORT'] || 10001 diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index 6100d6b..71d6956 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -125,7 +125,7 @@ class MirrorMain @defaults = {} @git_queue = Queue.new @log = JSONLogger.new - @es_client = Elasticsearch::Client.new(url: "http://#{ES_HOST}:#{ES_PORT}") + @es_client = Elasticsearch::Client.new(hosts: ES_HOSTS) clone_upstream_repo load_fork_info connection_init diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 932d362..c4d4edd 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -34,7 +34,7 @@ NOT_SHOW_FIELDS = %w[result_root].freeze ALL_FIELDS = FIELDS + NOT_SHOW_FIELDS NOT_NEED_EXIST_FIELDS = %w[error_ids upstream_repo].freeze PREFIX_SEARCH_FIELDS = ['tbox_group'].freeze -ES_CLIENT = Elasticsearch::Client.new(url: "http://#{ES_HOST}:#{ES_PORT}") +ES_CLIENT = Elasticsearch::Client.new(hosts: ES_HOSTS) COMPARE_RECORDS_NUMBER = 100 def es_query(query) -- 2.23.0

1 0

[PATCH compass-ci 3/5] services: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

services: lifecycle monitoring extract-stats Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/extract-stats/start | 6 ++++++ container/lifecycle/start | 6 ++++++ container/monitoring/start | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/container/extract-stats/start b/container/extract-stats/start index 621099a..7a0377e 100755 --- a/container/extract-stats/start +++ b/container/extract-stats/start @@ -12,6 +12,8 @@ names = Set.new %w[ REDIS_PORT ES_HOST ES_PORT + ES_USER + ES_PASSWORD SEND_MAIL_HOST SEND_MAIL_PORT SRV_HTTP_RESULT_HOST @@ -20,6 +22,10 @@ names = Set.new %w[ ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) DEFAULT_LKP = '/c/lkp-tests' diff --git a/container/lifecycle/start b/container/lifecycle/start index 6e9b4fb..484584f 100755 --- a/container/lifecycle/start +++ b/container/lifecycle/start @@ -8,9 +8,15 @@ require_relative '../defconfig.rb' names = Set.new %w[ LIFECYCLE_PORT + ES_USER + ES_PASSWORD ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) LIFECYCLE_PORT = defaults['LIFECYCLE_PORT'] || '11312' diff --git a/container/monitoring/start b/container/monitoring/start index f4214ed..23d5cb4 100755 --- a/container/monitoring/start +++ b/container/monitoring/start @@ -10,9 +10,15 @@ names = Set.new %w[ MONITOR_PORT MQ_PORT MQ_HOST + ES_USER + ES_PASSWORD ] defaults = relevant_defaults(names) + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) DEFAULT_CCI = '/c/cci' -- 2.23.0

1 0

[PATCH compass-ci 2/5] service/scheduler: configuring authentication for connecting to the ES
by Wu Zhende 21 Apr '21

21 Apr '21

Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/scheduler/my-start | 6 ++++++ container/scheduler/start | 6 ++++++ src/scheduler/elasticsearch_client.cr | 8 +++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/container/scheduler/my-start b/container/scheduler/my-start index e5855af..5dd871e 100755 --- a/container/scheduler/my-start +++ b/container/scheduler/my-start @@ -13,6 +13,8 @@ names = Set.new %w[ REDIS_PORT ES_HOST ES_PORT + ES_USER + ES_PASSWORD OS_HTTP_HOST OS_HTTP_PORT INITRD_HTTP_HOST @@ -38,6 +40,10 @@ defaults['SCHED_PORT'] ||= '3000' defaults['SCHED_HOST'] ||= '172.17.0.1' defaults['MASTER_FLUENTD_PORT'] ||= '24224' defaults['SRV_HTTP_CCI_PORT'] ||= '11307' + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) CCI_REPOS = ENV['CCI_REPOS'] || '/c' diff --git a/container/scheduler/start b/container/scheduler/start index 598b263..029dda8 100755 --- a/container/scheduler/start +++ b/container/scheduler/start @@ -14,6 +14,8 @@ names = Set.new %w[ REDIS_PORT ES_HOST ES_PORT + ES_USER + ES_PASSWORD OS_HTTP_HOST OS_HTTP_PORT INITRD_HTTP_HOST @@ -44,6 +46,10 @@ SCHED_PORT = defaults['SCHED_PORT'] sched_nodes.each do |node| docker_rm "s001-alpine-#{node['port']}" end + +service_authentication = relevant_service_authentication(names) +defaults.merge!(service_authentication) + env = docker_env(defaults) CCI_REPOS = ENV['CCI_REPOS'] || '/c' diff --git a/src/scheduler/elasticsearch_client.cr b/src/scheduler/elasticsearch_client.cr index 0e83ff1..1ef79a6 100644 --- a/src/scheduler/elasticsearch_client.cr +++ b/src/scheduler/elasticsearch_client.cr @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +require "uri" require "yaml" require "json" require "any_merge" @@ -30,7 +31,12 @@ class Elasticsearch::Client HOST = (ENV.has_key?("ES_HOST") ? ENV["ES_HOST"] : JOB_ES_HOST) PORT = (ENV.has_key?("ES_PORT") ? ENV["ES_PORT"] : JOB_ES_PORT).to_i32 - def initialize(host = HOST, port = PORT) + def initialize(host = HOST, port = PORT, auth = true) + if auth + user = ENV["ES_USER"]? + password = ENV["ES_PASSWORD"]? + host = "http://#{user}:#{URI.encode_www_form(password)}@#{host}" if user && password + end @client = Elasticsearch::API::Client.new({:host => host, :port => port}) end -- 2.23.0

1 0