March 2021 - Compass-ci - mailweb.openeuler.org

[PATCH v2 compass-ci 1/2] container/kibana: upgrade and addition
by Wu Zhende 31 Mar '21

31 Mar '21

1. upgrade from 7.6.2 to 7.11.1 2. add a new kibana for data. one display log and one display data 3. delete logtrail plugin because it don't support in new kibana version Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana-logging/Dockerfile | 12 ++++++++ container/kibana-logging/build | 16 ++++++++++ container/kibana-logging/start | 31 +++++++++++++++++++ container/kibana-logging/start-depends | 1 + container/kibana/Dockerfile | 10 ------- container/kibana/build | 8 ++--- container/kibana/logtrail.json | 41 -------------------------- container/kibana/start | 14 ++++----- container/kibana/start-depends | 2 +- 9 files changed, 72 insertions(+), 63 deletions(-) create mode 100644 container/kibana-logging/Dockerfile create mode 100755 container/kibana-logging/build create mode 100755 container/kibana-logging/start create mode 100755 container/kibana-logging/start-depends delete mode 100644 container/kibana/logtrail.json diff --git a/container/kibana-logging/Dockerfile b/container/kibana-logging/Dockerfile new file mode 100644 index 0000000..970eb5a --- /dev/null +++ b/container/kibana-logging/Dockerfile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +ARG BASE_IMAGE + +FROM ${BASE_IMAGE} + +# docker image borrowed from hub.docker.com/r/gagara/kibana-oss-arm64 + +MAINTAINER Wu Zhende <wuzhende666(a)163.com> + +RUN sed -i 's/server.host: "0"/server.host: "0.0.0.0"/' config/kibana.yml diff --git a/container/kibana-logging/build b/container/kibana-logging/build new file mode 100755 index 0000000..9c75aeb --- /dev/null +++ b/container/kibana-logging/build @@ -0,0 +1,16 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative '../defconfig' + +docker_skip_rebuild "kibana:7.11.1" + +BASE_IMAGE_DICT = { + 'aarch64' => 'jamesgarside/kibana:7.11.1', + 'x86_64' => 'kibana:7.11.1' }.freeze + +BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] + +system "docker build -t kibana:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} ." diff --git a/container/kibana-logging/start b/container/kibana-logging/start new file mode 100755 index 0000000..f2169a6 --- /dev/null +++ b/container/kibana-logging/start @@ -0,0 +1,31 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig.rb' + +names = Set.new %w[ + LOGGING_ES_HOST + LOGGING_ES_PORT +] + +defaults = relevant_defaults(names) +LOGGING_ES_HOST = defaults['LOGGING_ES_HOST'] || '172.17.0.1' +LOGGING_ES_PORT = defaults['LOGGING_ES_PORT'] || '9202' + +docker_rm 'kibana-logging' + +cmd = %W[ + docker run + --restart=always + --name kibana-logging + -v /etc/localtime:/etc/localtime:ro + -d + -e ELASTICSEARCH_HOSTS=http://#{LOGGING_ES_HOST}:#{LOGGING_ES_PORT} + -p 20000:5601 + kibana:7.11.1 +] + +system(*cmd) diff --git a/container/kibana-logging/start-depends b/container/kibana-logging/start-depends new file mode 100755 index 0000000..66c1996 --- /dev/null +++ b/container/kibana-logging/start-depends @@ -0,0 +1 @@ +logging-es diff --git a/container/kibana/Dockerfile b/container/kibana/Dockerfile index 60b889d..970eb5a 100644 --- a/container/kibana/Dockerfile +++ b/container/kibana/Dockerfile @@ -10,13 +10,3 @@ FROM ${BASE_IMAGE} MAINTAINER Wu Zhende <wuzhende666(a)163.com> RUN sed -i 's/server.host: "0"/server.host: "0.0.0.0"/' config/kibana.yml - -USER root -RUN yum -y install wget \ - && wget https://github.com/sivasamyk/logtrail/releases/download/v0.1.31/logtrail-7.… -O /logtrail-7.6.2-0.1.31.zip - -USER 1090 -RUN ./bin/kibana-plugin install file:///logtrail-7.6.2-0.1.31.zip \ - && rm -rf /tmp/logtrail-7.6.2-0.1.31.zip - -COPY --chown=1090:1090 logtrail.json /usr/share/kibana/plugins/logtrail/logtrail.json diff --git a/container/kibana/build b/container/kibana/build index 9a1fb6d..9c75aeb 100755 --- a/container/kibana/build +++ b/container/kibana/build @@ -5,12 +5,12 @@ require_relative '../defconfig' -docker_skip_rebuild "kibana:7.6.2" +docker_skip_rebuild "kibana:7.11.1" BASE_IMAGE_DICT = { - 'aarch64' => 'gagara/kibana-oss-arm64:7.6.2', - 'x86_64' => 'kibana:7.6.2' }.freeze + 'aarch64' => 'jamesgarside/kibana:7.11.1', + 'x86_64' => 'kibana:7.11.1' }.freeze BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] -system "docker build -t kibana:7.6.2 --build-arg BASE_IMAGE=#{BASE_IMAGE} ." +system "docker build -t kibana:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} ." diff --git a/container/kibana/logtrail.json b/container/kibana/logtrail.json deleted file mode 100644 index c63f2ac..0000000 --- a/container/kibana/logtrail.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "version": 2, - "index_patterns": [{ - "es": { - "default_index": "*", - "allow_url_parameter": false, - "timezone": "CST" - }, - "tail_interval_in_seconds": 10, - "es_index_time_offset_in_seconds": 0, - "display_timezone": "CST", - "display_timestamp_format": "YYYY MMM DD HH:mm:ss", - "max_buckets": 500, - "nested_objects": false, - "default_time_range_in_days": 30, - "max_hosts": 100, - "max_events_to_keep_in_viewer": 10000, - "default_search": "", - "fields": { - "mapping": { - "timestamp": "time", - "display_timestamp" : "time", - "hostname": "container_name", - "program": "tags", - "message": "log" - }, - "message_format": "{{{log}}}", - "keyword_suffix": "keyword" - }, - "color_mapping": { - "field": "level", - "mapping": { - "ERROR": "#FF0000", - "WARN": "#FFEF96", - "DEBUG": "#B5E7A0", - "TRACE": "#CFE0E8", - "INFO": "#339999" - } - } - }] -} diff --git a/container/kibana/start b/container/kibana/start index 26a4944..63cb8f8 100755 --- a/container/kibana/start +++ b/container/kibana/start @@ -7,13 +7,13 @@ require 'set' require_relative '../defconfig.rb' names = Set.new %w[ - LOGGING_ES_HOST - LOGGING_ES_PORT + ES_HOST + ES_PORT ] defaults = relevant_defaults(names) -LOGGING_ES_HOST = defaults['LOGGING_ES_HOST'] || '172.17.0.1' -LOGGING_ES_PORT = defaults['LOGGING_ES_PORT'] || '9202' +ES_HOST = defaults['ES_HOST'] || '172.17.0.1' +ES_PORT = defaults['ES_PORT'] || '9200' docker_rm 'kibana' @@ -23,9 +23,9 @@ cmd = %W[ --name kibana -v /etc/localtime:/etc/localtime:ro -d - -e ELASTICSEARCH_HOSTS=http://#{LOGGING_ES_HOST}:#{LOGGING_ES_PORT} - -p 20000:5601 - kibana:7.6.2 + -e ELASTICSEARCH_HOSTS=http://#{ES_HOST}:#{ES_PORT} + -p 20017:5601 + kibana:7.11.1 ] system(*cmd) diff --git a/container/kibana/start-depends b/container/kibana/start-depends index 66c1996..8357fca 100755 --- a/container/kibana/start-depends +++ b/container/kibana/start-depends @@ -1 +1 @@ -logging-es +es -- 2.23.0

1 0

[PATCH compass-ci] lib/git_mirror.rb: fix no positive? method error
by Li Yuanchao 31 Mar '21

31 Mar '21

The result of query es changed, the result['hits']['total'] changes from a number to a hash, so the positive? method can not be used. Now use the count method which contained by es_client. Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- lib/git_mirror.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/git_mirror.rb b/lib/git_mirror.rb index c967c8b..7f36be3 100644 --- a/lib/git_mirror.rb +++ b/lib/git_mirror.rb @@ -414,9 +414,9 @@ class MirrorMain new_refs_count: {} } query = { query: { match: { _id: git_repo } } } - result = @es_client.search(index: 'repo', body: query)['hits'] - return fork_stat unless result['total'].positive? + return fork_stat unless @es_client.count(index: 'repo', body: query)['count'].positive? + result = @es_client.search(index: 'repo', body: query)['hits'] fork_stat.each_key do |key| fork_stat[key] = result['hits'][0]['_source'][key.to_s] || fork_stat[key] end -- 2.23.0

1 0

[PATCH compass-ci 2/2] container/es: upgrade version
by Wu Zhende 31 Mar '21

31 Mar '21

Upgrade to 7.11.1. Matches the version of kibana so that data can be displayed on kibana. Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/es/Dockerfile | 48 ++++++++++++-------------------------- container/es/build | 4 ++-- container/es/start | 2 +- container/logging-es/build | 8 +++---- container/logging-es/start | 2 +- 5 files changed, 23 insertions(+), 41 deletions(-) diff --git a/container/es/Dockerfile b/container/es/Dockerfile index ed02490..44d87ee 100644 --- a/container/es/Dockerfile +++ b/container/es/Dockerfile @@ -1,41 +1,24 @@ # SPDX-License-Identifier: MulanPSL-2.0+ # Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -FROM alpine:3.11 +FROM elasticsearch:7.11.1@sha256:d52cda1e73d1b1915ba2d76ca1e426620c7b5d6942d9d2f432259503974ba786 ARG MEMORY -RUN sed -ri.origin 's|^https?://dl-cdn.alpinelinux.org|http://mirrors.huaweicloud.com|g' /etc/apk/repositories - -RUN apk add --no-cache elasticsearch curl - -RUN rm -rf /etc/init.d/elasticsearch \ - && rm -rf /usr/share/java/elasticsearch/plugins \ - && mv /usr/share/java/elasticsearch /usr/share/es \ - && echo "===> Creating Elasticsearch Paths..." \ - && for path in \ - /srv/es \ - /usr/share/es/logs \ - /usr/share/es/config \ - /usr/share/es/config/scripts \ - /usr/share/es/tmp \ - /usr/share/es/plugins \ - ; do \ - mkdir -p "$path"; \ - done \ - && cp /etc/elasticsearch/*.* /usr/share/es/config \ - && chown -R 1090:1090 /usr/share/es \ - && chown -R 1090:1090 /srv/es; - -RUN sed -i 's:#path.data\: /path/to/data:path.data\: /srv/es:' /usr/share/es/config/elasticsearch.yml; -RUN sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/es/config/elasticsearch.yml; -RUN sed -i "s/-Xms256m/-Xms${MEMORY}m/g" /usr/share/es/config/jvm.options -RUN sed -i "s/-Xmx256m/-Xmx${MEMORY}m/g" /usr/share/es/config/jvm.options - -WORKDIR /usr/share/es - -ENV PATH /usr/share/es/bin:$PATH -ENV ES_TMPDIR /usr/share/es/tmp +RUN sed -i 's:#network.host\: _site_:network.host\: 0.0.0.0:' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a path.data: /srv/es' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a node.name: node-1' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a cluster.initial_master_nodes: ["node-1"]' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i "s/-Xms256m/-Xms${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options && \ + sed -i "s/-Xmx256m/-Xmx${MEMORY}m/g" /usr/share/elasticsearch/config/jvm.options + +RUN mkdir /usr/share/elasticsearch/tmp && \ + chown -R 1090:1090 /usr/share/elasticsearch + +WORKDIR /usr/share/elasticsearch + +ENV PATH /usr/share/elasticsearch/bin:$PATH +ENV ES_TMPDIR /usr/share/elasticsearch/tmp VOLUME ["/srv/es"] @@ -43,4 +26,3 @@ EXPOSE 9200 9300 USER 1090 CMD ["elasticsearch"] - diff --git a/container/es/build b/container/es/build index c7d7115..db5145f 100755 --- a/container/es/build +++ b/container/es/build @@ -5,8 +5,8 @@ require_relative '../defconfig.rb' -docker_skip_rebuild "es643b:alpine311" +docker_skip_rebuild "es:7.11.1" available_memory = get_available_memory -system "docker build -t es643b:alpine311 --build-arg MEMORY=#{available_memory} --network=host ." +system "docker build -t es:7.11.1 --build-arg MEMORY=#{available_memory} --network=host ." diff --git a/container/es/start b/container/es/start index 67d6531..3aa9525 100755 --- a/container/es/start +++ b/container/es/start @@ -15,7 +15,7 @@ cmd=( -v /srv/es:/srv/es -v /etc/localtime:/etc/localtime:ro --name es-server01 - es643b:alpine311 + es:7.11.1 ) "${cmd[@]}" diff --git a/container/logging-es/build b/container/logging-es/build index 3be841a..b50830e 100755 --- a/container/logging-es/build +++ b/container/logging-es/build @@ -5,15 +5,15 @@ require_relative '../defconfig.rb' -docker_skip_rebuild "logging-es:7.6.2" +docker_skip_rebuild "logging-es:7.11.1" BASE_IMAGE_DICT = { - 'aarch64' => 'gagara/elasticsearch-oss-arm64:7.6.2', - 'x86_64' => 'elasticsearch:7.6.2' + 'aarch64' => 'elasticsearch:7.11.1@sha256:d52cda1e73d1b1915ba2d76ca1e426620c7b5d6942d9d2f432259503974ba786', + 'x86_64' => 'elasticsearch:7.11.1' }.freeze BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] available_memory = get_available_memory -system "docker build -t logging-es:7.6.2 --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg MEMORY=#{available_memory} ." +system "docker build -t logging-es:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} --build-arg MEMORY=#{available_memory} ." diff --git a/container/logging-es/start b/container/logging-es/start index 9be7a5b..05ac46f 100755 --- a/container/logging-es/start +++ b/container/logging-es/start @@ -15,7 +15,7 @@ cmd=( -p 9302:9300 -v /srv/es/logging-es:/srv/es/logging-es --name logging-es - logging-es:7.6.2 + logging-es:7.11.1 ) "${cmd[@]}" -- 2.23.0

2 1

[PATCH lkp-tests] stat: optimize to obtain error for build DockerFile
by Liu Shaofei 31 Mar '21

31 Mar '21

Signed-off-by: Liu Shaofei <370072077(a)qq.com> --- stats/openeuler_docker.rb | 102 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100755 stats/openeuler_docker.rb diff --git a/stats/openeuler_docker.rb b/stats/openeuler_docker.rb new file mode 100755 index 000000000..c4a2cd691 --- /dev/null +++ b/stats/openeuler_docker.rb @@ -0,0 +1,102 @@ +#!/usr/bin/env ruby + +def pre_handel(result, file_path) + status = false + repo_set = Set[] + sys_set = Set[] + + File.readlines(file_path).each do |line| + case line.chomp! + # Error: Unable to find a match: docker-registry mock xx + when /Error: Unable to find a match: (.+)/ + $1.split.each do |repo| + repo_set << repo + end + + # RUN yum -y swap -- remove fakesystemd -- install systemd systemd-libs + # yum swap: error: unrecognized arguments: install systemd systemd-libs + when /yum swap: error: .*: install (.+)/ + $1.split.each do |sys| + sys_set << sys + end + + # curl: (22) The requested URL returned error: 404 Not Found + # error: skipping https://dl.fedoraproject.org/pub/epel/bash-latest-7.noarch.rpm - transfer failed + when /.*error: .* (https.*)/ + result['requested-URL-returned.error'] = [1] + result['requested-URL-returned.error.message'] = [line.to_s] + status = true + + # Error: Unknown repo: 'powertools' + when /Error: Unknown repo: (.+)/ + repo = $1.delete!("'") + result["unknown-repo.#{repo}"] = [1] + result["unknown-repo.#{repo}.message"] = [line.to_s] + status = true + + # Error: Module or Group 'convert' does not exist. + when /Error: Module or Group ('[^\s]+')/ + repo = $1.delete!("'") + result["error.not-exist-module-or-group.#{repo}"] = [1] + result["error.not-exist-module-or-group.#{repo}.message"] = [line.to_s] + status = true + # /bin/sh: passwd: command not found + when /\/bin\/sh: (.+): command not found/ + result["sh.command-not-found.#{$1}"] = [1] + result["sh.command-not-found.#{$1}.message"] = [line.to_s] + status = true + end + + repo_set.each do |repo| + result["yum.error.Unable-to-find-a-match.#{repo}"] = [1] + result["yum.error.Unable-to-find-a-match.#{repo}.message"] = ["Error: Unable to find a match #{repo}"] + status = true + end + + sys_set.each do |sys| + result["yum.swap.error.unrecognized-arguments-install.#{sys}"] = [1] + result["yum.swap.error.unrecognized-arguments.#{sys}.message"] = + ["yum swap: error: unrecognized arguments install #{sys}"] + status = true + end + end + status +end + +def handle_unknown_error(_result, file_path) + line_num = %x(cat #{file_path} | grep -n 'Step ' | tail -1 | awk -F: '{print $1}') + + index = 1 + message = '' + File.readlines(file_path).each do |line| + if index == Integer(line_num) + message += line + else + index += 1 + next + end + end + + message = $1 if message =~ %r(\u001b\[91m(.+)) + message +end + +def openeuler_docker(log_lines) + result = Hash.new { |hash, key| hash[key] = [] } + + log_lines.each do |line| + next unless line =~ %r(([^\s]+).(build|run)\.fail) + + key, value = line.split(':') + key.chomp! + result[key] << value.to_i + + file_path = "#{RESULT_ROOT}/#{$1}" # $1 named by docker-image name + next unless File.exist?(file_path) + next if pre_handel(result, file_path) + + result["#{key}.message"] << handle_unknown_error(result, file_path) + end + + result +end -- 2.23.0

1 0

[PATCH lkp-tests] stats/ansible_test: optimize the output of error_id
by Wang Chenglong 31 Mar '21

31 Mar '21

[Why] optimize the output of error_id standard format: @module_name:error_id before: error.[message].Could-not-find-or-access-RedHat-20-yml-Searched-in-root-ansible-roles-ansible-role-postgresql-vars-RedHat-20-yml-root-ansible- roles-ansible-role-postgresql-RedHat-20-yml-root-ansible-roles-ansible-role-postgresql-tasks-vars-RedHat-20-yml-root-ansible-roles-ansible-role- postgresql-tasks-RedHat-20-yml-root-ansible-vars-RedHat-20-yml-root-ansible-RedHat-20-yml-on-the-Ansible-Controller-If-you-are-using-a-module- and-expect-the-file-to-exist-on-the-remote-see-the-remote_src-option: 1 after: @inculde_vars:Could-not-find-or-access-RedHat-20-yml: 1 Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 99 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 83 insertions(+), 16 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 3d3b0b0cb..cadc11786 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,40 +1,107 @@ #!/usr/bin/env ruby require 'json' +require 'yaml' -def output_error(error_msg, error_key) - return if error_msg.nil? - +def output_error(error_msg) if error_msg.is_a? Array error_msg.each do |i| error_id = common_error_id i - puts "error.[#{error_key}].#{error_id}: 1" - puts "error.#{error_id}.message: #{@ansible_failed_info}" + puts "@#{@module_name['action']}:#{error_id}: 1" + puts "@#{@module_name['action']}:#{error_id}.message: #{@ansible_failed_info}" end elsif error_msg.is_a? String error_id = common_error_id error_msg - puts "error.[#{error_key}].#{error_id}: 1" - puts "error.#{error_id}.message: #{@ansible_failed_info}" + puts "@#{@module_name['action']}:#{error_id}: 1" + puts "@#{@module_name['action']}:#{error_id}.message: #{@ansible_failed_info}" end end def common_error_id(line) - line.gsub!(/[^\w]/, '-') + line.gsub!(/[^\w]/, '--') + line.gsub!(/[0-9]{3,}/, '#') line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one line.gsub!(/^-|-$/, '') line end +def parse_msg(ansible_failed_json) + case ansible_failed_json['msg'] + when /package (.*64)/ + output_error "cannot install the #{$1} for the job" + when /(The error appears to be in .*')/ + output_error $1 + when /(Failed to import the required Python library .*python3\.)/ + output_error $1 + # when /(An unhandled exception occurred while running the lookup plugin .*'\.)/ + when /original message: (.*)/ + output_error $1 + when /(Unable to start service (\w{1,}):)/ + output_error $1 + when /(Failed to download metadata for repo .*'):/ + output_error $1 + when /(Failed to download packages: (.*):)/ + output_error $2 + when /(Failed to find required .* in paths):/ + output_error $1 + when /(Failure downloading .*),/ + output_error $1 + when /(Could not find or access.*)'/ + output_error $1 + when /(Unsupported parameters) for/ + output_error $1 + when /aajdhch/ + output_error $1 + when /non-zero return code|Failed to install some of the specified packages/ + return + else + output_error ansible_failed_json['msg'] + end +end + +def parse_stderr(ansible_failed_json) + case ansible_failed_json['stderr'] + when /^$/ + return + else + ansible_failed_json['stderr_lines'].each do |i| + output_error i + end + end +end + +def parse_message(ansible_failed_json) + case ansible_failed_json['message'] + when /(Could not find or access.*)'/ + output_error $1 + else + output_error ansible_failed_json['message'] + end +end + +def parse_failures(ansible_failed_json) + ansible_failed_json['failures'].each do |i| + case i + when /(.*)/ + output_error $1 + end + end +end + while (line = STDIN.gets) - next unless line =~ /(FAILED!|failed:).*=>(.*)/ + case line + when /({'action'.*})/ + @module_name = YAML.load($1) + when /(FAILED!|failed:).*=>(.*)/ - @ansible_failed_info = $2 - next if @ansible_failed_info.empty? + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? - ansible_failed_json = JSON.parse @ansible_failed_info + ansible_failed_json = JSON.parse @ansible_failed_info - output_error ansible_failed_json['msg'],'msg' - output_error ansible_failed_json['message'],'message' - output_error ansible_failed_json['cmd'],'cmd' - output_error ansible_failed_json['failures'],'failures' + parse_msg ansible_failed_json unless ansible_failed_json['msg'].nil? + parse_stderr ansible_failed_json unless ansible_failed_json['stderr'].nil? + parse_message ansible_failed_json unless ansible_failed_json['message'].nil? + parse_failures ansible_failed_json unless ansible_failed_json['failures'].nil? + end end -- 2.23.0

1 0

[PATCH compass-ci 3/4] doc/job/os_mount.md: add the check pv_device step into document
by Xu Xijian 31 Mar '21

31 Mar '21

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- doc/job/os_mount.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/job/os_mount.md b/doc/job/os_mount.md index eab95ed..b9b8308 100644 --- a/doc/job/os_mount.md +++ b/doc/job/os_mount.md @@ -47,6 +47,7 @@ The brief flow is as follows: ${boot_lv} -- boot logical volume: - will boot from this lv. - boot_lv=/dev/mapper/os-${os}_${os_arch}_${os_version} + - check whether ${pv_device} is given, if so, create physical volume(pv for short) and volume group(vg for short) on ${pv_device}. - if ${src_lv} not exists: create it, and rsync the rootfs from cluster nfs server. - if ${boot_lv} exists: delete it. - create ${boot_lv} as the snapshot of ${src_lv}. @@ -57,6 +58,7 @@ The brief flow is as follows: ## persistent rootfs data When you need to persist the rootfs data of a job, and use it in the subsequent job(s), two fields in `kernel_custom_params` will help you: 'src_lv_suffix', 'boot_lv_suffix'. +If you want to run on a brand new machine, you should use 'pv_device' to assign a disk device to create pv and vg. The brief flow is as follows: @@ -65,6 +67,7 @@ The brief flow is as follows: 2. initrd stage: use_root_partition seems like /dev/mapper/os-${os}_${os_arch}_${os_version}_${src_lv_suffix} save_root_partition seems like /dev/mapper/os-${os}_${os_arch}_${os_version}_${boot_lv_suffix} + check and ensure pv and vg exist for lv. - firstly, we need two logical volume: ${src_lv} -- src logical volume: - if have ${use_root_partition}, src_lv=${use_root_partition} @@ -95,6 +98,8 @@ Demo usage: data of job-20210218.yaml. Then you need add the follow field in your job-20210219.yaml: kernel_custom_params: src_lv_suffix=zhangsan_local_for_iperf_20210218 + - if you want to create pv and vg on /dev/sda, you can use like this: + kernel_custom_params: pv_device=/dev/sda ... ``` Notes: @@ -148,6 +153,9 @@ Notes: lvdisplay ${src_lv} > /dev/null && return + # need create volume group, usually in first use of this machine. $pv_device e.g. /dev/sda + [ -n "$pv_device" ] && { do some check and create pv and vg } + # create logical volume lvcreate --size 10G --name $(basename ${src_lv}) os || exit -- 2.23.0

1 0

[PATCH compass-ci 1/4] src/lib/job.cr: add initrd param pv_device to scheduler
by Xu Xijian 31 Mar '21

31 Mar '21

Signed-off-by: Xu Xijian <hdxuxijian(a)163.com> --- src/lib/job.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/job.cr b/src/lib/job.cr index 5767f61..3ba991d 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -90,6 +90,7 @@ class Job linux_vmlinuz_path src_lv_suffix boot_lv_suffix + pv_device ) macro method_missing(call) -- 2.23.0

1 0

[PATCH lkp-tests] stats/ansible_test:
by Wang Chenglong 31 Mar '21

31 Mar '21

[Why] optimize the output of error_id standard format: @module_name:error_id before: error.[message].Could-not-find-or-access-RedHat-20-yml-Searched-in-root-ansible-roles-ansible-role-postgresql-vars-RedHat-20-yml-root-ansible- roles-ansible-role-postgresql-RedHat-20-yml-root-ansible-roles-ansible-role-postgresql-tasks-vars-RedHat-20-yml-root-ansible-roles-ansible-role- postgresql-tasks-RedHat-20-yml-root-ansible-vars-RedHat-20-yml-root-ansible-RedHat-20-yml-on-the-Ansible-Controller-If-you-are-using-a-module- and-expect-the-file-to-exist-on-the-remote-see-the-remote_src-option: 1 after: @inculde_vars:Could-not-find-or-access-RedHat-20-yml: 1 Signed-off-by: Wang Chenglong <18509160991(a)163.com> --- stats/ansible_test | 99 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 83 insertions(+), 16 deletions(-) diff --git a/stats/ansible_test b/stats/ansible_test index 3d3b0b0cb..cadc11786 100755 --- a/stats/ansible_test +++ b/stats/ansible_test @@ -1,40 +1,107 @@ #!/usr/bin/env ruby require 'json' +require 'yaml' -def output_error(error_msg, error_key) - return if error_msg.nil? - +def output_error(error_msg) if error_msg.is_a? Array error_msg.each do |i| error_id = common_error_id i - puts "error.[#{error_key}].#{error_id}: 1" - puts "error.#{error_id}.message: #{@ansible_failed_info}" + puts "@#{@module_name['action']}:#{error_id}: 1" + puts "@#{@module_name['action']}:#{error_id}.message: #{@ansible_failed_info}" end elsif error_msg.is_a? String error_id = common_error_id error_msg - puts "error.[#{error_key}].#{error_id}: 1" - puts "error.#{error_id}.message: #{@ansible_failed_info}" + puts "@#{@module_name['action']}:#{error_id}: 1" + puts "@#{@module_name['action']}:#{error_id}.message: #{@ansible_failed_info}" end end def common_error_id(line) - line.gsub!(/[^\w]/, '-') + line.gsub!(/[^\w]/, '--') + line.gsub!(/[0-9]{3,}/, '#') line.gsub!(/-+/, '-') # Replace multiple consecutive '-' with a single one line.gsub!(/^-|-$/, '') line end +def parse_msg(ansible_failed_json) + case ansible_failed_json['msg'] + when /package (.*64)/ + output_error "cannot install the #{$1} for the job" + when /(The error appears to be in .*')/ + output_error $1 + when /(Failed to import the required Python library .*python3\.)/ + output_error $1 + # when /(An unhandled exception occurred while running the lookup plugin .*'\.)/ + when /original message: (.*)/ + output_error $1 + when /(Unable to start service (\w{1,}):)/ + output_error $1 + when /(Failed to download metadata for repo .*'):/ + output_error $1 + when /(Failed to download packages: (.*):)/ + output_error $2 + when /(Failed to find required .* in paths):/ + output_error $1 + when /(Failure downloading .*),/ + output_error $1 + when /(Could not find or access.*)'/ + output_error $1 + when /(Unsupported parameters) for/ + output_error $1 + when /aajdhch/ + output_error $1 + when /non-zero return code|Failed to install some of the specified packages/ + return + else + output_error ansible_failed_json['msg'] + end +end + +def parse_stderr(ansible_failed_json) + case ansible_failed_json['stderr'] + when /^$/ + return + else + ansible_failed_json['stderr_lines'].each do |i| + output_error i + end + end +end + +def parse_message(ansible_failed_json) + case ansible_failed_json['message'] + when /(Could not find or access.*)'/ + output_error $1 + else + output_error ansible_failed_json['message'] + end +end + +def parse_failures(ansible_failed_json) + ansible_failed_json['failures'].each do |i| + case i + when /(.*)/ + output_error $1 + end + end +end + while (line = STDIN.gets) - next unless line =~ /(FAILED!|failed:).*=>(.*)/ + case line + when /({'action'.*})/ + @module_name = YAML.load($1) + when /(FAILED!|failed:).*=>(.*)/ - @ansible_failed_info = $2 - next if @ansible_failed_info.empty? + @ansible_failed_info = $2 + next if @ansible_failed_info.empty? - ansible_failed_json = JSON.parse @ansible_failed_info + ansible_failed_json = JSON.parse @ansible_failed_info - output_error ansible_failed_json['msg'],'msg' - output_error ansible_failed_json['message'],'message' - output_error ansible_failed_json['cmd'],'cmd' - output_error ansible_failed_json['failures'],'failures' + parse_msg ansible_failed_json unless ansible_failed_json['msg'].nil? + parse_stderr ansible_failed_json unless ansible_failed_json['stderr'].nil? + parse_message ansible_failed_json unless ansible_failed_json['message'].nil? + parse_failures ansible_failed_json unless ansible_failed_json['failures'].nil? + end end -- 2.23.0

1 1

[PATCH lab-z9] hosts: add rootfs_disk field
by Yu Chuan 31 Mar '21

31 Mar '21

Signed-off-by: Yu Chuan <13186087857(a)163.com> --- hosts/taishan200-2280-2s48p-256g--a101 | 3 ++- hosts/taishan200-2280-2s48p-256g--a103 | 3 ++- hosts/taishan200-2280-2s48p-256g--a104 | 3 ++- hosts/taishan200-2280-2s48p-256g--a106 | 3 ++- hosts/taishan200-2280-2s48p-256g--a107 | 3 ++- hosts/taishan200-2280-2s48p-256g--a20 | 3 ++- hosts/taishan200-2280-2s48p-256g--a63 | 3 ++- hosts/taishan200-2280-2s48p-256g--a64 | 3 ++- hosts/taishan200-2280-2s48p-256g--a65 | 3 ++- hosts/taishan200-2280-2s48p-512g--a102 | 3 ++- hosts/taishan200-2280-2s48p-512g--a105 | 3 ++- hosts/taishan200-2280-2s48p-512g--a95 | 3 ++- hosts/taishan200-2280-2s64p-256g--a110 | 3 ++- hosts/taishan200-2280-2s64p-256g--a112 | 3 ++- hosts/taishan200-2280-2s64p-256g--a113 | 3 ++- hosts/taishan200-2280-2s64p-256g--a114 | 3 ++- hosts/taishan200-2280-2s64p-256g--a116 | 3 ++- hosts/taishan200-2280-2s64p-256g--a117 | 3 ++- hosts/taishan200-2280-2s64p-256g--a118 | 3 ++- hosts/taishan200-2280-2s64p-256g--a119 | 3 ++- hosts/taishan200-2280-2s64p-256g--a121 | 4 ++-- hosts/taishan200-2280-2s64p-256g--a123 | 3 ++- hosts/taishan200-2280-2s64p-256g--a124 | 4 ++-- hosts/taishan200-2280-2s64p-256g--a5 | 4 ++-- hosts/taishan200-2280-2s64p-256g--a6 | 4 ++-- hosts/taishan200-2280-2s64p-256g--a7 | 4 ++-- 26 files changed, 52 insertions(+), 31 deletions(-) diff --git a/hosts/taishan200-2280-2s48p-256g--a101 b/hosts/taishan200-2280-2s48p-256g--a101 index f8a042942dd7..9d40f98e6434 100644 --- a/hosts/taishan200-2280-2s48p-256g--a101 +++ b/hosts/taishan200-2280-2s48p-256g--a101 @@ -2,11 +2,12 @@ arch: aarch64 nr_node: 4 nr_cpu: 96 memory: 256G -nr_hdd_partitions: 3 +nr_hdd_partitions: 2 nr_ssd_partitions: 1 hdd_partitions: - /dev/disk/by-id/scsi-350000399b89192e1 - /dev/disk/by-id/scsi-350000399b88bf66d +rootfs_disk: - /dev/disk/by-id/scsi-350000399b88bef85 ssd_partitions: - /dev/disk/by-id/ata-INTEL_SSDSC2KB480G8_BTYF94730FF4480BGN diff --git a/hosts/taishan200-2280-2s48p-256g--a103 b/hosts/taishan200-2280-2s48p-256g--a103 index 9fcc7187a8a3..ca26e20ec5e4 100644 --- a/hosts/taishan200-2280-2s48p-256g--a103 +++ b/hosts/taishan200-2280-2s48p-256g--a103 @@ -1,9 +1,10 @@ <<: taishan200-2280-2s48p-256g -nr_hdd_partitions: 4 +nr_hdd_partitions: 3 hdd_partitions: - /dev/disk/by-id/scsi-350000399b88bdc69 - /dev/disk/by-id/scsi-350000399b88bdd85 - /dev/disk/by-id/scsi-350000399b88bdc4d +rootfs_disk: - /dev/disk/by-id/scsi-350000399b88bdc7d mac_addr: - 44:67:47:c9:d8:b8 diff --git a/hosts/taishan200-2280-2s48p-256g--a104 b/hosts/taishan200-2280-2s48p-256g--a104 index 41cdbb052709..d0169640f279 100644 --- a/hosts/taishan200-2280-2s48p-256g--a104 +++ b/hosts/taishan200-2280-2s48p-256g--a104 @@ -1,9 +1,10 @@ <<: taishan200-2280-2s48p-256g -nr_hdd_partitions: 4 +nr_hdd_partitions: 3 hdd_partitions: - /dev/disk/by-id/scsi-350000399b8892431 - /dev/disk/by-id/scsi-350000399b8892f81 - /dev/disk/by-id/scsi-350000399b8891c49 +rootfs_disk: - /dev/disk/by-id/scsi-350000399b88930d1 mac_addr: - 44:67:47:d7:6d:f0 diff --git a/hosts/taishan200-2280-2s48p-256g--a106 b/hosts/taishan200-2280-2s48p-256g--a106 index 95b2f0500c4c..de0f418250fd 100644 --- a/hosts/taishan200-2280-2s48p-256g--a106 +++ b/hosts/taishan200-2280-2s48p-256g--a106 @@ -1,9 +1,10 @@ <<: taishan200-2280-2s48p-256g -nr_hdd_partitions: 4 +nr_hdd_partitions: 3 hdd_partitions: - /dev/disk/by-id/scsi-350000399b883bf99 - /dev/disk/by-id/scsi-350000399b883ba89 - /dev/disk/by-id/scsi-350000399b883b94d +rootfs_disk: - /dev/disk/by-id/scsi-350000399b883b981 mac_addr: - 44:67:47:86:1c:30 diff --git a/hosts/taishan200-2280-2s48p-256g--a107 b/hosts/taishan200-2280-2s48p-256g--a107 index 31b8dc6cdf90..f8506b22e38e 100644 --- a/hosts/taishan200-2280-2s48p-256g--a107 +++ b/hosts/taishan200-2280-2s48p-256g--a107 @@ -1,9 +1,10 @@ <<: taishan200-2280-2s48p-256g -nr_hdd_partitions: 4 +nr_hdd_partitions: 3 hdd_partitions: - /dev/disk/by-id/scsi-350000399b8917c39 - /dev/disk/by-id/scsi-350000399b8918005 - /dev/disk/by-id/scsi-350000399b8900a4d +rootfs_disk: - /dev/disk/by-id/scsi-350000399b8900799 mac_addr: - 84:46:fe:73:a2:71 diff --git a/hosts/taishan200-2280-2s48p-256g--a20 b/hosts/taishan200-2280-2s48p-256g--a20 index 21bf8e1fcd9d..6d50eead2f31 100644 --- a/hosts/taishan200-2280-2s48p-256g--a20 +++ b/hosts/taishan200-2280-2s48p-256g--a20 @@ -1,9 +1,10 @@ memory: 256G -nr_hdd_partitions: 4 +nr_hdd_partitions: 3 hdd_partitions: - /dev/disk/by-id/scsi-350000399b888f2dd - /dev/disk/by-id/scsi-350000399b888f705 - /dev/disk/by-id/scsi-350000399b88906f5 +rootfs_disk: - /dev/disk/by-id/scsi-350000399b8890141 mac_addr: - 84:46:fe:3d:a7:d5 diff --git a/hosts/taishan200-2280-2s48p-256g--a63 b/hosts/taishan200-2280-2s48p-256g--a63 index 0b3c938372f3..fea8ccb98892 100644 --- a/hosts/taishan200-2280-2s48p-256g--a63 +++ b/hosts/taishan200-2280-2s48p-256g--a63 @@ -1,9 +1,10 @@ memory: 256G -nr_hdd_partitions: 4 +nr_hdd_partitions: 3 hdd_partitions: - /dev/disk/by-id/scsi-350000399886b0431 - /dev/disk/by-id/scsi-350000399886adc71 - /dev/disk/by-id/scsi-350000399886b66f5 +rootfs_disk: - /dev/disk/by-id/scsi-350000399886b6b75 mac_addr: - 44:67:47:c9:e5:44 diff --git a/hosts/taishan200-2280-2s48p-256g--a64 b/hosts/taishan200-2280-2s48p-256g--a64 index 7cd0c4813ac6..b1fc7d2a25d5 100644 --- a/hosts/taishan200-2280-2s48p-256g--a64 +++ b/hosts/taishan200-2280-2s48p-256g--a64 @@ -1,9 +1,10 @@ memory: 256G -nr_hdd_partitions: 4 +nr_hdd_partitions: 3 hdd_partitions: - /dev/disk/by-id/scsi-350000399b88b2fa1 - /dev/disk/by-id/scsi-350000399b88b4339 - /dev/disk/by-id/scsi-350000399b88b4329 +rootfs_disk: - /dev/disk/by-id/scsi-350000399b88aefc1 mac_addr: - 44:67:47:c9:ea:b2 diff --git a/hosts/taishan200-2280-2s48p-256g--a65 b/hosts/taishan200-2280-2s48p-256g--a65 index 88441cc536d9..2bafc158a6a0 100644 --- a/hosts/taishan200-2280-2s48p-256g--a65 +++ b/hosts/taishan200-2280-2s48p-256g--a65 @@ -1,9 +1,10 @@ memory: 256G -nr_hdd_partitions: 4 +nr_hdd_partitions: 3 hdd_partitions: - /dev/disk/by-id/scsi-350000399b88b4509 - /dev/disk/by-id/scsi-350000399b88b30dd - /dev/disk/by-id/scsi-350000399b88b4621 +rootfs_disk: - /dev/disk/by-id/scsi-350000399b88b4425 mac_addr: - 44:67:47:c9:e9:90 diff --git a/hosts/taishan200-2280-2s48p-512g--a102 b/hosts/taishan200-2280-2s48p-512g--a102 index e1d3f10b8adb..ca88543ffc9b 100644 --- a/hosts/taishan200-2280-2s48p-512g--a102 +++ b/hosts/taishan200-2280-2s48p-512g--a102 @@ -2,10 +2,11 @@ arch: aarch64 nr_node: 4 nr_cpu: 96 memory: 512G -nr_hdd_partitions: 2 +nr_hdd_partitions: 1 nr_ssd_partitions: 3 hdd_partitions: - /dev/disk/by-id/scsi-350000399b88bfbc9 +rootfs_disk: - /dev/disk/by-id/scsi-350000399b8919395 ssd_partitions: - /dev/disk/by-id/ata-SAMSUNG_MZ7LH1T9HMLT-00005_S455NY0MB55506 diff --git a/hosts/taishan200-2280-2s48p-512g--a105 b/hosts/taishan200-2280-2s48p-512g--a105 index 66a0aca1f7b0..5a295e906de7 100644 --- a/hosts/taishan200-2280-2s48p-512g--a105 +++ b/hosts/taishan200-2280-2s48p-512g--a105 @@ -1,9 +1,10 @@ <<: taishan200-2280-2s48p-512g -nr_hdd_partitions: 4 +nr_hdd_partitions: 3 hdd_partitions: - /dev/disk/by-id/scsi-350000399b891945d - /dev/disk/by-id/scsi-350000399b8919135 - /dev/disk/by-id/scsi-350000399b8919699 +rootfs_disk: - /dev/disk/by-id/scsi-350000399b89190d5 mac_addr: - 44:67:47:d7:76:c4 diff --git a/hosts/taishan200-2280-2s48p-512g--a95 b/hosts/taishan200-2280-2s48p-512g--a95 index 213d10f5ee77..ab649a3694c9 100644 --- a/hosts/taishan200-2280-2s48p-512g--a95 +++ b/hosts/taishan200-2280-2s48p-512g--a95 @@ -2,11 +2,12 @@ arch: aarch64 nr_node: 4 nr_cpu: 96 memory: 512G -nr_hdd_partitions: 4 +nr_hdd_partitions: 3 hdd_partitions: - /dev/disk/by-id/scsi-35000039988792e31 - /dev/disk/by-id/scsi-35000039988793705 - /dev/disk/by-id/scsi-35000039988792aa5 +rootfs_disk: - /dev/disk/by-id/scsi-350000399887932fd mac_addr: - cc:05:77:fd:fb:61 diff --git a/hosts/taishan200-2280-2s64p-256g--a110 b/hosts/taishan200-2280-2s64p-256g--a110 index 50625bc632ed..415baa86f1ff 100644 --- a/hosts/taishan200-2280-2s64p-256g--a110 +++ b/hosts/taishan200-2280-2s64p-256g--a110 @@ -1,7 +1,8 @@ memory: 128G -nr_hdd_partitions: 2 +nr_hdd_partitions: 1 hdd_partitions: - /dev/disk/by-id/scsi-350000399886b6079 +rootfs_disk: - /dev/disk/by-id/scsi-350000399886b69bd mac_addr: - 44:67:47:e9:79:3d diff --git a/hosts/taishan200-2280-2s64p-256g--a112 b/hosts/taishan200-2280-2s64p-256g--a112 index 30c0f805c2c4..e53c76b77fe4 100644 --- a/hosts/taishan200-2280-2s64p-256g--a112 +++ b/hosts/taishan200-2280-2s64p-256g--a112 @@ -1,7 +1,8 @@ memory: 128G -nr_hdd_partitions: 2 +nr_hdd_partitions: 1 hdd_partitions: - /dev/disk/by-id/scsi-350000399888173c1 +rootfs_disk: - /dev/disk/by-id/scsi-350000399888169c9 mac_addr: - 44:a1:91:9d:f3:3a diff --git a/hosts/taishan200-2280-2s64p-256g--a113 b/hosts/taishan200-2280-2s64p-256g--a113 index 826b18d6024d..015daaf0bcab 100644 --- a/hosts/taishan200-2280-2s64p-256g--a113 +++ b/hosts/taishan200-2280-2s64p-256g--a113 @@ -1,7 +1,8 @@ memory: 128G -nr_hdd_partitions: 2 +nr_hdd_partitions: 1 hdd_partitions: - /dev/disk/by-id/scsi-35000039988700409 +rootfs_disk: - /dev/disk/by-id/scsi-350000399886bfb25 mac_addr: - 44:67:47:85:ed:b7 diff --git a/hosts/taishan200-2280-2s64p-256g--a114 b/hosts/taishan200-2280-2s64p-256g--a114 index f3a4d68c87c2..448b2e95b640 100644 --- a/hosts/taishan200-2280-2s64p-256g--a114 +++ b/hosts/taishan200-2280-2s64p-256g--a114 @@ -1,7 +1,8 @@ memory: 128G -nr_hdd_partitions: 2 +nr_hdd_partitions: 1 hdd_partitions: - /dev/disk/by-id/scsi-350000399785bf105 +rootfs_disk: - /dev/disk/by-id/scsi-350000399785bf189 mac_addr: - 44:a1:91:9d:d5:44 diff --git a/hosts/taishan200-2280-2s64p-256g--a116 b/hosts/taishan200-2280-2s64p-256g--a116 index 5ca6cd4c7b2f..4b8dc936fe6d 100644 --- a/hosts/taishan200-2280-2s64p-256g--a116 +++ b/hosts/taishan200-2280-2s64p-256g--a116 @@ -1,7 +1,8 @@ memory: 128G -nr_hdd_partitions: 2 +nr_hdd_partitions: 1 hdd_partitions: - /dev/disk/by-id/scsi-350000399886b6fad +rootfs_disk: - /dev/disk/by-id/scsi-350000399886b6ba9 mac_addr: - 44:67:47:85:f0:af diff --git a/hosts/taishan200-2280-2s64p-256g--a117 b/hosts/taishan200-2280-2s64p-256g--a117 index 0f104951e8fd..696943cef060 100644 --- a/hosts/taishan200-2280-2s64p-256g--a117 +++ b/hosts/taishan200-2280-2s64p-256g--a117 @@ -1,7 +1,8 @@ memory: 128G -nr_hdd_partitions: 2 +nr_hdd_partitions: 1 hdd_partitions: - /dev/disk/by-id/scsi-350000399886b57d1 +rootfs_disk: - /dev/disk/by-id/scsi-350000399886b720d mac_addr: - 44:67:47:85:f4:0b diff --git a/hosts/taishan200-2280-2s64p-256g--a118 b/hosts/taishan200-2280-2s64p-256g--a118 index 9a93e034527e..4bcfeea10d66 100644 --- a/hosts/taishan200-2280-2s64p-256g--a118 +++ b/hosts/taishan200-2280-2s64p-256g--a118 @@ -1,7 +1,8 @@ memory: 128G -nr_hdd_partitions: 2 +nr_hdd_partitions: 1 hdd_partitions: - /dev/disk/by-id/scsi-35000039988707e91 +rootfs_disk: - /dev/disk/by-id/scsi-350000399887080e5 mac_addr: - 44:67:47:e9:78:cf diff --git a/hosts/taishan200-2280-2s64p-256g--a119 b/hosts/taishan200-2280-2s64p-256g--a119 index 57da86389ada..92d341a5a487 100644 --- a/hosts/taishan200-2280-2s64p-256g--a119 +++ b/hosts/taishan200-2280-2s64p-256g--a119 @@ -1,7 +1,8 @@ memory: 128G -nr_hdd_partitions: 2 +nr_hdd_partitions: 1 hdd_partitions: - /dev/disk/by-id/scsi-350000399886b6be9 +rootfs_disk: - /dev/disk/by-id/scsi-350000399886b5dbd mac_addr: - 44:67:47:e9:79:65 diff --git a/hosts/taishan200-2280-2s64p-256g--a121 b/hosts/taishan200-2280-2s64p-256g--a121 index 9c87315d14c0..94d466828a85 100644 --- a/hosts/taishan200-2280-2s64p-256g--a121 +++ b/hosts/taishan200-2280-2s64p-256g--a121 @@ -1,6 +1,6 @@ memory: 256G -nr_ssd_partitions: 1 -ssd_partitions: +nr_ssd_partitions: 0 +rootfs_disk: - /dev/disk/by-id/ata-SAMSUNG_MZ7LH480HAHQ-00005_S45PNA2MB43943 mac_addr: - 44:67:47:e9:76:95 diff --git a/hosts/taishan200-2280-2s64p-256g--a123 b/hosts/taishan200-2280-2s64p-256g--a123 index f7650926e30e..cfc9c58df7b6 100644 --- a/hosts/taishan200-2280-2s64p-256g--a123 +++ b/hosts/taishan200-2280-2s64p-256g--a123 @@ -1,7 +1,8 @@ memory: 128G -nr_hdd_partitions: 2 +nr_hdd_partitions: 1 hdd_partitions: - /dev/disk/by-id/scsi-3500003998881eb19 +rootfs_disk: - /dev/disk/by-id/scsi-3500003998881e9a1 mac_addr: - 44:a1:91:9d:f5:4a diff --git a/hosts/taishan200-2280-2s64p-256g--a124 b/hosts/taishan200-2280-2s64p-256g--a124 index 7e7bb4222377..975dab549b2b 100644 --- a/hosts/taishan200-2280-2s64p-256g--a124 +++ b/hosts/taishan200-2280-2s64p-256g--a124 @@ -1,6 +1,6 @@ memory: 256G -nr_ssd_partitions: 1 -ssd_partitions: +nr_ssd_partitions: 0 +rootfs_disk: - /dev/disk/by-id/ata-SAMSUNG_MZ7LH480HAHQ-00005_S45PNA2MB44344 mac_addr: - 44:67:47:e9:76:9f diff --git a/hosts/taishan200-2280-2s64p-256g--a5 b/hosts/taishan200-2280-2s64p-256g--a5 index e6c11b3def19..37881cce5b20 100644 --- a/hosts/taishan200-2280-2s64p-256g--a5 +++ b/hosts/taishan200-2280-2s64p-256g--a5 @@ -1,7 +1,7 @@ memory: 256G -nr_hdd_partitions: 1 +nr_hdd_partitions: 0 nr_ssd_partitions: 1 -hdd_partitions: +rootfs_disk: - /dev/disk/by-id/scsi-350000399981306c5 ssd_partitions: - /dev/disk/by-id/nvme-HWE36P43012M000N_032BEJ10GC000328 diff --git a/hosts/taishan200-2280-2s64p-256g--a6 b/hosts/taishan200-2280-2s64p-256g--a6 index 09441939752c..dc11bb047fc9 100644 --- a/hosts/taishan200-2280-2s64p-256g--a6 +++ b/hosts/taishan200-2280-2s64p-256g--a6 @@ -1,7 +1,7 @@ memory: 256G -nr_hdd_partitions: 1 +nr_hdd_partitions: 0 nr_ssd_partitions: 1 -hdd_partitions: +rootfs_disk: - /dev/disk/by-id/scsi-350000399981344d9 ssd_partitions: - /dev/disk/by-id/nvme-HWE36P43012M000N_032BEJ10GC000301 diff --git a/hosts/taishan200-2280-2s64p-256g--a7 b/hosts/taishan200-2280-2s64p-256g--a7 index 07701d3cf3f9..4329ebfaff1e 100644 --- a/hosts/taishan200-2280-2s64p-256g--a7 +++ b/hosts/taishan200-2280-2s64p-256g--a7 @@ -1,7 +1,7 @@ memory: 256G -nr_hdd_partitions: 1 +nr_hdd_partitions: 0 nr_ssd_partitions: 1 -hdd_partitions: +rootfs_disk: - /dev/disk/by-id/scsi-35000039998134611 ssd_partitions: - /dev/disk/by-id/ata-SAMSUNG_MZ7LH480HAHQ-00005_S45PNA2MB43104 -- 2.23.0

1 0

[PATCH compass-ci 1/2] container/kibana: upgrade and addition
by Wu Zhende 31 Mar '21

31 Mar '21

1. upgrade from 7.6.2 to 7.11.1 2. add a new kibana for data. one display log and one display data 3. delete logtrail plugin because it don't support in new kibana version Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- container/kibana-logging/Dockerfile | 12 ++++++++ container/kibana-logging/build | 16 ++++++++++ container/kibana-logging/start | 31 +++++++++++++++++++ container/kibana-logging/start-depends | 1 + container/kibana/Dockerfile | 10 ------- container/kibana/build | 8 ++--- container/kibana/logtrail.json | 41 -------------------------- container/kibana/start | 14 ++++----- container/kibana/start-depends | 2 +- 9 files changed, 72 insertions(+), 63 deletions(-) create mode 100644 container/kibana-logging/Dockerfile create mode 100755 container/kibana-logging/build create mode 100755 container/kibana-logging/start create mode 100755 container/kibana-logging/start-depends delete mode 100644 container/kibana/logtrail.json diff --git a/container/kibana-logging/Dockerfile b/container/kibana-logging/Dockerfile new file mode 100644 index 0000000..970eb5a --- /dev/null +++ b/container/kibana-logging/Dockerfile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. + +ARG BASE_IMAGE + +FROM ${BASE_IMAGE} + +# docker image borrowed from hub.docker.com/r/gagara/kibana-oss-arm64 + +MAINTAINER Wu Zhende <wuzhende666(a)163.com> + +RUN sed -i 's/server.host: "0"/server.host: "0.0.0.0"/' config/kibana.yml diff --git a/container/kibana-logging/build b/container/kibana-logging/build new file mode 100755 index 0000000..9c75aeb --- /dev/null +++ b/container/kibana-logging/build @@ -0,0 +1,16 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require_relative '../defconfig' + +docker_skip_rebuild "kibana:7.11.1" + +BASE_IMAGE_DICT = { + 'aarch64' => 'jamesgarside/kibana:7.11.1', + 'x86_64' => 'kibana:7.11.1' }.freeze + +BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] + +system "docker build -t kibana:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} ." diff --git a/container/kibana-logging/start b/container/kibana-logging/start new file mode 100755 index 0000000..f2169a6 --- /dev/null +++ b/container/kibana-logging/start @@ -0,0 +1,31 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'set' +require_relative '../defconfig.rb' + +names = Set.new %w[ + LOGGING_ES_HOST + LOGGING_ES_PORT +] + +defaults = relevant_defaults(names) +LOGGING_ES_HOST = defaults['LOGGING_ES_HOST'] || '172.17.0.1' +LOGGING_ES_PORT = defaults['LOGGING_ES_PORT'] || '9202' + +docker_rm 'kibana-logging' + +cmd = %W[ + docker run + --restart=always + --name kibana-logging + -v /etc/localtime:/etc/localtime:ro + -d + -e ELASTICSEARCH_HOSTS=http://#{LOGGING_ES_HOST}:#{LOGGING_ES_PORT} + -p 20000:5601 + kibana:7.11.1 +] + +system(*cmd) diff --git a/container/kibana-logging/start-depends b/container/kibana-logging/start-depends new file mode 100755 index 0000000..66c1996 --- /dev/null +++ b/container/kibana-logging/start-depends @@ -0,0 +1 @@ +logging-es diff --git a/container/kibana/Dockerfile b/container/kibana/Dockerfile index 60b889d..970eb5a 100644 --- a/container/kibana/Dockerfile +++ b/container/kibana/Dockerfile @@ -10,13 +10,3 @@ FROM ${BASE_IMAGE} MAINTAINER Wu Zhende <wuzhende666(a)163.com> RUN sed -i 's/server.host: "0"/server.host: "0.0.0.0"/' config/kibana.yml - -USER root -RUN yum -y install wget \ - && wget https://github.com/sivasamyk/logtrail/releases/download/v0.1.31/logtrail-7.… -O /logtrail-7.6.2-0.1.31.zip - -USER 1090 -RUN ./bin/kibana-plugin install file:///logtrail-7.6.2-0.1.31.zip \ - && rm -rf /tmp/logtrail-7.6.2-0.1.31.zip - -COPY --chown=1090:1090 logtrail.json /usr/share/kibana/plugins/logtrail/logtrail.json diff --git a/container/kibana/build b/container/kibana/build index 9a1fb6d..9c75aeb 100755 --- a/container/kibana/build +++ b/container/kibana/build @@ -5,12 +5,12 @@ require_relative '../defconfig' -docker_skip_rebuild "kibana:7.6.2" +docker_skip_rebuild "kibana:7.11.1" BASE_IMAGE_DICT = { - 'aarch64' => 'gagara/kibana-oss-arm64:7.6.2', - 'x86_64' => 'kibana:7.6.2' }.freeze + 'aarch64' => 'jamesgarside/kibana:7.11.1', + 'x86_64' => 'kibana:7.11.1' }.freeze BASE_IMAGE = BASE_IMAGE_DICT[%x(arch).chomp] -system "docker build -t kibana:7.6.2 --build-arg BASE_IMAGE=#{BASE_IMAGE} ." +system "docker build -t kibana:7.11.1 --build-arg BASE_IMAGE=#{BASE_IMAGE} ." diff --git a/container/kibana/logtrail.json b/container/kibana/logtrail.json deleted file mode 100644 index c63f2ac..0000000 --- a/container/kibana/logtrail.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "version": 2, - "index_patterns": [{ - "es": { - "default_index": "*", - "allow_url_parameter": false, - "timezone": "CST" - }, - "tail_interval_in_seconds": 10, - "es_index_time_offset_in_seconds": 0, - "display_timezone": "CST", - "display_timestamp_format": "YYYY MMM DD HH:mm:ss", - "max_buckets": 500, - "nested_objects": false, - "default_time_range_in_days": 30, - "max_hosts": 100, - "max_events_to_keep_in_viewer": 10000, - "default_search": "", - "fields": { - "mapping": { - "timestamp": "time", - "display_timestamp" : "time", - "hostname": "container_name", - "program": "tags", - "message": "log" - }, - "message_format": "{{{log}}}", - "keyword_suffix": "keyword" - }, - "color_mapping": { - "field": "level", - "mapping": { - "ERROR": "#FF0000", - "WARN": "#FFEF96", - "DEBUG": "#B5E7A0", - "TRACE": "#CFE0E8", - "INFO": "#339999" - } - } - }] -} diff --git a/container/kibana/start b/container/kibana/start index 26a4944..ac74ecc 100755 --- a/container/kibana/start +++ b/container/kibana/start @@ -7,13 +7,13 @@ require 'set' require_relative '../defconfig.rb' names = Set.new %w[ - LOGGING_ES_HOST - LOGGING_ES_PORT + ES_HOST + ES_PORT ] defaults = relevant_defaults(names) -LOGGING_ES_HOST = defaults['LOGGING_ES_HOST'] || '172.17.0.1' -LOGGING_ES_PORT = defaults['LOGGING_ES_PORT'] || '9202' +ES_HOST = defaults['ES_HOST'] || '172.17.0.1' +ES_PORT = defaults['ES_PORT'] || '9200' docker_rm 'kibana' @@ -23,9 +23,9 @@ cmd = %W[ --name kibana -v /etc/localtime:/etc/localtime:ro -d - -e ELASTICSEARCH_HOSTS=http://#{LOGGING_ES_HOST}:#{LOGGING_ES_PORT} - -p 20000:5601 - kibana:7.6.2 + -e ELASTICSEARCH_HOSTS=http://#{ES_HOST}:#{ES_PORT} + -p 20014:5601 + kibana:7.11.1 ] system(*cmd) diff --git a/container/kibana/start-depends b/container/kibana/start-depends index 66c1996..8357fca 100755 --- a/container/kibana/start-depends +++ b/container/kibana/start-depends @@ -1 +1 @@ -logging-es +es -- 2.23.0

1 0

2024

2023

2022

2021

2020

Compass-ci March 2021