[Why] Now we have three types of fluentd:sub-fluentd/fluentd/master-fluentd. sub-fluentd and fluentd have similar functions, can be merged. So change to two types of fluentd:sub-fluentd and fluentd. sub-fluentd only collections logs then forward to fluentd. All data processing is performed on fluentd.
Signed-off-by: Wu Zhende wuzhende666@163.com --- container/fluentd/docker-fluentd.conf | 86 +++++++------------- container/fluentd/start | 2 - container/master-fluentd/Dockerfile | 6 -- container/master-fluentd/build | 6 -- container/master-fluentd/build-depends | 1 - container/master-fluentd/docker-fluentd.conf | 48 ----------- container/master-fluentd/start | 26 ------ container/sub-fluentd/docker-fluentd.conf | 54 ++++++++++++ container/sub-fluentd/start | 12 +-- 9 files changed, 89 insertions(+), 152 deletions(-) delete mode 100644 container/master-fluentd/Dockerfile delete mode 100755 container/master-fluentd/build delete mode 100755 container/master-fluentd/build-depends delete mode 100644 container/master-fluentd/docker-fluentd.conf delete mode 100755 container/master-fluentd/start
diff --git a/container/fluentd/docker-fluentd.conf b/container/fluentd/docker-fluentd.conf index 23c7585..4660988 100644 --- a/container/fluentd/docker-fluentd.conf +++ b/container/fluentd/docker-fluentd.conf @@ -2,17 +2,9 @@ workers 10 </system>
-<worker 0> <source> - @type tail - path /srv/cci/serial/logs/* - pos_file /srv/cci/serial/fluentd-pos/serial.log.pos - tag serial.* - path_key serial_path - refresh_interval 1s - <parse> - @type none - </parse> + @type forward + bind 0.0.0.0 </source>
<filter **> @@ -24,34 +16,16 @@ </filter>
<match serial.**> - @type copy - <store> - @type rabbitmq - host 172.17.0.1 - exchange logging-test - exchange_type fanout - exchange_durable false - heartbeat 10 - <format> - @type json - </format> - </store> + @type rabbitmq + host 172.17.0.1 + exchange logging-test + exchange_type fanout + exchange_durable false + heartbeat 10 + <format> + @type json + </format> </match> -</worker> - -<worker 1-9> -<source> - @type forward - bind 0.0.0.0 -</source> - -<filter **> - @type record_transformer - enable_ruby - <record> - time ${time.strftime('%Y-%m-%d %H:%M:%S%z')} - </record> -</filter>
<filter **> @type parser @@ -65,31 +39,29 @@ @type copy
<store> - @type stdout - <format> - @type stdout - output_type single_value - message_key log - add_newline true - </format> + @type elasticsearch + host 172.17.0.1 + port 9202 + suppress_type_name true + flush_interval 1s + index_name ${tag} + ssl_verify false + log_es_400_reason true + with_transporter_log true </store>
<store> - @type forward - flush_interval 0 - send_timeout 60 - heartbeat_type udp - heartbeat_interval 1 - recover_wait 10 - hard_timeout 60 - <server> - master-fluentd - host 172.17.0.1 - port 24225 - </server> + @type rabbitmq + host 172.17.0.1 + exchange logging-test + exchange_type fanout + exchange_durable false + heartbeat 10 + <format> + @type json + </format> </store> </match> -</worker>
<label @FLUENT_LOG> <match fluent.*> diff --git a/container/fluentd/start b/container/fluentd/start index 8ad7c1e..b7f9c73 100755 --- a/container/fluentd/start +++ b/container/fluentd/start @@ -12,8 +12,6 @@ cmd = %w[ --restart=always --name fluentd -v /etc/localtime:/etc/localtime:ro - -v /srv/cci/serial/logs:/srv/cci/serial/logs:ro - -v /srv/cci/serial/fluentd-pos:/srv/cci/serial/fluentd-pos -d -u 1090:1090 -p 24224:24224/tcp diff --git a/container/master-fluentd/Dockerfile b/container/master-fluentd/Dockerfile deleted file mode 100644 index 192ece3..0000000 --- a/container/master-fluentd/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: MulanPSL-2.0+ -# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. - -FROM fluentd-base:alpine - -COPY --chown=1090:1090 docker-fluentd.conf /fluentd/etc/docker-fluentd.conf diff --git a/container/master-fluentd/build b/container/master-fluentd/build deleted file mode 100755 index 7d0291d..0000000 --- a/container/master-fluentd/build +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env ruby -# SPDX-License-Identifier: MulanPSL-2.0+ -# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -# frozen_string_literal: true - -system 'docker build -t master-fluentd:alpine .' diff --git a/container/master-fluentd/build-depends b/container/master-fluentd/build-depends deleted file mode 100755 index 29ee9d3..0000000 --- a/container/master-fluentd/build-depends +++ /dev/null @@ -1 +0,0 @@ -fluentd-base diff --git a/container/master-fluentd/docker-fluentd.conf b/container/master-fluentd/docker-fluentd.conf deleted file mode 100644 index ef97d5c..0000000 --- a/container/master-fluentd/docker-fluentd.conf +++ /dev/null @@ -1,48 +0,0 @@ -<system> - workers 10 -</system> - -<source> - @type forward - bind 0.0.0.0 -</source> - -<match **> - @type copy - - <store> - @type elasticsearch - host 172.17.0.1 - port 9202 - suppress_type_name true - flush_interval 1s - index_name ${tag} - ssl_verify false - log_es_400_reason true - with_transporter_log true - </store> - - <store> - @type rabbitmq - host 172.17.0.1 - exchange logging-test - exchange_type fanout - exchange_durable false - heartbeat 10 - <format> - @type json - </format> - </store> -</match> - -<label @FLUENT_LOG> - <match fluent.*> - @type stdout - <format> - @type stdout - output_type single_value - message_key message - add_newline true - </format> - </match> -</label> diff --git a/container/master-fluentd/start b/container/master-fluentd/start deleted file mode 100755 index 9e02980..0000000 --- a/container/master-fluentd/start +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env ruby -# SPDX-License-Identifier: MulanPSL-2.0+ -# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. -# frozen_string_literal: true - -require_relative '../defconfig.rb' - -docker_rm "master-fluentd" - -cmd = %w[ - docker run - --restart=always - --name master-fluentd - -v /etc/localtime:/etc/localtime:ro - -d - -u 1090:1090 - -p 24225:24224/tcp - -p 24225:24224/udp - -e FLUENTD_CONF=docker-fluentd.conf - --log-driver json-file - --log-opt max-size=1g - master-fluentd:alpine -] -cmd += ['sh', '-c', 'umask 002 & fluentd -c /fluentd/etc/docker-fluentd.conf'] - -system(*cmd) diff --git a/container/sub-fluentd/docker-fluentd.conf b/container/sub-fluentd/docker-fluentd.conf index 24ad2e6..0092982 100644 --- a/container/sub-fluentd/docker-fluentd.conf +++ b/container/sub-fluentd/docker-fluentd.conf @@ -1,3 +1,8 @@ +<system> + workers 10 +</system> + +<worker 0> <source> @type tail path /srv/cci/serial/logs/* @@ -26,3 +31,52 @@ overflow_action block </buffer> </match> +</worker> + +<worker 1-9> +<source> + @type forward + bind 0.0.0.0 +</source> + +<match **> + @type copy + + <store> + @type stdout + <format> + @type stdout + output_type single_value + message_key log + add_newline true + </format> + </store> + + <store> + @type forward + flush_interval 0 + send_timeout 60 + heartbeat_type udp + heartbeat_interval 1 + recover_wait 10 + hard_timeout 60 + <server> + master-fluentd + host "#{ENV['FLUENTD_SERVER_HOST']}" + port "#{ENV['FLUENTD_SERVER_PORT']}" + </server> + </store> +</match> +</worker> + +<label @FLUENT_LOG> + <match fluent.*> + @type stdout + <format> + @type stdout + output_type single_value + message_key message + add_newline true + </format> + </match> +</label> diff --git a/container/sub-fluentd/start b/container/sub-fluentd/start index fa64dba..53a74c4 100755 --- a/container/sub-fluentd/start +++ b/container/sub-fluentd/start @@ -12,12 +12,13 @@ names = Set.new %w[ ]
defaults = relevant_defaults(names) -FLUENTD_SERVER_PORT = defaults['FLUENTD_SERVER_PORT'] || '24224' +defaults['FLUENTD_SERVER_PORT'] ||= '24224' +env = docker_env(defaults) raise 'must config FLUENTD_SERVER_HOST' unless defaults['FLUENTD_SERVER_HOST']
docker_rm 'sub-fluentd'
-cmd = %W[ +cmd = %w[ docker run --restart=always --name sub-fluentd @@ -25,12 +26,11 @@ cmd = %W[ -v /srv/cci/serial/logs:/srv/cci/serial/logs:ro -v /srv/cci/serial/fluentd-pos:/srv/cci/serial/fluentd-pos -d +] + env + %w[ -u 1090:1090 - -p 24224:24224/tcp - -p 24224:24224/udp + -p 24225:24224/tcp + -p 24225:24224/udp -e FLUENTD_CONF=docker-fluentd.conf - -e FLUENTD_SERVER_HOST=#{defaults['FLUENTD_SERVER_HOST']} - -e FLUENTD_SERVER_PORT=#{FLUENTD_SERVER_PORT} --log-driver json-file --log-opt max-size=1g sub-fluentd:alpine