[PATCH compass-ci] log_system/fluentd: change the architecture

21 Oct 2020

[Why]
  Now we have three types of fluentd:sub-fluentd/fluentd/master-fluentd.
  sub-fluentd and fluentd have similar functions, can be merged.
  So change to two types of fluentd:sub-fluentd and fluentd.
  sub-fluentd only collections logs then forward to fluentd.
  All data processing is performed on fluentd.

Signed-off-by: Wu Zhende <wuzhende666@163.com>
---
 container/fluentd/docker-fluentd.conf        | 86 +++++++-------------
 container/fluentd/start                      |  2 -
 container/master-fluentd/Dockerfile          |  6 --
 container/master-fluentd/build               |  6 --
 container/master-fluentd/build-depends       |  1 -
 container/master-fluentd/docker-fluentd.conf | 48 -----------
 container/master-fluentd/start               | 26 ------
 container/sub-fluentd/docker-fluentd.conf    | 54 ++++++++++++
 container/sub-fluentd/start                  | 12 +--
 9 files changed, 89 insertions(+), 152 deletions(-)
 delete mode 100644 container/master-fluentd/Dockerfile
 delete mode 100755 container/master-fluentd/build
 delete mode 100755 container/master-fluentd/build-depends
 delete mode 100644 container/master-fluentd/docker-fluentd.conf
 delete mode 100755 container/master-fluentd/start

diff --git a/container/fluentd/docker-fluentd.conf b/container/fluentd/docker-fluentd.conf
index 23c7585..4660988 100644
--- a/container/fluentd/docker-fluentd.conf
+++ b/container/fluentd/docker-fluentd.conf
@@ -2,17 +2,9 @@
   workers 10
 </system>
 
-<worker 0>
 <source>
-  @type tail
-  path /srv/cci/serial/logs/*
-  pos_file /srv/cci/serial/fluentd-pos/serial.log.pos
-  tag serial.*
-  path_key serial_path
-  refresh_interval 1s
-  <parse>
-    @type none
-  </parse>
+  @type forward
+  bind 0.0.0.0
 </source>
 
 <filter **>
@@ -24,34 +16,16 @@
 </filter>
 
 <match serial.**>
-  @type copy
-  <store>
-    @type rabbitmq
-    host 172.17.0.1
-    exchange logging-test
-    exchange_type fanout
-    exchange_durable false
-    heartbeat 10
-    <format>
-      @type json
-    </format>
-  </store>
+  @type rabbitmq
+  host 172.17.0.1
+  exchange logging-test
+  exchange_type fanout
+  exchange_durable false
+  heartbeat 10
+  <format>
+    @type json
+  </format>
 </match>
-</worker>
-
-<worker 1-9>
-<source>
-  @type forward
-  bind 0.0.0.0
-</source>
-
-<filter **>
-  @type record_transformer
-  enable_ruby
-  <record>
-    time ${time.strftime('%Y-%m-%d %H:%M:%S%z')}
-  </record>
-</filter>
 
 <filter **>
   @type parser
@@ -65,31 +39,29 @@
   @type copy
 
   <store>
-    @type stdout
-    <format>
-      @type stdout
-      output_type single_value
-      message_key log
-      add_newline true
-    </format>
+    @type elasticsearch
+    host 172.17.0.1
+    port 9202
+    suppress_type_name true
+    flush_interval 1s
+    index_name ${tag}
+    ssl_verify false
+    log_es_400_reason true
+    with_transporter_log true
   </store>
 
   <store>
-    @type forward
-    flush_interval 0
-    send_timeout 60
-    heartbeat_type udp
-    heartbeat_interval 1
-    recover_wait 10
-    hard_timeout 60
-    <server>
-      master-fluentd
-      host 172.17.0.1
-      port 24225
-    </server>
+    @type rabbitmq
+    host 172.17.0.1
+    exchange logging-test
+    exchange_type fanout
+    exchange_durable false
+    heartbeat 10
+    <format>
+      @type json
+    </format>
   </store>
 </match>
-</worker>
 
 <label @FLUENT_LOG>
   <match fluent.*>
diff --git a/container/fluentd/start b/container/fluentd/start
index 8ad7c1e..b7f9c73 100755
--- a/container/fluentd/start
+++ b/container/fluentd/start
@@ -12,8 +12,6 @@ cmd = %w[
   --restart=always
   --name fluentd
   -v /etc/localtime:/etc/localtime:ro
-  -v /srv/cci/serial/logs:/srv/cci/serial/logs:ro
-  -v /srv/cci/serial/fluentd-pos:/srv/cci/serial/fluentd-pos
   -d
   -u 1090:1090
   -p 24224:24224/tcp
diff --git a/container/master-fluentd/Dockerfile b/container/master-fluentd/Dockerfile
deleted file mode 100644
index 192ece3..0000000
--- a/container/master-fluentd/Dockerfile
+++ /dev/null
@@ -1,6 +0,0 @@
-# SPDX-License-Identifier: MulanPSL-2.0+
-# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
-
-FROM fluentd-base:alpine
-
-COPY --chown=1090:1090 docker-fluentd.conf /fluentd/etc/docker-fluentd.conf
diff --git a/container/master-fluentd/build b/container/master-fluentd/build
deleted file mode 100755
index 7d0291d..0000000
--- a/container/master-fluentd/build
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/env ruby
-# SPDX-License-Identifier: MulanPSL-2.0+
-# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
-# frozen_string_literal: true
-
-system 'docker build -t master-fluentd:alpine .'
diff --git a/container/master-fluentd/build-depends b/container/master-fluentd/build-depends
deleted file mode 100755
index 29ee9d3..0000000
--- a/container/master-fluentd/build-depends
+++ /dev/null
@@ -1 +0,0 @@
-fluentd-base
diff --git a/container/master-fluentd/docker-fluentd.conf b/container/master-fluentd/docker-fluentd.conf
deleted file mode 100644
index ef97d5c..0000000
--- a/container/master-fluentd/docker-fluentd.conf
+++ /dev/null
@@ -1,48 +0,0 @@
-<system>
-  workers 10
-</system>
-
-<source>
-  @type forward
-  bind 0.0.0.0
-</source>
-
-<match **>
-  @type copy
-
-  <store>
-    @type elasticsearch
-    host 172.17.0.1
-    port 9202
-    suppress_type_name true
-    flush_interval 1s
-    index_name ${tag}
-    ssl_verify false
-    log_es_400_reason true
-    with_transporter_log true
-  </store>
-
-  <store>
-    @type rabbitmq
-    host 172.17.0.1
-    exchange logging-test
-    exchange_type fanout
-    exchange_durable false
-    heartbeat 10
-    <format>
-      @type json
-    </format>
-  </store>
-</match>
-
-<label @FLUENT_LOG>
-  <match fluent.*>
-    @type stdout
-    <format>
-      @type stdout
-      output_type single_value
-      message_key message
-      add_newline true
-    </format>
-  </match>
-</label>
diff --git a/container/master-fluentd/start b/container/master-fluentd/start
deleted file mode 100755
index 9e02980..0000000
--- a/container/master-fluentd/start
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env ruby
-# SPDX-License-Identifier: MulanPSL-2.0+
-# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved.
-# frozen_string_literal: true
-
-require_relative '../defconfig.rb'
-
-docker_rm "master-fluentd"
-
-cmd = %w[
-  docker run
-  --restart=always
-  --name master-fluentd
-  -v /etc/localtime:/etc/localtime:ro
-  -d
-  -u 1090:1090
-  -p 24225:24224/tcp
-  -p 24225:24224/udp
-  -e FLUENTD_CONF=docker-fluentd.conf
-  --log-driver json-file
-  --log-opt max-size=1g
-  master-fluentd:alpine
-]
-cmd += ['sh', '-c', 'umask 002 & fluentd -c /fluentd/etc/docker-fluentd.conf']
-
-system(*cmd)
diff --git a/container/sub-fluentd/docker-fluentd.conf b/container/sub-fluentd/docker-fluentd.conf
index 24ad2e6..0092982 100644
--- a/container/sub-fluentd/docker-fluentd.conf
+++ b/container/sub-fluentd/docker-fluentd.conf
@@ -1,3 +1,8 @@
+<system>
+  workers 10
+</system>
+
+<worker 0>
 <source>
   @type tail
   path /srv/cci/serial/logs/*
@@ -26,3 +31,52 @@
     overflow_action block
   </buffer>
 </match>
+</worker>
+
+<worker 1-9>
+<source>
+  @type forward
+  bind 0.0.0.0
+</source>
+
+<match **>
+  @type copy
+
+  <store>
+    @type stdout
+    <format>
+      @type stdout
+      output_type single_value
+      message_key log
+      add_newline true
+    </format>
+  </store>
+
+  <store>
+    @type forward
+    flush_interval 0
+    send_timeout 60
+    heartbeat_type udp
+    heartbeat_interval 1
+    recover_wait 10
+    hard_timeout 60
+    <server>
+      master-fluentd
+      host "#{ENV['FLUENTD_SERVER_HOST']}"
+      port "#{ENV['FLUENTD_SERVER_PORT']}"
+    </server>
+  </store>
+</match>
+</worker>
+
+<label @FLUENT_LOG>
+  <match fluent.*>
+    @type stdout
+    <format>
+      @type stdout
+      output_type single_value
+      message_key message
+      add_newline true
+    </format>
+  </match>
+</label>
diff --git a/container/sub-fluentd/start b/container/sub-fluentd/start
index fa64dba..53a74c4 100755
--- a/container/sub-fluentd/start
+++ b/container/sub-fluentd/start
@@ -12,12 +12,13 @@ names = Set.new %w[
 ]
 
 defaults = relevant_defaults(names)
-FLUENTD_SERVER_PORT = defaults['FLUENTD_SERVER_PORT'] || '24224'
+defaults['FLUENTD_SERVER_PORT'] ||= '24224'
+env = docker_env(defaults)
 raise 'must config FLUENTD_SERVER_HOST' unless defaults['FLUENTD_SERVER_HOST']
 
 docker_rm 'sub-fluentd'
 
-cmd = %W[
+cmd = %w[
   docker run
   --restart=always
   --name sub-fluentd
@@ -25,12 +26,11 @@ cmd = %W[
   -v /srv/cci/serial/logs:/srv/cci/serial/logs:ro
   -v /srv/cci/serial/fluentd-pos:/srv/cci/serial/fluentd-pos
   -d
+] + env + %w[
   -u 1090:1090
-  -p 24224:24224/tcp
-  -p 24224:24224/udp
+  -p 24225:24224/tcp
+  -p 24225:24224/udp
   -e FLUENTD_CONF=docker-fluentd.conf
-  -e FLUENTD_SERVER_HOST=#{defaults['FLUENTD_SERVER_HOST']}
-  -e FLUENTD_SERVER_PORT=#{FLUENTD_SERVER_PORT}
   --log-driver json-file
   --log-opt max-size=1g
   sub-fluentd:alpine
-- 
2.23.0

    

Wu Zhende

tags

participants (1)