[Why] fluentd uses "bulk" to continuously write data to es. The es write capability needs to be optimized to adapt to high-concurrency scenarios.

Signed-off-by: Wu Zhende wuzhende666@163.com --- container/logging-es/Dockerfile | 12 ++++++++---- container/logging-es/start | 13 +++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/container/logging-es/Dockerfile b/container/logging-es/Dockerfile index 7f91f65..d32d10f 100644 --- a/container/logging-es/Dockerfile +++ b/container/logging-es/Dockerfile @@ -7,10 +7,14 @@ FROM gagara/elasticsearch-oss-arm64:7.6.2

MAINTAINER Wu Zhende wuzhende666@163.com

-RUN sed -i 's:#network.host: _site_:network.host: 0.0.0.0:' /usr/share/elasticsearch/config/elasticsearch.yml; -RUN sed -i '$a path.data: /srv/es/logging-es' /usr/share/elasticsearch/config/elasticsearch.yml; -RUN sed -i '$a node.name: node-1' /usr/share/elasticsearch/config/elasticsearch.yml; -RUN sed -i '$a cluster.initial_master_nodes: ["node-1"]' /usr/share/elasticsearch/config/elasticsearch.yml; +RUN sed -i 's:#network.host: _site_:network.host: 0.0.0.0:' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a path.data: /srv/es/logging-es' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a node.name: node-1' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a cluster.initial_master_nodes: ["node-1"]' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a indices.memory.index_buffer_size: 30%' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i '$a thread_pool.write.queue_size: 2000' /usr/share/elasticsearch/config/elasticsearch.yml && \ + sed -i "s/-Xms1g/-Xms30g/g" /usr/share/elasticsearch/config/jvm.options && \ + sed -i "s/-Xmx1g/-Xmx30g/g" /usr/share/elasticsearch/config/jvm.options

RUN mkdir /usr/share/elasticsearch/tmp && \ chown -R 1090:1090 /usr/share/elasticsearch diff --git a/container/logging-es/start b/container/logging-es/start index 9be7a5b..be9ac6d 100755 --- a/container/logging-es/start +++ b/container/logging-es/start @@ -19,3 +19,16 @@ cmd=( )

"${cmd[@]}" + +echo "Please wait 60s for setting index" +sleep 60 + +curl -H "Content-Type: application/json" -XPUT "localhost:9202/_settings" -d ' +{ + "index": { + "refresh_interval": "300s", + "translog.durability": "async", + "translog.sync_interval": "300s", + "translog.flush_threshold_size": "1024mb", + "merge.scheduler.max_thread_count": 1} +}'