[Error] Exception: No free connection (used 64 of 64) after timeout of 0.01s (Redis::PoolTimeoutError)

[How] I looked at the number of concurrent Redis visits on our cluster, and it peaked at over 900. The scheduler and TaskQueue are about 50/50. So increase the size of the Redis connection pool. Adapt to our concurrency

Signed-off-by: Wu Zhende wuzhende666@163.com --- src/lib/sched.cr | 1 + src/scheduler/redis_client.cr | 9 +++++++-- src/taskqueue/constants.cr | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/lib/sched.cr b/src/lib/sched.cr index b90759f..dfec365 100644 --- a/src/lib/sched.cr +++ b/src/lib/sched.cr @@ -32,6 +32,7 @@ class Sched

def initialize(env : HTTP::Server::Context) @es = Elasticsearch::Client.new + Redis::Client.set_pool_size(1000) @redis = Redis::Client.instance @task_queue = TaskQueueAPI.new @rgc = RemoteGitClient.new diff --git a/src/scheduler/redis_client.cr b/src/scheduler/redis_client.cr index a8d9dcb..030d1b7 100644 --- a/src/scheduler/redis_client.cr +++ b/src/scheduler/redis_client.cr @@ -12,13 +12,18 @@ class Redis::Client class_property :client HOST = (ENV.has_key?("REDIS_HOST") ? ENV["REDIS_HOST"] : JOB_REDIS_HOST) PORT = (ENV.has_key?("REDIS_PORT") ? ENV["REDIS_PORT"] : JOB_REDIS_PORT).to_i32 + @@size = 25

def self.instance Singleton::Of(self).instance end

- def initialize(host = HOST, port = PORT) - @client = Redis::PooledClient.new(host: host, port: port, pool_size: 25, pool_timeout: 0.01) + def initialize(host = HOST, port = PORT, pool_size = @@size) + @client = Redis::PooledClient.new(host: host, port: port, pool_size: pool_size, pool_timeout: 0.01) + end + + def self.set_pool_size(pool_size) + @@size = pool_size end

def keys(pattern) diff --git a/src/taskqueue/constants.cr b/src/taskqueue/constants.cr index c702382..6240c54 100644 --- a/src/taskqueue/constants.cr +++ b/src/taskqueue/constants.cr @@ -11,7 +11,7 @@ REDIS_PORT = 6379 # delimiter and exttract-ststs will loop consume job # when use 32 (scheduler use 25), meet Exception: # Exception: No free connection (used 32 of 32) -REDIS_POOL_NUM = 64 +REDIS_POOL_NUM = 1000

REDIS_POOL_TIMEOUT = 10 # ms