January 2021 - Compass-ci - mailweb.openeuler.org

[PATCH v2 compass-ci] container/openresty-proxy-cache: add nginx.conf
by cuiyili 08 Jan '21

08 Jan '21

http/https cache proxy and request forwarding processing Signed-off-by: cuiyili <2268260388(a)qq.com> --- container/openresty-proxy-cache/nginx.conf | 118 +++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100755 container/openresty-proxy-cache/nginx.conf diff --git a/container/openresty-proxy-cache/nginx.conf b/container/openresty-proxy-cache/nginx.conf new file mode 100755 index 0000000..2e83286 --- /dev/null +++ b/container/openresty-proxy-cache/nginx.conf @@ -0,0 +1,118 @@ +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ + +user nobody; +worker_processes auto; +pid /run/nginx.pid; + +events {} + +http { + proxy_cache_path /nginx_cache levels=1:2 keys_zone=my_cache:10m max_size=10g + inactive=6000m use_temp_path=off; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for" ' + '"$upstream_cache_status" $body_bytes_sent'; + access_log /tmp/access.log main; + error_log /tmp/error.log; + + resolver 8.8.8.8; + client_max_body_size 700m; + server_tokens off; + + proxy_cache my_cache; + proxy_ignore_headers Set-Cookie Cache-Control Expires; + proxy_cache_valid 200 1440m; + proxy_cache_valid 301 302 5m; + proxy_set_header X-Real-Ip $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + add_header X-Cache $upstream_cache_status; + add_header X-Via $server_addr; + + server { + listen 443 ssl; + ssl_certificate /cert/ca.crt; + ssl_certificate_key /cert/ca.key; + proxy_connect; + proxy_connect_allow 443; + + location / { + allow all; + proxy_ssl_server_name on; + set_by_lua_block $cur_ups { + uri = ngx.var.request_uri + host = ngx.var.http_host + + path_file = "/srv/initrd/tar/"..uri + local f = io.open(path_file) + if f then + return "/127.0.0.1:443/tar/"..uri + end + return uri + } + proxy_pass $scheme:/$cur_ups; + } + + location /tar { + allow all; + root /srv/initrd; + autoindex on; + autoindex_exact_size off; + } + + } + server { + listen 80; + proxy_connect; + proxy_connect_allow 443; + + location / { + allow all; + + content_by_lua_block { + local http = require "resty.http" + local httpc = http.new() + uri = ngx.var.request_uri + host = ngx.var.http_host + if string.find(host, "11333") then + h="" + else + h="/"..host + end + path_file = "/srv/initrd/tar/"..h..uri + + local f = io.open(path_file) + if f then + url_forward = "http://127.0.0.1:80/tar/"..h..uri + else + url_forward = "http:/"..h..uri + end + + res, error = httpc:request_uri(url_forward) + if res then + ngx.status = res.status + for k, v in pairs(res.headers) do + if k ~= "Transfer-Encoding" and k ~= "Connection" then + ngx.header[k] = v + end + end + ngx.say(res.body) + else + ngx.exit(ngx.HTTP_FORBIDDEN) + end + httpc:close() + + } + } + + location /tar { + allow all; + root /srv/initrd; + autoindex on; + autoindex_exact_size off; + } + + } +} -- 2.23.0

1 0

[PATCH v2 compass-ci] container/openresty-proxy-cache: add start script
by cuiyili 08 Jan '21

08 Jan '21

Signed-off-by: cuiyili <2268260388(a)qq.com> --- container/openresty-proxy-cache/start | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100755 container/openresty-proxy-cache/start diff --git a/container/openresty-proxy-cache/start b/container/openresty-proxy-cache/start new file mode 100755 index 0000000..a1f095d --- /dev/null +++ b/container/openresty-proxy-cache/start @@ -0,0 +1,22 @@ +#!/bin/bash +# Refer to: https://hub.docker.com/r/openresty/openresty +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# SPDX-License-Identifier: MulanPSL-2.0+ + +. $CCI_SRC/container/defconfig.sh + +docker_rm openresty-proxy-cache + +cmd=( + docker run -d + --name openresty-proxy-cache + --restart=always + -p 11333:80 + -p 11334:443 + -v $PWD/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf + -v /srv/initrd/tar:/srv/initrd/tar:ro + -v /etc/localtime:/etc/localtime:ro + openresty:proxy_cache +) + +"${cmd[@]}" -- 2.23.0

1 0

[PATCH compass-ci] lua_script/forward.lua: http/https request forwarding processing
by cuiyili 08 Jan '21

08 Jan '21

Signed-off-by: cuiyili <2268260388(a)qq.com> --- .../lua_script/forward.lua | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 container/openresty-proxy-cache/lua_script/forward.lua diff --git a/container/openresty-proxy-cache/lua_script/forward.lua b/container/openresty-proxy-cache/lua_script/forward.lua new file mode 100755 index 0000000..bd54742 --- /dev/null +++ b/container/openresty-proxy-cache/lua_script/forward.lua @@ -0,0 +1,32 @@ +local http = require "resty.http" +local httpc = http.new() +uri = ngx.var.request_uri +host = ngx.var.http_host +if string.find(host, "11333") then + h="" +else + h="/"..host +end +path_file = "/srv/initrd/tar/"..h..uri + +local f = io.open(path_file) +if f then + url_forward = "http://127.0.0.1:80/tar/"..h..uri +else + url_forward = "http:/"..h..uri +end + +res, error = httpc:request_uri(url_forward) +if res then + ngx.status = res.status + for k, v in pairs(res.headers) do + if k ~= "Transfer-Encoding" and k ~= "Connection" then + ngx.header[k] = v + end + end + ngx.say(res.body) +else + ngx.exit(ngx.HTTP_FORBIDDEN) +end + +httpc:close() -- 2.23.0

1 0

[PATCH v2 compass-ci] container/openresty-proxy-cache: add nginx.conf
by cuiyili 08 Jan '21

08 Jan '21

Signed-off-by: cuiyili <2268260388(a)qq.com> --- container/openresty-proxy-cache/nginx.conf | 85 ++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100755 container/openresty-proxy-cache/nginx.conf diff --git a/container/openresty-proxy-cache/nginx.conf b/container/openresty-proxy-cache/nginx.conf new file mode 100755 index 0000000..e3f11e0 --- /dev/null +++ b/container/openresty-proxy-cache/nginx.conf @@ -0,0 +1,85 @@ +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ + +user nobody; +worker_processes auto; +pid /run/nginx.pid; + +events {} + +http { + proxy_cache_path /nginx_cache levels=1:2 keys_zone=my_cache:10m max_size=10g + inactive=6000m use_temp_path=off; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for" ' + '"$upstream_cache_status" $body_bytes_sent'; + access_log /tmp/access.log main; + error_log /tmp/error.log; + + resolver 8.8.8.8; + client_max_body_size 700m; + server_tokens off; + + proxy_cache my_cache; + proxy_ignore_headers Set-Cookie Cache-Control Expires; + proxy_cache_valid 200 1440m; + proxy_cache_valid 301 302 5m; + proxy_set_header X-Real-Ip $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + add_header X-Cache $upstream_cache_status; + add_header X-Via $server_addr; + + server { + listen 443 ssl; + ssl_certificate /cert/ca.crt; + ssl_certificate_key /cert/ca.key; + proxy_connect; + proxy_connect_allow 443; + + location / { + allow all; + proxy_ssl_server_name on; + set_by_lua_block $cur_ups { + uri = ngx.var.request_uri + host = ngx.var.http_host + + path_file = "/srv/initrd/tar/"..uri + local f = io.open(path_file) + if f then + return "/127.0.0.1:443/tar/"..uri + end + return uri + } + proxy_pass $scheme:/$cur_ups; + } + + location /tar { + allow all; + root /srv/initrd; + autoindex on; + autoindex_exact_size off; + } + + } + server { + listen 80; + proxy_connect; + proxy_connect_allow 443; + + location / { + allow all; + + content_by_lua_file lua_script/forward.lua; + } + + location /tar { + allow all; + root /srv/initrd; + autoindex on; + autoindex_exact_size off; + } + + } +} -- 2.23.0

1 0

[PATCH lkp-tests] hosts/dc-*g: add nr_cpu
by Hu Xuejiao 08 Jan '21

08 Jan '21

[why] Because when job.rb(expand_params) deal with params/nr_threads [ ($1.to_f * nr_cpu / 100).to_i ], it will handle the exception --"rescue KeyError"-- in running docker [so] i need add the nr_cpu in dc-*g Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- hosts/dc-1g | 1 + hosts/dc-2g | 1 + hosts/dc-4g | 1 + hosts/dc-8g | 1 + 4 files changed, 4 insertions(+) diff --git a/hosts/dc-1g b/hosts/dc-1g index ec1e59b9..24ff4227 100644 --- a/hosts/dc-1g +++ b/hosts/dc-1g @@ -1 +1,2 @@ +nr_cpu: 1 memory: 1g diff --git a/hosts/dc-2g b/hosts/dc-2g index b9509a2c..82c2522e 100644 --- a/hosts/dc-2g +++ b/hosts/dc-2g @@ -1 +1,2 @@ +nr_cpu: 2 memory: 2g diff --git a/hosts/dc-4g b/hosts/dc-4g index 52e0636d..30353733 100644 --- a/hosts/dc-4g +++ b/hosts/dc-4g @@ -1 +1,2 @@ +nr_cpu: 4 memory: 4g diff --git a/hosts/dc-8g b/hosts/dc-8g index 333d6250..470f3094 100644 --- a/hosts/dc-8g +++ b/hosts/dc-8g @@ -1 +1,2 @@ +nr_cpu: 8 memory: 8g -- 2.23.0

1 0

[PATCH compass-ci] container/openresty-proxy-cache: add nginx.conf
by cuiyili 08 Jan '21

08 Jan '21

Signed-off-by: cuiyili <2268260388(a)qq.com> --- container/openresty-proxy-cache/nginx.conf | 85 ++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100755 container/openresty-proxy-cache/nginx.conf diff --git a/container/openresty-proxy-cache/nginx.conf b/container/openresty-proxy-cache/nginx.conf new file mode 100755 index 0000000..79aa10e --- /dev/null +++ b/container/openresty-proxy-cache/nginx.conf @@ -0,0 +1,85 @@ +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ + +user nobody; +worker_processes auto; +pid /run/nginx.pid; + +events {} + +http { + proxy_cache_path /nginx_cache levels=1:2 keys_zone=my_cache:10m max_size=10g + inactive=6000m use_temp_path=off; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for" ' + '"$upstream_cache_status" $body_bytes_sent'; + access_log /tmp/access.log main; + error_log /tmp/error.log; + + resolver 8.8.8.8; + client_max_body_size 700m; + server_tokens off; + + proxy_cache my_cache; + proxy_ignore_headers Set-Cookie Cache-Control Expires; + proxy_cache_valid 200 1440m; + proxy_cache_valid 301 302 5m; + proxy_set_header X-Real-Ip $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + add_header X-Cache $upstream_cache_status; + add_header X-Via $server_addr; + + server { + listen 443 ssl; + ssl_certificate /cert/ca.crt; + ssl_certificate_key /cert/ca.key; + proxy_connect; + proxy_connect_allow 443; + + location / { + allow all; + proxy_ssl_server_name on; + set_by_lua_block $cur_ups { + uri = ngx.var.request_uri + host = ngx.var.http_host + + path_file = "/srv/initrd/tar/"..uri + local f = io.open(path_file) + if f then + return "/127.0.0.1:443/tar/"..uri + end + return uri + } + proxy_pass $scheme:/$cur_ups; + } + + location /tar { + allow all; + root /srv/initrd; + autoindex on; + autoindex_exact_size off; + } + + } + server { + listen 80; + proxy_connect; + proxy_connect_allow 443; + + location / { + allow all; + + content_by_lua_file lua_script/forward.lua; + } + + location /tar { + allow all; + root /srv/initrd; + autoindex on; + autoindex_exact_size off; + } + + } +} -- 2.23.0

1 0

[PATCH compass-ci] container/openresty-proxy-cache: add start script
by cuiyili 08 Jan '21

08 Jan '21

Signed-off-by: cuiyili <2268260388(a)qq.com> --- container/openresty-proxy-cache/start | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 container/openresty-proxy-cache/start diff --git a/container/openresty-proxy-cache/start b/container/openresty-proxy-cache/start new file mode 100755 index 0000000..b946426 --- /dev/null +++ b/container/openresty-proxy-cache/start @@ -0,0 +1,23 @@ +#!/bin/bash +# Refer to: https://hub.docker.com/r/openresty/openresty +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# SPDX-License-Identifier: MulanPSL-2.0+ + +. $CCI_SRC/container/defconfig.sh + +docker_rm openresty-proxy-cache + +cmd=( + docker run -d + --name openresty-proxy-cache + --restart=always + -p 11333:80 + -p 11334:443 + -v $PWD/nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf + -v $PWD/lua_script/forward.lua:/usr/local/openresty/nginx/lua_script/forward.lua + -v /srv/initrd/tar:/srv/initrd/tar:ro + -v /etc/localtime:/etc/localtime:ro + openresty:proxy_cache +) + +"${cmd[@]}" -- 2.23.0

1 0

[PATCH v5 compass-ci 7/7] send-internet-mail/start: add key REDIS_PORT
by Luan Shengde 08 Jan '21

08 Jan '21

Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/send-internet-mail/start | 1 + 1 file changed, 1 insertion(+) diff --git a/container/send-internet-mail/start b/container/send-internet-mail/start index 02c6ab6..136435d 100755 --- a/container/send-internet-mail/start +++ b/container/send-internet-mail/start @@ -13,6 +13,7 @@ names = Set.new %w[ ROBOT_EMAIL_PASSWORD SEND_MAIL_PORT SENT_MAILDIR + REDIS_PORT ] defaults = relevant_defaults(names) -- 2.23.0

1 0

[PATCH v5 compass-ci 6/7] lib/mail-post: add email checks
by Luan Shengde 08 Jan '21

08 Jan '21

add check email count only email addresses that email count within the limited value can be sent emails add check email mapping if has set email mapping for the entered address. the address will be converted to the mapping value Signed-off-by: Luan Shengde <shdluan(a)163.com> --- lib/mail-post.rb | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) mode change 100644 => 100755 lib/mail-post.rb diff --git a/lib/mail-post.rb b/lib/mail-post.rb old mode 100644 new mode 100755 index 9d3e601..803a134 --- a/lib/mail-post.rb +++ b/lib/mail-post.rb @@ -57,14 +57,32 @@ post '/send_mail_encode' do send_mail(mail_info) end +def check_email_limit(mail_info) + email_limit = EmailRateLimit.new(mail_info) + email_limit.check_email_counts +end + +def check_email_mapping(mail_info) + email_mapping = EmailAddrMapping.new(mail_info) + email_mapping.check_email_mapping +end + def check_send_mail(mail_info) raise 'no/empty subject.' if mail_info['subject'].nil? || mail_info['subject'].empty? raise 'no/empty email_to address.' if mail_info['to'].nil? || mail_info['to'].empty? raise 'no/empty email content.' if mail_info['body'].nil? || mail_info['body'].empty? + + return mail_info unless ENV['SEND_MAIL_PORT'].to_s == '49000' + + mail_info = check_email_mapping(mail_info.clone) + mail_info = check_email_limit(mail_info.clone) + return mail_info end def send_mail(mail_info) - check_send_mail(mail_info) + mail_info = check_send_mail(mail_info) + return if mail_info['to'].empty? + mail = Mail.new do from ENV['ROBOT_EMAIL_ADDRESS'] subject mail_info['subject'] @@ -79,7 +97,6 @@ def send_mail(mail_info) end def check_to_store_email(mail) - return if ENV['SEND_MAIL_PORT'].to_s != '49000' return if ENV['HOST_SERVER'] != 'z9' -- 2.23.0

1 0

[PATCH v5 compass-ci 5/7] send-internet-mail: email rate limit
by Luan Shengde 08 Jan '21

08 Jan '21

add email_in_limit to store email address that send count less than limit value add email_out_limit to store email address that send count beyond limit value the email queues email_in_limit/email_out_limit will be reset everyday at 00:00, every time send mail to an email address, the value for the email address in email_in_limit will +1, until the value up to the limited value, the email address will be moved to email_out_limit, and the email send to the address will be throw away. Signed-off-by: Luan Shengde <shdluan(a)163.com> --- .../send-internet-mail/email_limit_queue.rb | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 container/send-internet-mail/email_limit_queue.rb diff --git a/container/send-internet-mail/email_limit_queue.rb b/container/send-internet-mail/email_limit_queue.rb new file mode 100755 index 0000000..c7e3954 --- /dev/null +++ b/container/send-internet-mail/email_limit_queue.rb @@ -0,0 +1,62 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'json' +require 'redis' + +# check if the emails count has beyond the limit. +# email_in_limit is used to store email address email count +# less than the limit value. +# email_out_limit is used to store email address email count +# beyond the limit. +# case the email is the first one for the day, the email will +# be add to email_in_limit with value 1. +# case the email in email_in_limit but email count less than the +# limit value, its value will +1. +# case the email's email count up to the limit value, the email +# will be moved to email_out_limit. +# when the email is in email_out_limit, if send mail to the email +# address, the email will be kicked out from the email address bar. +class EmailRateLimit + def initialize(mail_info) + @mail_info = mail_info + @redis = Redis.new('host' => REDIS_HOST, 'port' => REDIS_PORT) + end + + def check_email_counts + email_to = @mail_info['to'].clone + email_cc = @mail_info['cc'].clone + @mail_info['to'] = check_emails(email_to) + @mail_info['cc'] = check_emails(email_cc) + + return @mail_info + end + + def check_emails(mail_list) + return if mail_list.nil? || mail_list.empty? + + mail_list.clone.each do |email| + if @redis.hexists 'email_out_limit', email + mail_list -= [email] + next + elsif @redis.hexists 'email_in_limit', email + email_account = @redis.hget 'email_in_limit', email + @redis.hset 'email_in_limit', email, email_account.to_i + 1 + else + @redis.hset 'email_in_limit', email, 1 + end + + change_queue(email) + end + return mail_list + end + + def change_queue(email) + return unless (@redis.hget 'email_in_limit', email).to_i >= 10 + + @redis.hdel 'email_in_limit', email + @redis.hset 'email_out_limit', email, 10 + end +end -- 2.23.0

1 0