- Compass-ci - mailweb.openeuler.org

[PATCH v2 lkp-tests 2/3] iso2rootfs: delete the unnecessary fields
by Yu Chuan 26 Nov '20

26 Nov '20

[Why] 'INITRD_HTTP_HOST' and 'INITRD_HTTP_PORT' should be added by the user on demand at runtime, no need to merge into the git repository. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 2 -- tests/iso2rootfs | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 4f3e9c83e6cd..a0ffd280676d 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -7,8 +7,6 @@ iso2rootfs: rootfs_protocol: rootfs_server: rootfs_path: - initrd_http_host: - initrd_http_port: # config rootfs related fields ## supported fields: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index 761fbb8b70cf..9adbc930b48f 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -49,8 +49,6 @@ check_yaml_vars() "rootfs_protocol" "rootfs_server" "rootfs_path" - "initrd_http_host" - "initrd_http_port" ) for yaml_t in "${yaml_vars[@]}" @@ -190,7 +188,7 @@ check_os() get_cgz() { - local cgz_url="http://${initrd_http_host}:${initrd_http_port}/initrd/pkg/nfs/${os}/${os_arch}/${os_version}/iso2rootfs.cgz" + local cgz_url="http://${INITRD_HTTP_HOST}:${INITRD_HTTP_PORT}/initrd/pkg/nfs/${os}/${os_arch}/${os_version}/iso2rootfs.cgz" local cgz_name cgz_name=$(basename "${cgz_url}") local cgz_path_t=${ENV_DIR}/${cgz_name} -- 2.23.0

1 0

[PATCH v2 lkp-tests 1/3] iso2rootfs: delete the meaningless field
by Yu Chuan 26 Nov '20

26 Nov '20

[Why] The 'iso2rootfs_cgz' field will always be 'iso2rootfs.cgz', so delete this field, and use the 'iso2rootfs.cgz' directly. Signed-off-by: Yu Chuan <13186087857(a)163.com> --- jobs/iso2rootfs.yaml | 1 - tests/iso2rootfs | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/jobs/iso2rootfs.yaml b/jobs/iso2rootfs.yaml index 1a643b538ac1..4f3e9c83e6cd 100644 --- a/jobs/iso2rootfs.yaml +++ b/jobs/iso2rootfs.yaml @@ -1,7 +1,6 @@ suite: iso2rootfs category: benchmark iso2rootfs: - iso2rootfs_cgz: iso_os: iso_arch: iso_version: diff --git a/tests/iso2rootfs b/tests/iso2rootfs index fcaadaa1ba46..761fbb8b70cf 100755 --- a/tests/iso2rootfs +++ b/tests/iso2rootfs @@ -43,7 +43,6 @@ check_yaml_vars() "os" "os_arch" "os_version" - "iso2rootfs_cgz" "iso_os" "iso_arch" "iso_version" @@ -191,7 +190,7 @@ check_os() get_cgz() { - local cgz_url="http://${initrd_http_host}:${initrd_http_port}/initrd/pkg/nfs/${os}/${os_arch}/${os_version}/${iso2rootfs_cgz}" + local cgz_url="http://${initrd_http_host}:${initrd_http_port}/initrd/pkg/nfs/${os}/${os_arch}/${os_version}/iso2rootfs.cgz" local cgz_name cgz_name=$(basename "${cgz_url}") local cgz_path_t=${ENV_DIR}/${cgz_name} -- 2.23.0

1 0

[PATCH v4 lkp-tests] sbin/submit: submit merge_yaml/*.yaml
by Hu Xuejiao 26 Nov '20

26 Nov '20

[why] We will use 'rspec spec/submit_spec.rb' to merge job.yaml, it support merging from external file by "<<: job.yaml" Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- spec/submit/merge_yaml/atomic.yaml | 3 +++ .../merge_yaml/atomic/atomic-defaults.yaml | 19 +++++++++++++++++++ .../merge_yaml/yaml_merge_included_files.yaml | 1 + .../yaml_merge_included_files-defaults.yaml | 19 +++++++++++++++++++ spec/submit_spec.rb | 4 ++++ 5 files changed, 46 insertions(+) create mode 100644 spec/submit/merge_yaml/atomic.yaml create mode 100644 spec/submit/merge_yaml/atomic/atomic-defaults.yaml create mode 100644 spec/submit/merge_yaml/yaml_merge_included_files.yaml create mode 100644 spec/submit/merge_yaml/yaml_merge_included_files/yaml_merge_included_files-defaults.yaml diff --git a/spec/submit/merge_yaml/atomic.yaml b/spec/submit/merge_yaml/atomic.yaml new file mode 100644 index 00000000..331cb470 --- /dev/null +++ b/spec/submit/merge_yaml/atomic.yaml @@ -0,0 +1,3 @@ +suite: atomic + +atomic: diff --git a/spec/submit/merge_yaml/atomic/atomic-defaults.yaml b/spec/submit/merge_yaml/atomic/atomic-defaults.yaml new file mode 100644 index 00000000..42b53c13 --- /dev/null +++ b/spec/submit/merge_yaml/atomic/atomic-defaults.yaml @@ -0,0 +1,19 @@ +--- +suite: atomic +atomic: +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + atomic: + cpufreq_governor: +monitors: diff --git a/spec/submit/merge_yaml/yaml_merge_included_files.yaml b/spec/submit/merge_yaml/yaml_merge_included_files.yaml new file mode 100644 index 00000000..30019c6c --- /dev/null +++ b/spec/submit/merge_yaml/yaml_merge_included_files.yaml @@ -0,0 +1 @@ +<<: atomic.yaml diff --git a/spec/submit/merge_yaml/yaml_merge_included_files/yaml_merge_included_files-defaults.yaml b/spec/submit/merge_yaml/yaml_merge_included_files/yaml_merge_included_files-defaults.yaml new file mode 100644 index 00000000..42b53c13 --- /dev/null +++ b/spec/submit/merge_yaml/yaml_merge_included_files/yaml_merge_included_files-defaults.yaml @@ -0,0 +1,19 @@ +--- +suite: atomic +atomic: +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + atomic: + cpufreq_governor: +monitors: diff --git a/spec/submit_spec.rb b/spec/submit_spec.rb index a83d2102..92d8fa3d 100644 --- a/spec/submit_spec.rb +++ b/spec/submit_spec.rb @@ -44,4 +44,8 @@ describe 'submit job spec' do it 'link matrix' do submit_job('matrix') end + + it 'merge yaml spec' do + submit_job('merge_yaml') + end end -- 2.23.0

1 0

[PATCH v3 lkp-tests] sbin/submit: submit separate_yaml/*.yaml
by Hu Xuejiao 26 Nov '20

26 Nov '20

[why] We will use 'rspec spec/submit_spec.rb' to separate job.yaml, it contains several yaml documents separated by "---" Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- spec/submit/separate_yaml/ftq.yaml | 19 +++++++++++++ .../ftq/ftq-1000-2-100000ss-add.yaml | 27 +++++++++++++++++++ .../ftq/ftq-10000-2-100000ss-add.yaml | 27 +++++++++++++++++++ .../ftq/ftq-20-2-6000ss-cache.yaml | 27 +++++++++++++++++++ spec/submit_spec.rb | 4 +++ 5 files changed, 104 insertions(+) create mode 100644 spec/submit/separate_yaml/ftq.yaml create mode 100644 spec/submit/separate_yaml/ftq/ftq-1000-2-100000ss-add.yaml create mode 100644 spec/submit/separate_yaml/ftq/ftq-10000-2-100000ss-add.yaml create mode 100644 spec/submit/separate_yaml/ftq/ftq-20-2-6000ss-cache.yaml diff --git a/spec/submit/separate_yaml/ftq.yaml b/spec/submit/separate_yaml/ftq.yaml new file mode 100644 index 00000000..fbb9dc55 --- /dev/null +++ b/spec/submit/separate_yaml/ftq.yaml @@ -0,0 +1,19 @@ +suite: ftq + +nr_task: 100% +samples: 100000ss + +ftq: + test: add + freq: 10000 + +--- +ftq: + test: add + freq: 1000 + +--- +samples: 6000ss +ftq: + test: cache + freq: 20 diff --git a/spec/submit/separate_yaml/ftq/ftq-1000-2-100000ss-add.yaml b/spec/submit/separate_yaml/ftq/ftq-1000-2-100000ss-add.yaml new file mode 100644 index 00000000..f3e2c553 --- /dev/null +++ b/spec/submit/separate_yaml/ftq/ftq-1000-2-100000ss-add.yaml @@ -0,0 +1,27 @@ +--- +suite: ftq +nr_task: 2 +samples: 100000ss +ftq: + test: add + freq: 1000 +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + ftq: + nr_task: 2 + samples: 100000ss + freq: 1000 + test: add + cpufreq_governor: +monitors: diff --git a/spec/submit/separate_yaml/ftq/ftq-10000-2-100000ss-add.yaml b/spec/submit/separate_yaml/ftq/ftq-10000-2-100000ss-add.yaml new file mode 100644 index 00000000..63bac5cf --- /dev/null +++ b/spec/submit/separate_yaml/ftq/ftq-10000-2-100000ss-add.yaml @@ -0,0 +1,27 @@ +--- +suite: ftq +nr_task: 2 +samples: 100000ss +ftq: + test: add + freq: 10000 +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + ftq: + nr_task: 2 + samples: 100000ss + freq: 10000 + test: add + cpufreq_governor: +monitors: diff --git a/spec/submit/separate_yaml/ftq/ftq-20-2-6000ss-cache.yaml b/spec/submit/separate_yaml/ftq/ftq-20-2-6000ss-cache.yaml new file mode 100644 index 00000000..a802dc1b --- /dev/null +++ b/spec/submit/separate_yaml/ftq/ftq-20-2-6000ss-cache.yaml @@ -0,0 +1,27 @@ +--- +suite: ftq +nr_task: 2 +samples: 6000ss +ftq: + test: cache + freq: 20 +lab: spec_lab +testbox: vm-2p8g--spec_submit +tbox_group: vm-2p8g--spec_submit +provider: qemu +template: kvm +nr_node: 1 +nr_cpu: 2 +memory: 8G +arch: aarch64 +cpufreq_governor: +need_kconfig: +- CONFIG_KVM_GUEST=y +pp: + ftq: + nr_task: 2 + samples: 6000ss + freq: 20 + test: cache + cpufreq_governor: +monitors: diff --git a/spec/submit_spec.rb b/spec/submit_spec.rb index a83d2102..169d7812 100644 --- a/spec/submit_spec.rb +++ b/spec/submit_spec.rb @@ -44,4 +44,8 @@ describe 'submit job spec' do it 'link matrix' do submit_job('matrix') end + + it 'separate yaml spec' do + submit_job('separate_yaml') + end end -- 2.23.0

1 0

[PATCH v4 lkp-tests] setup/sysctl: add system parameters
by Zhang Yu 26 Nov '20

26 Nov '20

[why] When testing need to setup system parameters. [how] Add parameters to setup/sysctl. Signed-off-by: Zhang Yu <2134782174(a)qq.com> --- setup/sysctl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup/sysctl b/setup/sysctl index 81374939..2ace4db2 100755 --- a/setup/sysctl +++ b/setup/sysctl @@ -202,6 +202,9 @@ # - sc_netdev_max_backlog # - sc_rmem_max # - sc_wmem_max +# - sc_ipv4_tcp_rmem +# - sc_ipv4_tcp_wmem +# - sc_sched_features . $LKP_SRC/lib/reproduce-log.sh @@ -405,8 +408,11 @@ [ -n "$sc_ipv4_tcp_max_tw_buckets" ] && log_eval "echo '$sc_ipv4_tcp_max_tw_buckets' > '/proc/sys/net/ipv4/tcp_max_tw_buckets'" [ -n "$sc_ipv4_tcp_max_syn_backlog" ] && log_eval "echo '$sc_ipv4_tcp_max_syn_backlog' > '/proc/sys/net/ipv4/tcp_max_syn_backlog'" [ -n "$sc_ipv4_ip_local_port_range" ] && log_eval "echo '$sc_ipv4_ip_local_port_range' > '/proc/sys/net/ipv4/ip_local_port_range'" +[ -n "$sc_ipv4_tcp_rmem" ] && log_eval "echo '$sc_ipv4_tcp_rmem' > '/proc/sys/net/ipv4/tcp_rmem'" +[ -n "$sc_ipv4_tcp_wmem" ] && log_eval "echo '$sc_ipv4_tcp_wmem' > '/proc/sys/net/ipv4/tcp_wmem'" [ -n "$sc_netdev_max_backlog" ] && log_eval "echo '$sc_netdev_max_backlog' > '/proc/sys/net/core/netdev_max_backlog'" [ -n "$sc_rmem_max" ] && log_eval "echo '$sc_rmem_max' > '/proc/sys/net/core/rmem_max'" [ -n "$sc_wmem_max" ] && log_eval "echo '$sc_wmem_max' > '/proc/sys/net/core/wmem_max'" +[ -n "$sc_sched_features" ] && log_eval "echo '$sc_sched_features' > '/sys/kernel/debug/sched_features'" exit 0 -- 2.23.0

1 0

[PATCH v3 compass-ci] web_backend.rb: fix : compare_groups_matrices needs arg suites_list
by Li Yuanchao 26 Nov '20

26 Nov '20

compare_groups_matrices needs three args, but now only get two, lost suites_list. divide method to pass rubocop limit Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 1f7224e..90c8a58 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -147,6 +147,20 @@ def get_es_must(params) must end +def _get_group_matrices(query, dimension) + result = es_query(query) + matrices, suites_list = combine_group_query_data(result, dimension) + while matrices.empty? + from += size + break if from > total + + query[:from] = from + result = es_query(query) + matrices, suites_list = combine_group_query_data(result, dimension) + end + [matrices, suites_list] +end + def get_groups_matrices(conditions, dimension, must, size, from) must += build_mutli_field_subquery_body(conditions) count_query = { query: { bool: { must: must } } } @@ -166,27 +180,17 @@ def get_groups_matrices(conditions, dimension, must, size, from) }] } - result = es_query(query) - matrices = combine_group_query_data(result, dimension) - while matrices.empty? - from += size - break if from > total - - query[:from] = from - result = es_query(query) - matrices = combine_group_query_data(result, dimension) - end - matrices + matrices, suites_list = _get_groups_matrices(query, dimension) end def get_compare_body(params) dimension, conditions = get_dimension_conditions(params) must = get_es_must(params) - groups_matrices = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) + groups_matrices, suites_list = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) if !groups_matrices || groups_matrices.empty? body = 'No Data.' else - body = compare_group_matrices(groups_matrices, { no_print: true }) + body = compare_group_matrices(groups_matrices, suites_list, { no_print: true }) body = 'No Difference.' if !body || body.empty? end return body -- 2.23.0

1 0

[PATCH v2 compass-ci] doc: about multi-device test
by Ren Wen 26 Nov '20

26 Nov '20

Signed-off-by: Ren Wen <15991987063(a)163.com> --- doc/multi-device-test.md | 109 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 doc/multi-device-test.md diff --git a/doc/multi-device-test.md b/doc/multi-device-test.md new file mode 100644 index 0000000..1be8c79 --- /dev/null +++ b/doc/multi-device-test.md @@ -0,0 +1,109 @@ +# Multi-device Test + +在进行需要多台机器的测试的时候, 需要搭建多台机器的环境. + +## When using multi-device + + * 当被测试程序需要启动一个守护进程时 + * 当执行测试时需要访问另外的机器时 + +## How to run a multi-device test + +在要提交的job.yaml中, 需要一个**cluster**字段, 它的值是一个多机环境的配 +置文件的文件名, 配置文件的内容可以参考: *lab-z9/cluster/*. +调度器会根据配置文件的内容将该job拆分为对应机器数量的job, 然后分别提交到 +相应的队列中. + +那么如何对指定的机器执行指定的测试呢? +在job.yaml中增加这样的内容: + +``` yaml +if role <role01> + <test_script01>: + <test_script02>: + +if role <role02> + <test_script03>: + <test_script04>: + +... +``` + +在多机的配置文件中, 会给每台机器赋予一个角色. 当每台机器取得自己队列中的 +job时, 按照自己的角色去执行相应角色所对应的脚本. + +比如说: 一台多机测试中的机器的角色为*role01*, 那么它会去寻找脚本 +*test_script01*, *test_script02*, 然后去执行. + +### how to find test script + +在lkp-tests的**daemon**, **tests**目录下查找这些脚本. +假设在**daemon**下找到了*test_script01*, 则将该脚本作为`start_daemon`的参 +数运行. 如果在**tests**下找到了*test_script01*, 则作为`run_test`的参数去 +运行. 如果没有找到相应的文件, 则按变量导出. + +### about `start_daemon` and `run_test` + +用于启动脚本的这2个函数位于**lkp-tests/lib/job.sh**. +要搞清楚多机的流程, 就得明白这两个函数是怎么去做的. + +#### the processes of `start_damon` + + 1. 运行在**daemon**文件夹下找到的脚本. 如果运行成功, 向调度器报告一个状态: + `started`, 调度器记录该机器的状态为: `start`; 执行失败, 则报告: + `failed`, 调度器记录为: `abort`, 然后退出执行. 状态可以查看redis的 + key: `HGETALL sched/cluster_state`. + 2. 向调度器报告一个状态: `write_state`, 并携带一些机器相关的参数: + * `node_roles`: 机器的角色 + * `ip`: 通过命令`hostname -I`取得的第一个值 + * `direct_macs`: 多机配置文件中该机器的**macs**字段 + * `direct_ips`: 调度器根据多机配置文件中的**ip0**字段配置的直连ip, 与 + 配置文件中**macs**的值一一对应 + 此时调度器将这些参数记录为每台机器相关的信息, 可以查看redis的key: + **sched/cluster_state**. + 3. 将**direct_ips**的值一一绑定到机器中的**direct_macs**的相应设备上. + 4. 向调度器报告一个状态: `wait_ready`, 调度器将其记录为: `ready`; 与此 + 同时, 调度器查看是否多机测试中的所有机器都准备好了. 如果每一台机器 + 都准备好了, 则返回`ready`; 如果其中一台的状态为`abort`, 则返回 + `abort`, 然后退出执行; 其余情况, 则循环查询, 直到前2种情形发生. + 5. 向调度器报告一个状态: `roles_ip`, 并将调度器的返回值导出为环境变量. + 比如当返回值为: `ip=ip01`时, 执行: `export ip=ip01`. + 6. 向调度器报告一个状态: `wait_finish`, 调度器将其记录为: `finish`; 与 + 此同时, 调度器查看是否多机测试中的所有机器的状态都为`finish`. 如果 + 都是`finish`, 则返回`finish`; 如果其中一台的状态为`abort`, 则返回 + `abort`, 然后退出执行; 其余情况, 则循环查询, 直到前2种情形发生. + 7. 执行完成. + +#### the processes of `run_test` + +步骤1,2,3,4同上述2,3,4,5. + + 1. 向调度器报告一个状态: `write_state`, 并携带一些机器相关的参数: + * `node_roles`: 机器的角色 + * `ip`: 通过命令`hostname -I`取得的第一个值 + * `direct_macs`: 多机配置文件中该机器的**macs**字段 + * `direct_ips`: 调度器根据多机配置文件中的**ip0**字段配置的直连ip, 与 + 配置文件中**macs**的值一一对应 + 此时调度器将这些参数记录为每台机器相关的信息, 可以查看redis的key: + **sched/cluster_state**. + 2. 将**direct_ips**的值一一绑定到机器中的**direct_macs**的相应设备上. + 3. 向调度器报告一个状态: `wait_ready`, 调度器将其记录为: `ready`; 与此 + 同时, 调度器查看是否多机测试中的所有机器都准备好了. 如果每一台机器 + 都准备好了, 则返回`ready`; 如果其中一台的状态为`abort`, 则返回 + `abort`, 然后退出执行; 其余情况, 则循环查询, 直到前2种情形发生. + 4. 向调度器报告一个状态: `roles_ip`, 并将调度器的返回值导出为环境变量. + 比如当返回值为: `ip=ip01`时, 执行: `export ip=ip01` + 5. 运行在**tests**文件夹下找到的脚本. 运行完成, 向调度器报告一个状态: + `finished`, 调度器记录为: `finish`; 执行失败, 则报告: `failed`, 调 + 度器记录为: `abort`, 然后退出执行. + 6. 执行完成. + +#### others + +`start_daemon`和`run_test`还会报告一个`abort`状态: 循环100次向调度器发 +起请求, 查看所有机器是否全部`ready`或`finish`, 如果循环结束仍然返回的是 +`retry`, 则报告该状态, 并结束执行. + +当调度器接受到与之相关的请求时, 会阻塞(或循环)查询这些机器的状态, 因此只 +需返回`ready`(`finish`)或`abort`. 所以不会触发循环100次请求这个过程, 因 +此不会报告`abort`这个状态. -- 2.23.0

1 0

[PATCH lkp-tests] distro/*: add exit when failed download packages
by Wang Yong 26 Nov '20

26 Nov '20

now when download packages failed, echo error message and $? is 0, so add exit 1 to get last_state Signed-off-by: Wang Yong <wangyong0117(a)qq.com> --- distro/aliyun | 1 + distro/archlinux | 1 + distro/debian | 1 + 3 files changed, 3 insertions(+) diff --git a/distro/aliyun b/distro/aliyun index 79656f0e4..598243d8e 100755 --- a/distro/aliyun +++ b/distro/aliyun @@ -126,6 +126,7 @@ pack_benchmark_deps() yum_pack else echo "failed to pack-deps $BM_NAME" >&2 + exit 1 fi done } diff --git a/distro/archlinux b/distro/archlinux index 3f8d62ebc..3942080a6 100755 --- a/distro/archlinux +++ b/distro/archlinux @@ -24,6 +24,7 @@ pack_benchmark_deps() pacman_pack else echo "failed to pack-deps $BM_NAME" >&2 + exit 1 fi done } diff --git a/distro/debian b/distro/debian index 13c4e983a..8821e3ddd 100755 --- a/distro/debian +++ b/distro/debian @@ -271,6 +271,7 @@ pack_benchmark_deps() echo "$PACKAGE_LIST" > $pack_to/.${BM_NAME}.packages else echo "failed to pack-deps $BM_NAME" >&2 + exit 1 fi rm -fr "$work_dir" -- 2.23.0

1 0

[PATCH compass-ci 1/4] answerback-email: add new options
by Luan Shengde 26 Nov '20

26 Nov '20

add option for answerback-email tool -n|--name name specify my_name for user when assigning account -l|--login Y|y|N|n enable/disable user login permission -u|--update enable update configuration for assigned account -u option enable update for: my_name my_ssh_pubkey generate sshkey on jumper reset user's login permission Signed-off-by: Luan Shengde <shdluan(a)163.com> --- container/assign-account/answerback-email.rb | 147 +++++++++++++++---- 1 file changed, 120 insertions(+), 27 deletions(-) diff --git a/container/assign-account/answerback-email.rb b/container/assign-account/answerback-email.rb index 5ddc451..bffba17 100755 --- a/container/assign-account/answerback-email.rb +++ b/container/assign-account/answerback-email.rb @@ -14,6 +14,7 @@ require 'optparse' require_relative '../defconfig' require_relative '../../lib/es_client' require_relative 'build-send-account-email' +require_relative 'build-update-email' names = Set.new %w[ JUMPER_HOST @@ -33,25 +34,61 @@ my_info = { 'my_email' => nil, 'my_name' => nil, 'my_commit_url' => nil, - 'my_uuid' => %x(uuidgen).chomp, - 'my_ssh_pubkey' => [], - 'gen_sshkey' => false + 'my_uuid' => nil, + 'my_login_name' => nil, + 'my_ssh_pubkey' => [] } -def init_info(mail_content, my_info) +# stdin_info is used to store infos added with option: +# -e email_address +# -n name +# -s pubkey_file +# email_info is used to store infos parsed from email_file with option: +# -f email_file +# my_info_es is used to store info read from ES when update user +# when use --update option +# stdin_info, email_info, my_info_es have different priority +# stdin_info > email_info > my_info_es +# when assigning account or update conf for account +# if they have the same key, my_info will use the value with higher priority. +# conf_info is used to store keys used to config the account +stdin_info = {} +email_info = {} +my_info_es = {} +conf_info = { + 'gen_sshkey' => false, + 'enable_login' => true, + 'is_update_account' => false +} + +def init_info(mail_content, email_info, my_info) my_info['my_email'] = mail_content.from[0] - my_info['my_name'] = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') + email_info['my_name'] = mail_content.From.unparsed_value.gsub(/ <[^<>]*>/, '') return if mail_content.attachments.empty? - my_info['my_ssh_pubkey'] << mail_content.attachments[0].body.decoded + email_info['new_email_pubkey'] = mail_content.attachments[0].body.decoded.strip +end + +def read_my_login_name(my_email, my_info_es) + my_account_info_str = %x(curl -XGET localhost:9200/accounts/_doc/#{my_email}) + my_account_info = YAML.safe_load my_account_info_str + message = "No such email found from the ES: #{my_email}" + raise message unless my_account_info['found'] + + my_info_es.update my_account_info['_source'] end options = OptionParser.new do |opts| - opts.banner = 'Usage: answerback-mail.rb [-e|--email email] ' - opts.banner += "[-s|--ssh-pubkey pub_key_file] [-f|--raw-email email_file] [-g|--gen-sshkey]\n" - opts.banner += " -e or -f is required\n" + opts.banner = 'Usage: answerback-mail.rb [-e|--email email] [-n|--name name] ' + opts.banner += "[-s|--ssh-pubkey pub_key_file] [-g|--gen-sshkey] [--login y|Y|n|N] [--update]\n" + opts.banner += ' answerback-mail.rb [-f|--raw-email email_file] ' + opts.banner += "[-g|--gen-sshkey] [--login y|Y|n|N] [--update]\n" + opts.banner += " -e|-f is required when applying account or updating account\n" + opts.banner += " -n is require when assigning account with -e\n" opts.banner += " -s is optional when use -e\n" - opts.banner += ' -g is optional, used to generate sshkey for user' + opts.banner += " -g is optional, used to generate sshkey for user\n" + opts.banner += " -u is require when updating an account\n" + opts.banner += ' -l is optional, used to enable/disable login permission' opts.separator '' opts.separator 'options:' @@ -60,18 +97,37 @@ options = OptionParser.new do |opts| my_info['my_email'] = email_address end + opts.on('-n name', '--name name', 'appoint name') do |name| + stdin_info['my_name'] = name + end + opts.on('-s pub_key_file', '--ssh-pubkey pub_key_file', \ 'ssh pub_key file, enable password-less login') do |pub_key_file| - my_info['my_ssh_pubkey'] << File.read(pub_key_file).chomp + stdin_info['new_ssh_pubkey'] = File.read(pub_key_file).strip end opts.on('-f email_file', '--raw-email email_file', 'email file') do |email_file| mail_content = Mail.read(email_file) - init_info(mail_content, my_info) + init_info(mail_content, email_info, my_info) end opts.on('-g', '--gen-sshkey', 'generate jumper ras public/private key and return pubkey') do - my_info['gen_sshkey'] = true + conf_info['gen_sshkey'] = true + end + + opts.on('-u', '--update', 'updata user account') do + read_my_login_name(my_info['my_email'], my_info_es) + conf_info['is_update_account'] = true + end + + opts.on('-l Y|y|N|n', '--login Y|y|N|n', 'enable/disable user login permission') do |value| + if %w[y Y].include? value + conf_info['enable_login'] = true + elsif %w[n N].include? value + conf_info['enable_login'] = false + else + raise 'invalid parameter, please use y|Y|n|N' + end end opts.on_tail('-h', '--help', 'show this message') do @@ -82,33 +138,70 @@ end options.parse!(ARGV) -def apply_account(my_info) - account_info_str = %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d '#{my_info.to_json}') +def apply_account(my_info, conf_info) + apply_info = {} + apply_info.update my_info + apply_info.update conf_info + + account_info_str = %x(curl -XGET '#{JUMPER_HOST}:#{JUMPER_PORT}/assign_account' -d '#{apply_info.to_json}') JSON.parse account_info_str end -def send_account(my_info) +def check_my_email(my_info) message = "No email address specified\n" - message += "use -e to add a email address\n" + message += "use -e to add a email address for applying account\n" message += 'or use -f to add a email file' - raise message if my_info['my_email'].nil? - account_info = apply_account(my_info) - my_info['my_login_name'] = account_info['my_login_name'] + raise message if my_info['my_email'].empty? +end +def build_my_info_from_input(my_info, email_info, my_info_es, stdin_info) + new_email_pubkey = email_info.delete 'new_email_pubkey' + new_stdin_pubkey = stdin_info.delete 'new_ssh_pubkey' + new_pubkey = new_stdin_pubkey || new_email_pubkey + + my_info.update my_info_es unless my_info_es.empty? + my_info.update email_info unless email_info.empty? + my_info.update stdin_info unless stdin_info.empty? + + return if new_pubkey.nil? + return if my_info['my_ssh_pubkey'].include? new_pubkey + + my_info['my_ssh_pubkey'].insert(0, new_pubkey) +end + +def build_my_info_from_account_info(my_info, account_info, conf_info) unless account_info['my_jumper_pubkey'].nil? - my_info['my_ssh_pubkey'] << account_info['my_jumper_pubkey'].chomp + return if my_info['my_ssh_pubkey'][-1] == account_info['my_jumper_pubkey'] + + my_info['my_ssh_pubkey'] << account_info['my_jumper_pubkey'] end - my_info.delete 'gen_sshkey' + my_info['my_login_name'] = account_info['my_login_name'] unless conf_info['is_update_account'] +end + +def send_account(my_info, conf_info, email_info, my_info_es, stdin_info) + check_my_email(my_info) + my_info['my_uuid'] = %x(uuidgen).chomp unless conf_info['is_update_account'] + build_my_info_from_input(my_info, email_info, my_info_es, stdin_info) + + message = 'No my_name found, please use -n to add one' + raise message if my_info['my_name'].nil? + + account_info = apply_account(my_info, conf_info) + build_my_info_from_account_info(my_info, account_info, conf_info) + store_account_info(my_info) - send_mail(my_info, account_info) + send_mail(my_info, account_info, conf_info) end -def send_mail(my_info, account_info) - message = build_message(my_info['my_email'], account_info) - +def send_mail(my_info, account_info, conf_info) + message = if conf_info['is_update_account'] + build_update_message(my_info['my_email'], account_info, conf_info) + else + build_message(my_info['my_email'], account_info) + end %x(curl -XPOST '#{SEND_MAIL_HOST}:#{SEND_MAIL_PORT}/send_mail_text' -d "#{message}") end @@ -117,4 +210,4 @@ def store_account_info(my_info) es.put_source_by_id(my_info['my_email'], my_info) end -send_account(my_info) +send_account(my_info, conf_info, email_info, my_info_es, stdin_info) -- 2.23.0

2 2

[PATCH compass-ci] web_backend.rb: fix : compare_groups_matrices needs arg suites_list
by Li Yuanchao 26 Nov '20

26 Nov '20

compare_groups_matrices needs three args, but now only get two, lost suites_list. divide method to pass rubocop limit Signed-off-by: Li Yuanchao <lyc163mail(a)163.com> --- src/lib/web_backend.rb | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/lib/web_backend.rb b/src/lib/web_backend.rb index 1f7224e..b5b06b4 100644 --- a/src/lib/web_backend.rb +++ b/src/lib/web_backend.rb @@ -147,6 +147,20 @@ def get_es_must(params) must end +def _get_group_matrices(query, dimension) + result = es_query(query) + matrices, suites_list = combine_group_query_data(result, dimension) + while matrices.empty? + from += size + break if from > total + + query[:from] = from + result = es_query(query) + matrices, suites_list = combine_group_query_data(result, dimension) + end + [matrices, suites_list] +end + def get_groups_matrices(conditions, dimension, must, size, from) must += build_mutli_field_subquery_body(conditions) count_query = { query: { bool: { must: must } } } @@ -166,27 +180,18 @@ def get_groups_matrices(conditions, dimension, must, size, from) }] } - result = es_query(query) - matrices = combine_group_query_data(result, dimension) - while matrices.empty? - from += size - break if from > total - - query[:from] = from - result = es_query(query) - matrices = combine_group_query_data(result, dimension) - end - matrices + matrices, suites_list = _get_groups_matrices(query, dimension) + [matrices, suites_list] end def get_compare_body(params) dimension, conditions = get_dimension_conditions(params) must = get_es_must(params) - groups_matrices = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) + groups_matrices, suites_list = get_groups_matrices(conditions, dimension, must, COMPARE_RECORDS_NUMBER, 0) if !groups_matrices || groups_matrices.empty? body = 'No Data.' else - body = compare_group_matrices(groups_matrices, { no_print: true }) + body = compare_group_matrices(groups_matrices, suites_list, { no_print: true }) body = 'No Difference.' if !body || body.empty? end return body -- 2.23.0

1 0