- Compass-ci - mailweb.openeuler.org

[PATCH v2 compass-ci] lib/es_jobs.rb: compact es-jobs output results
by Lu Kaiyi 15 Jan '21

15 Jan '21

[why] Reduce the nesting of output results [example] es-jobs submit_id=a7c2f144-aa64-4a23-a390-cfe5bca3b430 [output] {"kvcount.suite=iperf"=>2, "kvcount.os=openeuler"=>2, "kvcount.arch=aarch64"=>2, "kvcount.job_state=finished"=>2, "kvcount.tbox_group=vm-2p16g"=>2, "kvcount.submit_id=a7c2f144-aa64-4a23-a390-cfe5bca3b430"=>2, "sum.stats.kmsg.timestamp:last"=>2, "sum.stats.dmesg.timestamp:last"=>2, "raw.stats.kmsg.timestamp:last"=>[75.256233, 48.990905], "raw.stats.iperf.tcp.receiver.bps"=>[34017924155.510155, nil], "raw.stats.dmesg.timestamp:last"=>[75.256233, 48.990905], "raw.stats.iperf.tcp.sender.bps"=>[34073687935.01113, nil], "raw.stats.iperf.udp.bps"=>[nil, 1048573.3083402428], "avg.stats.iperf.tcp.receiver.bps"=>34017924155.510155, "avg.stats.iperf.tcp.sender.bps"=>34073687935.01113, "avg.stats.iperf.udp.bps"=>1048573.3083402428, "max.stats.iperf.tcp.receiver.bps"=>34017924155.510155, "max.stats.iperf.tcp.sender.bps"=>34073687935.01113, "max.stats.iperf.udp.bps"=>1048573.3083402428, "min.stats.iperf.tcp.receiver.bps"=>34017924155.510155, "min.stats.iperf.tcp.sender.bps"=>34073687935.01113, "min.stats.iperf.udp.bps"=>1048573.3083402428} Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_jobs.rb | 115 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 102 insertions(+), 13 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..754578d 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' +KEYWORD = %w[suite os arch category job_state tbox_group upstream_repo submit_id group_id] require "#{LKP_SRC}/lib/stats" require_relative './es_query' @@ -18,10 +19,10 @@ class ESJobs @refine_jobs = [] @jobs = {} @stats_level = { - 0 => 'stats.success', - 1 => 'stats.unknown', - 2 => 'stats.warning', - 3 => 'stats.has_error' + 0 => 'success', + 1 => 'unknown', + 2 => 'warning', + 3 => 'has_error' } set_defaults deal_jobs @@ -123,17 +124,105 @@ class ESJobs return 0 end - def output - result = { - 'stats.count' => @stats['stats.count'] - } + def query_jobs_from_es(items) + es = ESQuery.new(ES_HOST, ES_PORT) + result = es.multi_field_query items + jobs = result['hits']['hits'] + jobs.map! { |job| job['_source'] } + return jobs + end + + def get_all_metrics(jobs) + metrics = [] + jobs.each do |job| + stats = job['stats'] + next unless stats + + metrics.concat(stats.keys) + end + metrics.uniq! + end + + def initialize_result_hash(jobs, metrics) + result = {} + result['kvcount'] = {} + result['sum.stats'] = {} + result['raw.stats'] = {} + result['avg.stats'] = {} + result['max.stats'] = {} + result['min.stats'] = {} + metrics.each { |metric| result['raw.stats'][metric] = [] } + result + end + + def set_default_value(result, stats, metrics) + job_metrics = stats.keys + left_metrics = metrics - job_metrics + left_metrics.each { |metric| result['raw.stats'][metric] << nil } + + stats.each do |key, value| + result['raw.stats'][key] << value + end + end + + def kvcount(result, job) + KEYWORD.each do |keyword| + next unless job[keyword] - @stats.each do |key, value| - result[key] = value if @fields.include?(key) + result['kvcount']["#{keyword}=#{job[keyword]}"] ||= 0 + result['kvcount']["#{keyword}=#{job[keyword]}"] += 1 end + end - @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? - @result.merge!(result) - puts JSON.pretty_generate(@result) + def stats_count(result) + result['raw.stats'].each do |key, value| + if function_stat?(key) + result['sum.stats'][key] = value.compact.size + else + result['avg.stats'][key] = value.compact.sum / value.compact.size.to_f + result['max.stats'][key] = value.compact.max + result['min.stats'][key] = value.compact.min + end + end + end + + def query_jobs_state(jobs) + metrics = get_all_metrics(jobs) + result = initialize_result_hash(jobs, metrics) + jobs.each do |job| + stats = job['stats'] + next unless stats + + set_default_value(result, stats, metrics) + kvcount(result, job) + end + + stats_count(result) + result + end + + def compact_hash(prefix, result) + result.each do |key, value| + if prefix.empty? + prefix_key = "#{key}" + else + prefix_key = "#{prefix}.#{key}" + end + + if value.is_a? Hash + compact_hash(prefix_key, value) + else + @results[prefix_key] = value + end + end + end + + def output + jobs = query_jobs_from_es(@es_query) + @result = query_jobs_state(jobs) + @results = {} + compact_hash('', @result) + pp @results + #puts JSON.pretty_generate(@results) end end -- 2.23.0

2 1

[PATCH compass-ci] sbin/run_job: kill process after running qemu/docker
by Hu Xuejiao 15 Jan '21

15 Jan '21

[why] 1.host-info is adapted in physical machine, so it replaced to ebizzy.yaml 2.after running multi-docker and multi-qemu, these processes need to be deleted Signed-off-by: Hu XueJiao <1034502035(a)qq.com> --- sbin/run_job | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sbin/run_job b/sbin/run_job index 325fa9f..24d35cc 100755 --- a/sbin/run_job +++ b/sbin/run_job @@ -24,7 +24,7 @@ submit_one_job() submit_job() { - yaml_args=("$cci_makepkg_file" "host-info.yaml") + yaml_args=("$cci_makepkg_file" "ebizzy.yaml") for yaml in "${yaml_args[@]}" do submit_one_job & @@ -75,3 +75,6 @@ run_docker() run_qemu run_docker wait + +kill $(cat pid) +kill $(cat $CCI_SRC/providers/dc.pid) -- 2.23.0

2 1

[PATCH v4 compass-ci] tool/es-jobs: a new tool for summarizing jobs information
by Wu Zhende 15 Jan '21

15 Jan '21

Use es-jobs to summarizes information about a group of related jobs. [Example] es-jobs submit_id=xxx -f 'job_state,dmesg.timestamp:last' -r '-1' -s 'on_fail' means: 1. query the job whose submit_id is xxx 2. I want to get job_state,dmesg.timestamp:last 3. don't refine jobs 4. -s 'on_fail' to get data in stats which contain 'on_fail' [Output] { "job_state": [ "crystal.605716.failed" ], "dmesg.timestamp:last": [ 657.181408 ], "stats_filter_result": { "crystal.605716.openeuler_docker.\u001b[0mwordpress_build_on_fail": 1 }, "stats.count": { "stats.has_error": 1, "stats.has_error_jobs": [ "crystal.605716" ] } } Signed-off-by: Wu Zhende <wuzhende666(a)163.com> --- lib/es_jobs.rb | 139 +++++++++++++++++++++++++++++++++++++++++++++++++ sbin/es-jobs | 55 +++++++++++++++++++ 2 files changed, 194 insertions(+) create mode 100644 lib/es_jobs.rb create mode 100755 sbin/es-jobs diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb new file mode 100644 index 0000000..a90b1ba --- /dev/null +++ b/lib/es_jobs.rb @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: GPL-2.0-only +# frozen_string_literal: true + +LKP_SRC = ENV['LKP_SRC'] || '/c/lkp-tests' + +require "#{LKP_SRC}/lib/stats" +require_relative './es_query' + +# deal jobs search from es +class ESJobs + def initialize(es_query, my_refine = [], fields = [], stats_filter = []) + @es_query = es_query + @es = ESQuery.new(ES_HOST, ES_PORT) + @refine = my_refine + @fields = fields + @stats_filter = stats_filter + @stats_filter_result = {} + @refine_jobs = [] + @jobs = {} + @stats_level = { + 0 => 'stats.success', + 1 => 'stats.unknown', + 2 => 'stats.warning', + 3 => 'stats.has_error' + } + set_defaults + deal_jobs + end + + def set_defaults + query_result = @es.multi_field_query(@es_query) + query_result['hits']['hits'].each do |job| + @jobs[job['_id']] = job['_source'] + end + + @stats = { + 'stats.count' => Hash.new(0), + 'stats.sum' => Hash.new(0), + 'stats.avg' => Hash.new(0) + } + @result = {} + @fields.each do |field| + @result[field] = [] + end + end + + def add_result_fields(job, level) + return unless @refine.include?(level) || @refine.include?(-1) + + @refine_jobs << job['id'] + @fields.each do |field| + value = job[field] + if value + value = job['id'] + '.' + value if field == 'job_state' + @result[field] << value + end + + next unless job['stats'] + + @result[field] << job['stats'][field] if job['stats'][field] + end + end + + def deal_jobs + stats_count = Hash.new(0) + stats_jobs = {} + + @jobs.each do |job_id, job| + level = deal_stats(job) + add_result_fields(job, level) + + stat_key = @stats_level[level] + stat_jobs_key = stat_key + '_jobs' + + stats_count[stat_key] += 1 + stats_jobs[stat_jobs_key] ||= [] + stats_jobs[stat_jobs_key] << job_id + end + + @stats['stats.count'].merge!(stats_count) + @stats['stats.count'].merge!(stats_jobs) + end + + def deal_stats(job, level = 0) + return 1 unless job['stats'] + + job['stats'].each do |key, value| + match_stats_filter(key, value, job['id']) + calculate_stat(key, value) + level = get_stat_level(key, level) + end + return level + end + + def match_stats_filter(key, value, job_id) + @stats_filter.each do |filter| + next unless key.include?(filter) + + key = job_id + '.' + key + @stats_filter_result[key] = value + + break + end + end + + def calculate_stat(key, value) + if function_stat?(key) + return unless @fields.include?('stats.sum') + + @stats['stats.sum'][key] += value + else + return unless @fields.include?('stats.avg') + + @stats['stats.avg'][key] = (@stats['stats.avg'][key] + value) / 2 + end + end + + def get_stat_level(stat, level) + return level if level >= 3 + return 3 if stat.match(/error|fail/i) + return 2 if stat.match(/warn/i) + + return 0 + end + + def output + result = { + 'stats.count' => @stats['stats.count'] + } + + @stats.each do |key, value| + result[key] = value if @fields.include?(key) + end + + @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? + @result.merge!(result) + puts JSON.pretty_generate(@result) + end +end diff --git a/sbin/es-jobs b/sbin/es-jobs new file mode 100755 index 0000000..c055fe9 --- /dev/null +++ b/sbin/es-jobs @@ -0,0 +1,55 @@ +#!/usr/bin/env ruby +# SPDX-License-Identifier: MulanPSL-2.0+ +# Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. +# frozen_string_literal: true + +require 'optparse' +require 'json' + +require_relative '../lib/es_jobs' + +def parse_argv + items = {} + ARGV.each do |item| + key, value = item.split('=') + if key && value + value_list = value.split(',') + items[key] = value_list.length > 1 ? value_list : value + end + end + items +end +opt_refine = [-1] +opt_fields = [] +opt_stats_filter = [] +opt_parser = OptionParser.new do |opts| + opts.banner = 'Usage: es-jobs [options] search_key1=val1[,val2..] ..' + + opts.separator 'search_key can be submit_id, group_id' + opts.separator 'How to use -r' + opts.separator 'Like es-jobs submit_id=xxx -r "0,1,2,3"' + opts.separator '-1 means not refine, is the default value' + opts.separator ' 0 means stats.succes, refine the jobs whose test cases are all successfully executed' + opts.separator ' 1 means stats.unknown, refine the jobs without the stats' + opts.separator ' 2 means stats.warning, refine the jobs with warnings in tese cases' + opts.separator ' 3 means stats.has_error, refine the jobs with errors in tese cases' + + opts.on('-r fields', '--refine fields', 'refine jobs') do |fields| + opt_refine = fields.split(',').map(&:to_i) + end + + opts.on('-f fields', '--fields fields', 'fields you want to see') do |fields| + opt_fields = fields.split(',') + end + + opts.on('-s fields', '--stats-filter fields', 'return data contains fields in stats') do |fields| + opt_stats_filter = fields.split(',') + end +end + +opt_parser.parse!(ARGV) +items = parse_argv +raise 'Please enter a query' if items.empty? + +es_jobs = ESJobs.new(items, opt_refine, opt_fields, opt_stats_filter) +es_jobs.output -- 2.23.0

2 7

[PATCH compass-ci] use cd "$xxx" for safety reason
by Liu Yinsi 15 Jan '21

15 Jan '21

Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- container/docker2rootfs/common | 4 ++-- sparrow/4-docker/buildall | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/container/docker2rootfs/common b/container/docker2rootfs/common index e908079..c26bbc8 100755 --- a/container/docker2rootfs/common +++ b/container/docker2rootfs/common @@ -96,7 +96,7 @@ load_docker_img() cp_rootfs() { docker cp -a rootfs-docker:/tmp/$1 /tmp/ - cd $2 + cd "$2" zcat /tmp/$1 |cpio -idmv } @@ -148,7 +148,7 @@ create_links_vmlinuz_initrd() create_get_initrd echo "Creating links to initrd.lkp and vmlinuz..." - cd $ROOTFS_DIR + cd "$ROOTFS_DIR" ln -fs $ROOTFS_INITRD_LKP initrd.lkp cd $ROOTFS_DIR/boot ln -fs $ROOTFS_VMLINUZ vmlinuz diff --git a/sparrow/4-docker/buildall b/sparrow/4-docker/buildall index ca4406b..f24226c 100755 --- a/sparrow/4-docker/buildall +++ b/sparrow/4-docker/buildall @@ -30,7 +30,7 @@ do_one() [ -n "$LKP_SRC" ] && log_info "start build $container." mkdir $tmpdir/$container_name 2>/dev/null && ( - cd $container + cd "$container" [ "$container_name" == 'ssh-r' ] && exit docker images | grep -wq "$container_name" if [ "$?" == 0 ] && [ "$action" != "reboot" ]; then -- 2.23.0

2 4

[PATCH compass-ci] lib/es_jobs.rb: compact es-jobs output results
by Lu Kaiyi 15 Jan '21

15 Jan '21

[why] Reduce the nesting of output results [example] es-jobs submit_id=a7c2f144-aa64-4a23-a390-cfe5bca3b430 [output before] { "stats.count": { "stats.unknown": 2, "stats.success": 2, "stats.unknown_jobs": [ "crystal.613986", "crystal.613985" ], "stats.success_jobs": [ "crystal.597093", "crystal.597090" ] } } [output after] { "stats.count.unknown": 2, "stats.count.success": 2, "stats.count.unknown_jobs": [ "crystal.613986", "crystal.613985" ], "stats.count.success_jobs": [ "crystal.597093", "crystal.597090" ] } Signed-off-by: Lu Kaiyi <2392863668(a)qq.com> --- lib/es_jobs.rb | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/es_jobs.rb b/lib/es_jobs.rb index a90b1ba..c536fe3 100644 --- a/lib/es_jobs.rb +++ b/lib/es_jobs.rb @@ -18,10 +18,10 @@ class ESJobs @refine_jobs = [] @jobs = {} @stats_level = { - 0 => 'stats.success', - 1 => 'stats.unknown', - 2 => 'stats.warning', - 3 => 'stats.has_error' + 0 => 'success', + 1 => 'unknown', + 2 => 'warning', + 3 => 'has_error' } set_defaults deal_jobs @@ -123,6 +123,22 @@ class ESJobs return 0 end + def compact_hash(prefix, result) + result.each do |key, value| + if prefix.empty? + prefix_key = "#{key}" + else + prefix_key = "#{prefix}.#{key}" + end + + if value.is_a? Hash + compact_hash(prefix_key, value) + else + @results[prefix_key] = value + end + end + end + def output result = { 'stats.count' => @stats['stats.count'] @@ -134,6 +150,8 @@ class ESJobs @result['stats_filter_result'] = @stats_filter_result unless @stats_filter.empty? @result.merge!(result) - puts JSON.pretty_generate(@result) + @results = {} + compact_hash('', @result) + puts JSON.pretty_generate(@results) end end -- 2.23.0

2 3

[PATCH compass-ci] scheduler: fix "no job now" when consuming jobs within docker
by Ren Wen 14 Jan '21

14 Jan '21

[Error] "message":"Missing required job key: 'my_token'. [Why] The field 'ipxe_kernel_params' doesn't need to be initialized when consuming the job. [How] Remove 'ipxe_kernel_params' from 'initialized_keys'. Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/lib/job.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/job.cr b/src/lib/job.cr index 8377bab..1d25236 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -427,6 +427,7 @@ class Job "kernel_version", "kernel_uri", "kernel_params", + "ipxe_kernel_params", "linux_vmlinuz_path"] end -- 2.23.0

1 1

[PATCH v2 compass-ci] scheduler: fix "no job now" when consuming jobs within docker
by Ren Wen 14 Jan '21

14 Jan '21

[Error] "message":"Missing required job key: 'my_token'. [Why] The field 'ipxe_kernel_params' doesn't need to be initialized when consuming the job, otherwise, will do the account check. [How] Remove 'ipxe_kernel_params' from 'initialized_keys'. Signed-off-by: Ren Wen <15991987063(a)163.com> --- src/lib/job.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/job.cr b/src/lib/job.cr index 8377bab..1d25236 100644 --- a/src/lib/job.cr +++ b/src/lib/job.cr @@ -427,6 +427,7 @@ class Job "kernel_version", "kernel_uri", "kernel_params", + "ipxe_kernel_params", "linux_vmlinuz_path"] end -- 2.23.0

1 0

[PATCH v7 lkp-tests] lib/bootstrap: fix failed to get MAC and IP
by Liu Yinsi 14 Jan '21

14 Jan '21

[Error Msg] LKP: HOSTNAME vm-2p8g.2288hv5-2s44p-384g--b5-1573861, MAC , IP , kernel 5.9.0-5-riscv64 1, serial console /dev/ttyS0 [Root Cause] [ 93.815004] + network_ok [ 93.874456] ++ cat /sys/class/net/eth0//operstate [ 93.879278] + '[' up = up ']' [ 93.882356] + return 0 [ 93.941910] + export_ip_mac [ 94.018548] + has_cmd ip [ 94.022832] + command -v ip [ 94.029742] ++ awk '{print $5; exit}' [ 94.036137] ++ ip route get 1.2.3.4 [ 94.045622] ++ /usr/bin/mawk '{print $5; exit}' [ 94.056504] RTNETLINK answers: Network is unreachable [ 94.061418] + export PUB_NIC= [ 94.064587] + PUB_NIC= [Why] even network interfaces ready, but routing not, get ip by routing failed, so add a check export_ip_mac to ensure network is ok. [how] before: network interfaces "up"|"1" -> network is ok after: network interfaces "up"|"1" && export_ip_mac ok -> network is ok Signed-off-by: Liu Yinsi <liuyinsi(a)163.com> --- lib/bootstrap.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh index b04ea3dd4..2969544a6 100755 --- a/lib/bootstrap.sh +++ b/lib/bootstrap.sh @@ -91,8 +91,8 @@ network_ok() for i in /sys/class/net/*/ do [ "${i#*/lo/}" != "$i" ] && continue - [ "$(cat $i/operstate)" = 'up' ] && return 0 - [ "$(cat $i/carrier 2>/dev/null)" = '1' ] && return 0 + [ "$(cat $i/operstate)" = 'up' ] && export_ip_mac && return 0 + [ "$(cat $i/carrier 2>/dev/null)" = '1' ] && export_ip_mac && return 0 done is_clearlinux && { @@ -296,6 +296,8 @@ export_ip_mac() export PUB_IP=$(ifconfig $PUB_NIC | awk '/inet / {print $2}') fi + [ -n "$PUB_NIC" ] || return 1 + export PUB_MAC=$(cat /sys/class/net/$PUB_NIC/address) [ -n "$SCHED_HOST" ] && PUB_MAC=$(echo "$PUB_MAC" | tr : -) } @@ -746,7 +748,6 @@ boot_init() setup_network run_ntpdate - export_ip_mac announce_bootup -- 2.23.0

4 8

help star
by Wu Fengguang 14 Jan '21

14 Jan '21

Xueliang is our help star, thanks Xueliang! caoxl@crystal ~/code/todo/bin% ./help-star ../people/*/credit/2021-01-14.md 24.6 Wu Fengguang 16.2 Cao Xueliang 16.1 Yu Chuan 12.2 Luan Shengde 11.8 Wei Jihui 11.7 Liu Shaofei 10.2 Zhang Dewan 8.9 Xiao Shenwei 8.5 Zhang Yale 8.3 Liu Yinsi 7.6 Wu Zhende 7.2 Ren Wen 6.9 Wang Yong 6.5 Lu Weitao 5.2 Lin Jiaxin 5.0 Li Ping 4.7 Bai Jing 4.6 Xu Xijian 4.3 Cui Yili 2.4 Li Yuanchao 2.4 Wang Chenglong 2.2 Du Kaitian 1.3 Lu Kaiyi 1.1 Hu Xuejiao 0.0 Tong Qunfeng 0.0 Wang Shuai 0.0 Cui Xucui

1 0

[PATCH v5 lkp-tests] stats: add ansible_test stats script
by Zhang Yale 14 Jan '21

14 Jan '21

[Why] Output needs to be displayed in the stats.json file, so this output need to processed as a key-value pair. [input] PLAY RECAP ********************************************************************* localhost : ok=2 changed=0 unreachable=0 failed=1 skipped=0 rescued=0 ignored=0 playbook_run_on_fail: 1 [output] ansible_test.total.nr_ok: 2 ansible_test.total.nr_changed: 0 ansible_test.total.nr_failed: 1 ansible_test.total.nr_skipped: 0 ansible_test.total.nr_rescued: 0 ansible_test.total.nr_ignored: 0 playbook_run_on_fail: 1 Signed-off-by: Zhang Yale <ylzhangah(a)qq.com> --- stats/ansible_test | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 stats/ansible_test diff --git a/stats/ansible_test b/stats/ansible_test new file mode 100755 index 00000000..4c7d07b3 --- /dev/null +++ b/stats/ansible_test @@ -0,0 +1,15 @@ +#!/usr/bin/env ruby + +while (line = STDIN.gets) + case line.chomp! + when /ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)\s+skipped=(\d+)\s+rescued=(\d+)\s+ignored=(\d+)/ + puts "ansible_test.total.nr_ok: #{$1}" + puts "ansible_test.total.nr_changed: #{$2}" + puts "ansible_test.total.nr_failed: #{$4}" + puts "ansible_test.total.nr_skipped: #{$5}" + puts "ansible_test.total.nr_rescued: #{$6}" + puts "ansible_test.total.nr_ignored: #{$7}" + when /playbook_run_on_fail/ + puts line + end +end -- 2.23.0

3 4

2025

2024

2023

2022

2021

2020

Compass-ci