Signed-off-by: Zeng Heng <zengheng4@huawei.com> --- bw_dynamic.py | 143 ++++++++++++++++++++++++++++++++++++++++++++++ calc_cpu_usage.py | 27 +++++++++ draw_bw.py | 105 ++++++++++++++++++++++++++++++++++ draw_cpu.py | 34 +++++++++++ mb_calculate.py | 24 ++++++++ mv_tasks.py | 21 +++++++ pidcontroller.py | 35 ++++++++++++ 7 files changed, 389 insertions(+) create mode 100644 bw_dynamic.py create mode 100644 calc_cpu_usage.py create mode 100644 draw_bw.py create mode 100644 draw_cpu.py create mode 100644 mb_calculate.py create mode 100644 mv_tasks.py create mode 100644 pidcontroller.py diff --git a/bw_dynamic.py b/bw_dynamic.py new file mode 100644 index 000000000000..a5ca83ccdd90 --- /dev/null +++ b/bw_dynamic.py @@ -0,0 +1,143 @@ +import os, time, signal, json, sys +from pidcontroller import PIDController +from calc_cpu_usage import read_cpu_times, calculate_cpu_utilization + +numa_bw_limit = 80000 +target_percent = 60 + +bw_list = [] +set_list = [] +cpu_list = [] + +def read_bw(grp): + resctrl_mon_data_dir = '/sys/fs/resctrl/%s/mon_data/' % grp + mon_data_dirs = os.listdir(resctrl_mon_data_dir) + mon_MB_dirs = [dir for dir in mon_data_dirs if dir.startswith('mon_MB_')] + + vals = [] + for mb_dir in mon_MB_dirs: + with open(resctrl_mon_data_dir + mb_dir + '/mbm_total_bytes', 'r') as f: + line = f.readline() + vals.append(int(line.strip())) + + return vals + +def increase_bw(i, percent): + print("increase NUMA%d %d%%" % (i, percent)) + + resctrl_par_dir = '/sys/fs/resctrl/p1/schemata' + with open(resctrl_par_dir, 'r') as f: + lines = f.readlines() + + for line in lines: + if "MB:" in line: + config = line.split(":") + config = config[1].split(";") + origin_p = int(config[i].split("=")[1]) + + new_p = (origin_p + percent) + if new_p > 100: + new_p = 100 + elif new_p < 1: + new_p = 1 + + # if percent < 0: + # cnt = "MBHDL:%d=1\n" % i + # with open(resctrl_par_dir, 'w+') as f: + # f.write(cnt) + + cnt = "MB:%d=%d" % (i, new_p) + print("%s<=%d" % (cnt, origin_p)) + if new_p == origin_p: + return origin_p + + with open(resctrl_par_dir, 'w+') as f: + f.write("%s\n" % cnt) + + return origin_p + +def gain_numa_bw(pid_ctl, adjust_enable): + lc_val = read_bw(".") + be_val = read_bw("p1") + + numa_bw_state = [] + numa_set = [] + + for i in range(len(lc_val)): + bw_state = {"total_bw": {}, "lc_bw": {}, "be_bw": {}} + + bw_sum = lc_val[i] + be_val[i] + + bw_state["total_bw"] = bw_sum + bw_state["lc_bw"] = lc_val[i] + bw_state["be_bw"] = be_val[i] + numa_bw_state.append(bw_state) + + set_state = {"setting": {}, "delta": {}} + + if adjust_enable == True: + if bw_state["lc_bw"] < (0.04 * numa_bw_limit): + # enlarge BE load + diff = 5 + elif bw_state["lc_bw"] > (target_percent / 100 * numa_bw_limit): + # shutdown BE load + diff = -100 + else: + diff = pid_ctl[i].update(bw_sum * 100 // numa_bw_limit , 1) + else: + diff = 0 + + curr = increase_bw(i, diff) + set_state['setting'] = curr + set_state['delta'] = diff + + numa_set.append(set_state) + + bw_list.append(numa_bw_state) + set_list.append(numa_set) + return + +def gain_cpu(time1, time2): + usage = calculate_cpu_utilization(time1, time2) + print(f"CPU usage: {usage:.2f}%") + cpu_list.append(usage) + return + +def save_file(sig, frame): + with open(f"ctrlgrp_bw.data", 'w') as fl: + json.dump(bw_list, fl) + + with open(f"schemata_set.data", 'w') as fl: + json.dump(set_list, fl) + + with open(f"cpu_usage.data", 'w') as fl: + json.dump(cpu_list, fl) + + exit(0) + +if __name__ == "__main__": + signal.signal(signal.SIGINT, save_file) + + pid_ctl = [] + pid_ctl.append(PIDController(kp=1.0, ki=0.02, kd=0.05, set_point=target_percent)) + pid_ctl.append(PIDController(kp=1.0, ki=0.02, kd=0.05, set_point=target_percent)) + pid_ctl.append(PIDController(kp=1.0, ki=0.02, kd=0.05, set_point=target_percent)) + pid_ctl.append(PIDController(kp=1.0, ki=0.02, kd=0.05, set_point=target_percent)) + + enable = True + num_args = len(sys.argv) - 1 + if num_args == 1: + if sys.argv[1] == "mon": + enable = False + + time1 = read_cpu_times() + + while True: + gain_numa_bw(pid_ctl, enable) + + time2 = read_cpu_times() + gain_cpu(time1, time2) + time1 = time2 + + print("..........................") + time.sleep(1) diff --git a/calc_cpu_usage.py b/calc_cpu_usage.py new file mode 100644 index 000000000000..98fef7a2c1f5 --- /dev/null +++ b/calc_cpu_usage.py @@ -0,0 +1,27 @@ +import time + +def read_cpu_times(): + with open('/proc/stat', 'r') as f: + line = f.readline() + parts = line.split() + return list(map(int, parts[1:])) + +def calculate_cpu_utilization(times1, times2): + total1 = sum(times1) + idle1 = times1[3] + total2 = sum(times2) + idle2 = times2[3] + total_diff = total2 - total1 + idle_diff = idle2 - idle1 + return (total_diff - idle_diff) / total_diff * 100 + +# def gain_cpu(time1, time2): +# usage = calculate_cpu_utilization(time1, time2) +# print(f"CPU usage: {usage:.2f}%") +# return time2 + +# time1 = read_cpu_times() +# while True: +# time.sleep(1) +# time2 = read_cpu_times() +# time1 = gain_cpu(time1, time2) diff --git a/draw_bw.py b/draw_bw.py new file mode 100644 index 000000000000..17f28842cbcc --- /dev/null +++ b/draw_bw.py @@ -0,0 +1,105 @@ +import json +import matplotlib.pyplot as plt + +target_percent = 60 / 100 +bw_limit = target_percent * 80000 + +def get_all_bw(data_list, idx): + total = [] + lc = [] + be = [] + + for data in data_list: + total.append(data[idx]['total_bw']) + lc.append(data[idx]['lc_bw']) + be.append(data[idx]['be_bw']) + + return total, lc, be + +def get_all_set(set_list, idx): + setting = [] + delta = [] + + for data in set_list: + setting.append(data[idx]['setting']) + delta.append(data[idx]['delta']) + + return setting, delta + +def draw_data(bw_list, set_list, cpu_list): + fig = plt.figure() + x_arr = list(range(len(bw_list))) + + # numa_cnt = len(bw_list[0]) + # only draw numa1 + numa_cnt_display = 1 + numa_th = 1 + for cnt in range(numa_cnt_display): + ax1 = fig.add_subplot(numa_cnt_display * 2 + 1, 1, cnt * 2 + 1) + # total, lc, be = get_all_bw(bw_list, cnt) + total, lc, be = get_all_bw(bw_list, numa_th) + ref = [bw_limit] * len(bw_list) + + ax1.plot(x_arr, total, label='total bw') + ax1.plot(x_arr, lc, label='lc bw') + ax1.plot(x_arr, be, label='be bw') + ax1.plot(x_arr, ref, label='reference') + ax1.legend() + + ax2 = fig.add_subplot(numa_cnt_display * 2 + 1, 1, cnt * 2 + 2) + # setting, delta = get_all_set(set_list, cnt) + setting, delta = get_all_set(set_list, numa_th) + + ref = [0] * len(bw_list) + ax2.plot(x_arr, setting, label='setting') + ax2.plot(x_arr, delta, label='delta') + ax2.plot(x_arr, ref, label='reference') + ax2.legend() + + # ax3 = fig.add_subplot(5, 1, 3) + # total, lc, be = get_all_bw(bw_list, 1) + + # ax3.plot(x_arr, total, label='total bw') + # ax3.plot(x_arr, lc, label='lc bw') + # ax3.plot(x_arr, be, label='be bw') + # ax3.plot(x_arr, ref, label='reference') + # ax3.legend() + + # ax4 = fig.add_subplot(5, 1, 4) + # setting, delta = get_all_set(set_list, 1) + + # ax4.plot(x_arr, setting, label='setting') + # ax4.plot(x_arr, delta, label='delta') + # ax4.legend() + + ax5 = fig.add_subplot(numa_cnt_display * 2 + 1, 1, numa_cnt_display * 2 + 1) + + ax5.plot(x_arr, cpu_list, label='CPU usage') + ax5.legend() + + plt.tight_layout() + plt.show() + + return + +def get_data(file): + try: + with open(file, 'r') as f: + data_list = json.load(f) + except FileNotFoundError: + return None + + return data_list + +if __name__ == '__main__': + filename = 'ctrlgrp_bw.data' + bw_list = get_data(filename) + filename = 'schemata_set.data' + set_list = get_data(filename) + filename = 'cpu_usage.data' + cpu_list = get_data(filename) + + if bw_list and set_list: + draw_data(bw_list, set_list, cpu_list) + else: + print('FileNotFound') diff --git a/draw_cpu.py b/draw_cpu.py new file mode 100644 index 000000000000..02e5b0e48376 --- /dev/null +++ b/draw_cpu.py @@ -0,0 +1,34 @@ +import json +import matplotlib.pyplot as plt + +def draw_data(cpu_list): + fig = plt.figure() + x_arr = list(range(len(cpu_list))) + + ax1 = fig.add_subplot(1, 1, 1) + + ax1.plot(x_arr, cpu_list, label='CPU usage') + ax1.legend() + + plt.tight_layout() + plt.show() + + return + +def get_data(file): + try: + with open(file, 'r') as f: + cpu_list = json.load(f) + except FileNotFoundError: + return None + + return cpu_list + +if __name__ == '__main__': + filename = 'cpu_usage.data' + cpu_list = get_data(filename) + + if cpu_list: + draw_data(cpu_list) + else: + print('FileNotFound') diff --git a/mb_calculate.py b/mb_calculate.py new file mode 100644 index 000000000000..eb67cc85ec91 --- /dev/null +++ b/mb_calculate.py @@ -0,0 +1,24 @@ +import os, time, signal, json + +def read_bw(grp): + resctrl_mon_data_dir = '/sys/fs/resctrl/%s/mon_data/' % grp + mon_data_dirs = os.listdir(resctrl_mon_data_dir) + mon_MB_dirs = [dir for dir in mon_data_dirs if dir.startswith('mon_MB_')] + + vals = [] + for mb_dir in mon_MB_dirs: + with open(resctrl_mon_data_dir + mb_dir + '/mbm_total_bytes', 'r') as f: + line = f.readline() + vals.append(int(line.strip())) + + return vals + + +sum_val = [0] * 2 +while True: + for i in range(5): + vals = read_bw('.') + for i in range(len(sum_val)): + sum_val[i] += vals[i] + print(sum_val) + time.sleep(1) diff --git a/mv_tasks.py b/mv_tasks.py new file mode 100644 index 000000000000..27ad18f77e0c --- /dev/null +++ b/mv_tasks.py @@ -0,0 +1,21 @@ +import os, subprocess + +# proc = subprocess.Popen("ps -e -T | grep 'Executor task'", +proc = subprocess.Popen("jps | grep YarnCoarseGrainedExecutorBackend", + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + +pid_list = [] +cnt = proc.stdout.readlines() +for line in cnt: + line = line.decode("utf-8") + res = line.split() + pid_list.append(int(res[0])) + +print(",".join(map(str, pid_list))) + +for pid in pid_list: + ret = os.system("echo %d > /sys/fs/resctrl/hpri/tasks" % pid) + if ret: + print(ret) diff --git a/pidcontroller.py b/pidcontroller.py new file mode 100644 index 000000000000..4d03d2f28aa1 --- /dev/null +++ b/pidcontroller.py @@ -0,0 +1,35 @@ +class PIDController: + def __init__(self, kp, ki, kd, set_point): + self.kp = kp # 比例增益 + self.ki = ki # 积分增益 + self.kd = kd # 微分增益 + self.set_point = set_point # 设定值 + self.last_error = 0 # 上一次的误差 + self.integral = 0 # 积分项 + + self.max_output = 5 # 缓升 + self.min_output = -100 # 快降 + + def update(self, current_value, dt): + """ + 更新PID控制器的输出 + :param current_value: 当前系统的输出值 + :param dt: 时间间隔 + :return: 控制器的输出 + """ + error = self.set_point - current_value # 计算当前误差 + self.integral += error * dt # 更新积分项 + derivative = (error - self.last_error) / dt # 计算微分项 + + # 计算PID控制器的输出 + output = (self.kp * error) + (self.ki * self.integral) + (self.kd * derivative) + + self.last_error = error # 更新上一次的误差 + + # 限制输出范围 + if output > self.max_output: + output = self.max_output + elif output < self.min_output: + output = self.min_output + + return output -- 2.25.1