hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/ICWIH7 -------------------------------- For co-location scenarios, add a data-sampling script based on the resctrl file system, which monitors memory bandwidth and cache usage. The script also provides a dynamic bandwidth-adjustment policy that guarantees the response performance of latency-critical services by limiting best-effort services. Signed-off-by: Zeng Heng <zengheng4@huawei.com> --- tools/mpam/dynamic_adjust/bw_dynamic.py | 265 ++++++++++++++++++++ tools/mpam/dynamic_adjust/calc_cpu_usage.py | 25 ++ tools/mpam/dynamic_adjust/draw_bw.py | 154 ++++++++++++ tools/mpam/dynamic_adjust/draw_cpu.py | 41 +++ tools/mpam/dynamic_adjust/draw_lib.py | 38 +++ tools/mpam/dynamic_adjust/draw_llc.py | 126 ++++++++++ tools/mpam/dynamic_adjust/pid_controller.py | 45 ++++ 7 files changed, 694 insertions(+) create mode 100644 tools/mpam/dynamic_adjust/bw_dynamic.py create mode 100644 tools/mpam/dynamic_adjust/calc_cpu_usage.py create mode 100644 tools/mpam/dynamic_adjust/draw_bw.py create mode 100644 tools/mpam/dynamic_adjust/draw_cpu.py create mode 100644 tools/mpam/dynamic_adjust/draw_lib.py create mode 100644 tools/mpam/dynamic_adjust/draw_llc.py create mode 100644 tools/mpam/dynamic_adjust/pid_controller.py diff --git a/tools/mpam/dynamic_adjust/bw_dynamic.py b/tools/mpam/dynamic_adjust/bw_dynamic.py new file mode 100644 index 000000000000..9cbbab29e4da --- /dev/null +++ b/tools/mpam/dynamic_adjust/bw_dynamic.py @@ -0,0 +1,265 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Dynamic Bandwidth Adjustment +# +# Copyright (C) 2025, Technologies Co., Ltd. +# Author: Zeng Heng <zengheng4@huawei.com> + +import os, time, signal, json, sys +import argparse +import subprocess +from pid_controller import PID_Controller +from calc_cpu_usage import read_cpu_times, calculate_cpu_utilization + +__version__ = "1.0.0" + +numa_bw_limit = 140000 + +bw_list = [] +llc_list = [] +set_list = [] +cpu_list = [] + +def read_bw(grp): + grps_val = {} + + for g in grp: + vals = [] + + resctrl_mon_data_dir = '/sys/fs/resctrl/%s/mon_data/' % g + mon_data_dirs = os.listdir(resctrl_mon_data_dir) + mon_MB_dirs = [dir for dir in mon_data_dirs if dir.startswith('mon_MB_')] + mon_MB_dirs.sort() + + for mb_dir in mon_MB_dirs: + with open(resctrl_mon_data_dir + mb_dir + '/mbm_total_bytes', 'r') as f: + # with open(resctrl_mon_data_dir + mb_dir, 'r') as f: + line = f.readline() + vals.append(int(line.strip())) + + grps_val[g] = vals + + totals = [] + for i in range(len(mon_MB_dirs)): + total = 0 + for g in grp: + total += grps_val[g][i] + totals.append(total) + + grps_val["__grp_total__"] = totals + return grps_val + +def increase_bw(be_grp, i, percent): + resctrl_par_dir = '/sys/fs/resctrl/%s/schemata' % be_grp[0] + with open(resctrl_par_dir, 'r') as f: + lines = f.readlines() + + for line in lines: + if "MB:" in line: + config = line.split(":") + config = config[1].split(";") + origin_p = int(config[i].split("=")[1]) + + new_p = (origin_p + percent) + if new_p > 100: + new_p = 100 + elif new_p < 1: + new_p = 1 + + for g in be_grp: + print("%s NUMA%d MB %d%%->%d%% delta %d%%" % (g, i, origin_p, new_p, percent)) + + cnt = "MB:%d=%d" % (i, new_p) + if new_p == origin_p: + continue + + resctrl_par_dir = '/sys/fs/resctrl/%s/schemata' % g + with open(resctrl_par_dir, 'w+') as f: + f.write("%s\n" % cnt) + + return origin_p + +def gain_numa_bw(lc_grp, be_grp, pid_ctl, adjust_enable, target_percent): + lc_val = read_bw(lc_grp) + be_val = read_bw(be_grp) + + bw_state = {} + bw_state["lc_bw"] = lc_val + bw_state["be_bw"] = be_val + bw_state["total_bw"] = [] + + numa_set = [] + + for i in range(len(lc_val["__grp_total__"])): + bw_state["total_bw"].append(lc_val["__grp_total__"][i] + be_val["__grp_total__"][i]) + + set_state = {} + + if adjust_enable == True: + if bw_state["lc_bw"]["__grp_total__"][i] < (0.04 * numa_bw_limit): + # enlarge BE load + diff = pid_ctl[i].max_output + elif bw_state["lc_bw"]["__grp_total__"][i] > (target_percent * numa_bw_limit / 100): + # shutdown BE load + diff = pid_ctl[i].min_output + else: + diff = pid_ctl[i].update(bw_state["total_bw"][i] * 100 // numa_bw_limit , 1) + else: + diff = 0 + + curr = increase_bw(be_grp, i, diff) + set_state['setting'] = curr + numa_set.append(set_state) + + bw_list.append(bw_state) + set_list.append(numa_set) + return + +def read_llc(grp): + grps_val = {} + + for g in grp: + vals = [] + + resctrl_mon_data_dir = '/sys/fs/resctrl/%s/mon_data/' % g + mon_data_dirs = os.listdir(resctrl_mon_data_dir) + mon_llc_dirs = [dir for dir in mon_data_dirs if dir.startswith('mon_L3_')] + mon_llc_dirs.sort() + + for mb_dir in mon_llc_dirs: + with open(resctrl_mon_data_dir + mb_dir + '/llc_occupancy', 'r') as f: + # with open(resctrl_mon_data_dir + mb_dir, 'r') as f: + line = f.readline() + vals.append(int(line.strip())) + + grps_val[g] = vals + + totals = [] + for i in range(len(mon_llc_dirs)): + total = 0 + for g in grp: + total += grps_val[g][i] + totals.append(total) + + grps_val["__grp_total__"] = totals + return grps_val + +def gain_numa_llc(lc_grp, be_grp): + lc_llc = read_llc(lc_grp) + be_llc = read_llc(be_grp) + + llc_state = {} + llc_state["lc_llc"] = lc_llc + llc_state["be_llc"] = be_llc + + llc_list.append(llc_state) + + return + +def gain_cpu(time1, time2): + usage = calculate_cpu_utilization(time1, time2) + print(f"CPU load: {usage:.2f}%") + cpu_list.append(usage) + return + +def save_file(sig, frame): + with open(f"ctrlgrp_bw.data", 'w') as fl: + json.dump(bw_list, fl) + + with open(f"schemata_set.data", 'w') as fl: + json.dump(set_list, fl) + + with open(f"ctrlgrp_llc.data", 'w') as fl: + json.dump(llc_list, fl) + + with open(f"cpu_usage.data", 'w') as fl: + json.dump(cpu_list, fl) + + exit(0) + +def get_numa_count(): + try: + output = subprocess.check_output(['lscpu'], text=True) + for line in output.splitlines(): + if line.startswith('NUMA node(s):'): + return int(line.split(':')[1].strip()) + + except Exception as e: + print('Error:', e) + return None + +def flatten_comma_separated(raw_list): + out = [] + for item in raw_list: + out.extend([x.strip() for x in item.split(",") if x.strip()]) + return out + +def parse_args(): + parser = argparse.ArgumentParser( + description="Example: read execution time and isolation percentage from CLI") + + parser.add_argument("-t", "--time", required=False, type=int, + help="Expected execution time in seconds, positive integer") + parser.add_argument("-i", "--isolation", required=False, type=int, + help="Isolation percentage, integer between 1 and 100") + parser.add_argument("-l", "--lc", action="append", default=["."], + help="Comma-separated or repeated latency-critical group names.") + parser.add_argument("-b", "--be", action="append", default=[], + help="Comma-separated or repeated best-effort group names.") + parser.add_argument("-v", "--version", action="version", + version=f"%(prog)s {__version__}") + + args = parser.parse_args() + + if args.time and args.time <= 0: + sys.exit("ERROR: execution time must be > 0 seconds") + + if args.isolation and not 1 <= args.isolation <= 100: + sys.exit("ERROR: isolation percentage must be between 0 and 100") + + if not args.be or not args.lc: + sys.exit("ERROR: at least one -l or -b group name must be provided") + + args.lc = flatten_comma_separated(args.lc) + args.be = flatten_comma_separated(args.be) + + return args + +if __name__ == "__main__": + signal.signal(signal.SIGINT, save_file) + + args = parse_args() + exec_time = args.time + target_percent = args.isolation + lc_group = args.lc + be_group = args.be + + adj_enable = False + if target_percent: + adj_enable = True + + pid_ctl = [] + numa_cnt = get_numa_count() + for i in range(numa_cnt): + pid_ctl.append(PID_Controller(kp=1.0, ki=0.02, kd=0.05, + set_point=target_percent)) + + now = 0 + time1 = read_cpu_times() + + while True: + gain_numa_bw(lc_group, be_group, pid_ctl, adj_enable, target_percent) + gain_numa_llc(lc_group, be_group) + + time2 = read_cpu_times() + gain_cpu(time1, time2) + time1 = time2 + + print("..........................") + time.sleep(1) + now += 1 + + if exec_time and exec_time <= now: + break + + save_file(None, None) diff --git a/tools/mpam/dynamic_adjust/calc_cpu_usage.py b/tools/mpam/dynamic_adjust/calc_cpu_usage.py new file mode 100644 index 000000000000..75f591f4443a --- /dev/null +++ b/tools/mpam/dynamic_adjust/calc_cpu_usage.py @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# CPU Load Sampling for Dynamic Bandwidth Adjustment +# +# Copyright (C) 2025, Technologies Co., Ltd. +# Author: Zeng Heng <zengheng4@huawei.com> + +import time + +def read_cpu_times(): + with open('/proc/stat', 'r') as f: + line = f.readline() + + parts = line.split() + return list(map(int, parts[1:])) + +def calculate_cpu_utilization(times1, times2): + total1 = sum(times1) + idle1 = times1[3] + total2 = sum(times2) + idle2 = times2[3] + + total_diff = total2 - total1 + idle_diff = idle2 - idle1 + return (total_diff - idle_diff) * 100 / total_diff diff --git a/tools/mpam/dynamic_adjust/draw_bw.py b/tools/mpam/dynamic_adjust/draw_bw.py new file mode 100644 index 000000000000..c84d9d2047eb --- /dev/null +++ b/tools/mpam/dynamic_adjust/draw_bw.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Memory Bandwidth Analysis for Dynamic Bandwidth Adjustment +# +# Copyright (C) 2025, Technologies Co., Ltd. +# Author: Zeng Heng <zengheng4@huawei.com> + +import json +import matplotlib.pyplot as plt +from draw_lib import parse_args, get_data + +bw_max_limit = 140000 + +def get_lc_bw(data_list, idx): + table = {} + + table['total_bw'] = [] + table['lc_bw'] = [] + + for k, v in data_list[0]['lc_bw'].items(): + if k != "__grp_total__": + table[k] = [] + + for data in data_list: + table['total_bw'].append(data['total_bw'][idx]) + table['lc_bw'].append(data['lc_bw']["__grp_total__"][idx]) + + for k, v in data['lc_bw'].items(): + if k != "__grp_total__": + table[k].append(v[idx]) + + return table + +def get_be_bw(data_list, idx): + table = {} + + table['total_bw'] = [] + table['be_bw'] = [] + + for k, v in data_list[0]['be_bw'].items(): + if k != "__grp_total__": + table[k] = [] + + for data in data_list: + table['total_bw'].append(data['total_bw'][idx]) + table['be_bw'].append(data['be_bw']["__grp_total__"][idx]) + + for k, v in data['be_bw'].items(): + if k != "__grp_total__": + table[k].append(v[idx]) + + return table + +def get_all_bw(data_list, idx): + table = {} + + table['total_bw'] = [] + table['lc_bw'] = [] + table['be_bw'] = [] + + for k, v in data_list[0]['lc_bw'].items(): + if k != "__grp_total__": + table[k] = [] + for k, v in data_list[0]['be_bw'].items(): + if k != "__grp_total__": + table[k] = [] + + for data in data_list: + table['total_bw'].append(data['total_bw'][idx]) + table['lc_bw'].append(data['lc_bw']["__grp_total__"][idx]) + table['be_bw'].append(data['be_bw']["__grp_total__"][idx]) + + for k, v in data['lc_bw'].items(): + if k != "__grp_total__": + table[k].append(v[idx]) + + for k, v in data['be_bw'].items(): + if k != "__grp_total__": + table[k].append(v[idx]) + + return table + +def get_all_set(set_list, idx): + setting = [] + + for data in set_list: + setting.append(data[idx]['setting']) + + return setting + +def draw_data(bw_list, set_list, cpu_list, numa_th, percent): + fig = plt.figure() + x_arr = list(range(len(bw_list))) + ref = [bw_max_limit * percent / 100] * len(bw_list) + + ax1 = fig.add_subplot(3, 1, 1) + table = get_lc_bw(bw_list, numa_th) + + for k, v in table.items(): + ax1.plot(x_arr, v, label=k) + ax1.plot(x_arr, ref, label='Reference') + + ax1.set_ylabel(f"LC Mem bandwidth (MB)") + ax1.set_xlabel(f"Time (s)") + ax1.legend() + + ax2 = fig.add_subplot(3, 1, 2) + table = get_be_bw(bw_list, numa_th) + + for k, v in table.items(): + ax2.plot(x_arr, v, label=k) + ax2.plot(x_arr, ref, label='Reference') + + ax2.set_ylabel(f"BE Mem bandwidth (MB)") + ax2.set_xlabel(f"Time (s)") + ax2.legend() + + ax3 = fig.add_subplot(3, 1, 3) + setting = get_all_set(set_list, numa_th) + ax3.plot(x_arr, setting, label='MBMAX') + ax3.plot(x_arr, cpu_list, label='CPU Load') + + ax3.set_ylabel(f"Percent(%)") + ax3.set_xlabel(f"Time (s)") + ax3.legend() + + plt.tight_layout() + plt.show() + + return + +if __name__ == '__main__': + start, percent = parse_args() + if not start: + start = 0 + if not percent: + percent = 0 + + filename = 'ctrlgrp_bw.data' + bw_list = get_data(filename) + filename = 'schemata_set.data' + set_list = get_data(filename) + filename = 'cpu_usage.data' + cpu_list = get_data(filename) + + if not bw_list or not set_list: + print('FileNotFound') + + if len(bw_list) != len(cpu_list): + # length sync + bw_list.pop() + set_list.pop() + + draw_data(bw_list, set_list, cpu_list, start, percent) diff --git a/tools/mpam/dynamic_adjust/draw_cpu.py b/tools/mpam/dynamic_adjust/draw_cpu.py new file mode 100644 index 000000000000..5e9d8a738545 --- /dev/null +++ b/tools/mpam/dynamic_adjust/draw_cpu.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# CPU Load Analysis for Dynamic Bandwidth Adjustment +# +# Copyright (C) 2025, Technologies Co., Ltd. +# Author: Zeng Heng <zengheng4@huawei.com> + +import json +import matplotlib.pyplot as plt + +def draw_data(cpu_list): + fig = plt.figure() + x_arr = list(range(len(cpu_list))) + + ax1 = fig.add_subplot(1, 1, 1) + + ax1.plot(x_arr, cpu_list, label='CPU usage') + ax1.legend() + + plt.tight_layout() + plt.show() + + return + +def get_data(file): + try: + with open(file, 'r') as f: + cpu_list = json.load(f) + except FileNotFoundError: + return None + + return cpu_list + +if __name__ == '__main__': + filename = 'cpu_usage.data' + cpu_list = get_data(filename) + + if cpu_list: + draw_data(cpu_list) + else: + print('FileNotFound') diff --git a/tools/mpam/dynamic_adjust/draw_lib.py b/tools/mpam/dynamic_adjust/draw_lib.py new file mode 100644 index 000000000000..8a11c13f8ec9 --- /dev/null +++ b/tools/mpam/dynamic_adjust/draw_lib.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Analysis Lib for Dynamic Bandwidth Adjustment +# +# Copyright (C) 2025, Technologies Co., Ltd. +# Author: Zeng Heng <zengheng4@huawei.com> + +import sys, json, argparse + +__version__ = "1.0.0" + +def get_data(file): + try: + with open(file, 'r') as f: + data_list = json.load(f) + except FileNotFoundError: + return None + + return data_list + +def parse_args(): + parser = argparse.ArgumentParser(description="Display selected NUMA's Analysis") + parser.add_argument("-s", "--start", type=int, required=False, + help="Starting NUMA index (0-based)") + parser.add_argument("-i", "--isolation", required=False, type=int, + help="Isolation target percentage, integer between 1 and 100") + parser.add_argument("-v", "--version", action="version", + version=f"%(prog)s {__version__}") + + args = parser.parse_args() + + if args.start and args.start < 0: + sys.exit("ERROR: --index must be non-negative") + + if args.isolation and not 1 <= args.isolation <= 100: + sys.exit("ERROR: isolation percentage must be between 0 and 100") + + return args.start, args.isolation diff --git a/tools/mpam/dynamic_adjust/draw_llc.py b/tools/mpam/dynamic_adjust/draw_llc.py new file mode 100644 index 000000000000..9a6f112fcc2f --- /dev/null +++ b/tools/mpam/dynamic_adjust/draw_llc.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Cache Storage Analysis for Dynamic Bandwidth Adjustment +# +# Copyright (C) 2025, Technologies Co., Ltd. +# Author: Zeng Heng <zengheng4@huawei.com> + +import matplotlib.pyplot as plt +from draw_lib import parse_args, get_data + +def get_lc_llc(data_list, idx): + table = {} + + table['lc_llc'] = [] + + for k, v in data_list[0]['lc_llc'].items(): + if k != "__grp_total__": + table[k] = [] + + for data in data_list: + table['lc_llc'].append(data['lc_llc']["__grp_total__"][idx]) + for k, v in data['lc_llc'].items(): + if k != "__grp_total__": + table[k].append(v[idx]) + + return table + +def get_be_llc(data_list, idx): + table = {} + + table['be_llc'] = [] + + for k, v in data_list[0]['be_llc'].items(): + if k != "__grp_total__": + table[k] = [] + + for data in data_list: + table['be_llc'].append(data['be_llc']["__grp_total__"][idx]) + for k, v in data['be_llc'].items(): + if k != "__grp_total__": + table[k].append(v[idx]) + + return table + +def get_all_llc(data_list, idx): + table = {} + + table['lc_llc'] = [] + table['be_llc'] = [] + + for k, v in data_list[0]['lc_llc'].items(): + if k != "__grp_total__": + table[k] = [] + for k, v in data_list[0]['be_llc'].items(): + if k != "__grp_total__": + table[k] = [] + + for data in data_list: + table['lc_llc'].append(data['lc_llc']["__grp_total__"][idx]) + table['be_llc'].append(data['be_llc']["__grp_total__"][idx]) + + for k, v in data['lc_llc'].items(): + if k != "__grp_total__": + table[k].append(v[idx]) + + for k, v in data['be_llc'].items(): + if k != "__grp_total__": + table[k].append(v[idx]) + + return table + +def draw_data(llc_list, cpu_list, numa_th): + fig = plt.figure() + x_arr = list(range(len(llc_list))) + + ax1 = fig.add_subplot(3, 1, 1) + table = get_lc_llc(llc_list, numa_th) + + for k, v in table.items(): + ax1.plot(x_arr, v, label=k) + + ax1.set_ylabel(f"LC L3 Cache (KB)") + ax1.set_xlabel(f"Time (s)") + ax1.legend() + + ax2 = fig.add_subplot(3, 1, 2) + table = get_be_llc(llc_list, numa_th) + + for k, v in table.items(): + ax2.plot(x_arr, v, label=k) + + ax2.set_ylabel(f"BE L3 Cache (KB)") + ax2.set_xlabel(f"Time (s)") + ax2.legend() + + ax3 = fig.add_subplot(3, 1, 3) + ax3.plot(x_arr, cpu_list, label='CPU usage') + + ax3.set_ylabel(f"CPU Load (%)") + ax3.set_xlabel(f"Time (s)") + + ax3.legend() + + plt.tight_layout() + plt.show() + + return + +if __name__ == '__main__': + start, p = parse_args() + if not start: + start = 0 + + filename = 'ctrlgrp_llc.data' + llc_list = get_data(filename) + filename = 'cpu_usage.data' + cpu_list = get_data(filename) + + if not llc_list: + print('FileNotFound') + + if len(llc_list) != len(cpu_list): + # length sync + llc_list.pop() + + draw_data(llc_list, cpu_list, start) diff --git a/tools/mpam/dynamic_adjust/pid_controller.py b/tools/mpam/dynamic_adjust/pid_controller.py new file mode 100644 index 000000000000..db0533273b8f --- /dev/null +++ b/tools/mpam/dynamic_adjust/pid_controller.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# PID Algorithm Controller for Dynamic Bandwidth Adjustment +# +# Copyright (C) 2025, Technologies Co., Ltd. +# Author: Zeng Heng <zengheng4@huawei.com> + +class PID_Controller: + def __init__(self, kp, ki, kd, set_point): + self.kp = kp # Proportional Gain + self.ki = ki # Integral Gain + self.kd = kd # Derivative Gain + self.set_point = set_point + self.last_error = 0 + self.integral = 0 # Integral Term + + self.max_output = 5 # rise slowly + self.min_output = -100 # fall quickly + + def update(self, current_value, dt): + """ + Update the PID controller's output + :param current_value: Current system output value + :param dt: Time interval + :return: Controller output + """ + error = self.set_point - current_value # Calculate current error + self.integral += error * dt # Update Integral Term + if self.integral < -100: + self.integral = -100 + elif self.integral > 100: + self.integral = 100 + derivative = (error - self.last_error) / dt # Calculate Derivative Term + + output = (self.kp * error) + (self.ki * self.integral) + (self.kd * derivative) + + self.last_error = error + + # Limit the output range + if output > self.max_output: + output = self.max_output + elif output < self.min_output: + output = self.min_output + + return output -- 2.25.1