driver inclusion category: bugfix bugzilla: https://atomgit.com/openeuler/kernel/issues/9163 -------------------------------- This patch adopts differentiated performance counter sampling strategies, depending on how the BIOS exposes AMU counter values. - FFH-supported BIOS: If the BIOS exposes the counters in the FFH address space, they can be accessed highly efficiently. In this case, cppc_get_perf_ctrs() reads both counters in a single IPI on the target core (e.g., via cpc_read_ffh_fb_ctrs on ARM64). Since they are sampled together quickly, a short udelay(2) window is sufficient, completely avoiding unnecessary cross-CPU scheduling overhead. - Legacy / Non-FFH BIOS: If it is a legacy BIOS (counters exposed via PCC or system memory), we preserve the existing frequency reading behavior. The driver routes the sampling through smp_call_on_cpu() and explicitly waits 1ms using cond_resched() to amortize the cpc_read() latency and jitter under memory pressure. Fixes: 12f136b2134d ("cpufreq: CPPC: Keep the target core awake when reading its cpufreq rate") Signed-off-by: Pengjie Zhang <zhangpengjie2@huawei.com> Signed-off-by: Hongye Lin <linhongye@h-partners.com> --- drivers/acpi/cppc_acpi.c | 38 ++++++++++++------ drivers/cpufreq/cppc_cpufreq.c | 72 +++++++++++++++++++++++++++++++--- include/acpi/cppc_acpi.h | 5 +++ 3 files changed, 98 insertions(+), 17 deletions(-) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 33390f7e77ee..0ec2ea6a7682 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -1397,20 +1397,9 @@ EXPORT_SYMBOL_GPL(cppc_get_perf_caps); bool cppc_perf_ctrs_in_pcc_cpu(unsigned int cpu) { struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpu); - struct cpc_register_resource *ref_perf_reg; - - /* - * If reference perf register is not supported then we should use the - * nominal perf value - */ - ref_perf_reg = &cpc_desc->cpc_regs[REFERENCE_PERF]; - if (!CPC_SUPPORTED(ref_perf_reg)) - ref_perf_reg = &cpc_desc->cpc_regs[NOMINAL_PERF]; return CPC_IN_PCC(&cpc_desc->cpc_regs[DELIVERED_CTR]) || - CPC_IN_PCC(&cpc_desc->cpc_regs[REFERENCE_CTR]) || - CPC_IN_PCC(&cpc_desc->cpc_regs[CTR_WRAP_TIME]) || - CPC_IN_PCC(ref_perf_reg); + CPC_IN_PCC(&cpc_desc->cpc_regs[REFERENCE_CTR]); } EXPORT_SYMBOL_GPL(cppc_perf_ctrs_in_pcc_cpu); @@ -1463,6 +1452,31 @@ bool cppc_perf_ctrs_in_pcc(void) } EXPORT_SYMBOL_GPL(cppc_perf_ctrs_in_pcc); +/** + * cppc_fb_ctrs_in_ffh - Check if any feedback counters are in a FFH region. + * + * Return: true if any of the counters are in FFH regions, false otherwise + */ +bool cppc_fb_ctrs_in_ffh(void) +{ + int cpu; + + for_each_present_cpu(cpu) { + struct cpc_desc *cpc_desc; + + cpc_desc = per_cpu(cpc_desc_ptr, cpu); + if (!cpc_desc) + continue; + + if (CPC_IN_FFH(&cpc_desc->cpc_regs[DELIVERED_CTR]) || + CPC_IN_FFH(&cpc_desc->cpc_regs[REFERENCE_CTR])) + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(cppc_fb_ctrs_in_ffh); + /** * cppc_get_perf_ctrs - Read a CPU's performance feedback counters. * @cpunum: CPU from which to read counters. diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 9efaea2dbe83..f8b69ce42d74 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -27,6 +27,7 @@ #include <acpi/cppc_acpi.h> static bool boost_supported; +static bool ffh_supported; struct cppc_workaround_oem_info { char oem_id[ACPI_OEM_ID_SIZE + 1]; @@ -74,6 +75,12 @@ static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data, struct cppc_perf_fb_ctrs *fb_ctrs_t0, struct cppc_perf_fb_ctrs *fb_ctrs_t1); +struct fb_ctr_pair { + u32 cpu; + struct cppc_perf_fb_ctrs *fb_ctrs_t0; + struct cppc_perf_fb_ctrs *fb_ctrs_t1; +}; + /** * __cppc_scale_freq_tick - CPPC arch_freq_scale updater for frequency invariance * @cppc_fi: per-cpu CPPC FIE data. @@ -801,6 +808,47 @@ static int cppc_get_perf_ctrs_sample(int cpu, return cppc_get_perf_ctrs(cpu, fb_ctrs_t1); } +static int cppc_get_perf_ctrs_pair(void *val) +{ + struct fb_ctr_pair *fb_ctrs = val; + int cpu = fb_ctrs->cpu; + int ret; + ktime_t timeout; + + ret = cppc_get_perf_ctrs(cpu, fb_ctrs->fb_ctrs_t0); + if (ret) + return ret; + + if (likely(!in_atomic() && !irqs_disabled())) { + /* + * Set 1ms as sampling interval, but never schedule + * to the idle task to prevent the AMU counters from + * stopping working. + */ + timeout = ktime_add_ms(ktime_get(), 1); + while (ktime_before(ktime_get(), timeout)) + cond_resched(); + } else { + pr_warn_once("CPU%d: Get rate in atomic context", cpu); + udelay(2); /* 2usec delay between sampling */ + } + + return cppc_get_perf_ctrs(cpu, fb_ctrs->fb_ctrs_t1); +} + +static int cppc_get_perf_ctrs_on_cpu(unsigned int cpu, + struct cppc_perf_fb_ctrs *fb_ctrs_t0, + struct cppc_perf_fb_ctrs *fb_ctrs_t1) +{ + struct fb_ctr_pair fb_ctrs = { + .cpu = cpu, + .fb_ctrs_t0 = fb_ctrs_t0, + .fb_ctrs_t1 = fb_ctrs_t1, + }; + + return smp_call_on_cpu(cpu, cppc_get_perf_ctrs_pair, &fb_ctrs, false); +} + static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) { struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0}; @@ -816,12 +864,24 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) cpufreq_cpu_put(policy); - ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t0); - if (ret) - return 0; - udelay(2); /* 2usec delay between sampling */ + /* + * Pick the feedback-counter sampling strategy from how BIOS exposes + * the counters (ffh_supported is cached once in cppc_cpufreq_init): + * + * - FFH : cppc_get_perf_ctrs() reads both counters in a single IPI on + * the target core, so they are sampled together and a short udelay(2) + * window suffices. + * + * - non-FFH (PCC / system memory): run t0/window/t1 on the target core + * via smp_call_on_cpu() and wait 1ms with cond_resched() to amortize + * cpc_read() latency jitter under memory pressure (udelay(2) in atomic + * context). + */ + if (ffh_supported) + ret = cppc_get_perf_ctrs_sample(cpu, &fb_ctrs_t0, &fb_ctrs_t1); + else + ret = cppc_get_perf_ctrs_on_cpu(cpu, &fb_ctrs_t0, &fb_ctrs_t1); - ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t1); if (ret) { if (ret == -EFAULT) /* Any of the associated CPPC regs is 0. */ @@ -1072,6 +1132,8 @@ static int __init cppc_cpufreq_init(void) cppc_freq_invariance_init(); populate_efficiency_class(); + ffh_supported = cppc_fb_ctrs_in_ffh(); + ret = cpufreq_register_driver(&cppc_cpufreq_driver); if (ret) cppc_freq_invariance_exit(); diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index cdd5b3756fec..4d189d5c0abf 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -147,6 +147,7 @@ extern int cppc_set_enable(int cpu, bool enable); extern int cppc_get_perf_caps(int cpu, struct cppc_perf_caps *caps); extern bool cppc_perf_ctrs_in_pcc_cpu(unsigned int cpu); extern bool cppc_perf_ctrs_in_pcc(void); +extern bool cppc_fb_ctrs_in_ffh(void); extern unsigned int cppc_perf_to_khz(struct cppc_perf_caps *caps, unsigned int perf); extern unsigned int cppc_khz_to_perf(struct cppc_perf_caps *caps, unsigned int freq); extern bool acpi_cpc_valid(void); @@ -206,6 +207,10 @@ static inline bool cppc_perf_ctrs_in_pcc(void) { return false; } +static inline bool cppc_fb_ctrs_in_ffh(void) +{ + return false; +} static inline bool acpi_cpc_valid(void) { return false; -- 2.33.0