From: James Morse james.morse@arm.com
hulk inclusion category: feature bugzilla: 34278 CVE: NA
-------------------------------------------------
Once we know where all the devices are, we can register cpu hotplug callbacks to probe the devices each CPU can access. Once we've probed all the devices, we can enable MPAM.
As a first step, we learn whether the MSC supports MPAMv1.x, and update our system wide view of the commonly supported partid/pmg range.
As noted in the ACPI code, we learn the cache affinities as CPUs come online. This ensures the data we export via resctrl matches the data cacheinfo exports via sysfs.
[Wang ShaoBo: version adaption and few changes in mpam_sysprops_prop]
Signed-off-by: James Morse james.morse@arm.com Link: http://www.linux-arm.org/git?p=linux-jm.git;a=patch;h=b91f071ae923de34a0b0f7... Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Reviewed-by: Xiongfeng Wang wangxiongfeng2@huawei.com Reviewed-by: Cheng Jian cj.chengjian@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com Signed-off-by: Cheng Jian cj.chengjian@huawei.com --- arch/arm64/include/asm/mpam.h | 3 + arch/arm64/kernel/mpam/mpam_device.c | 231 ++++++++++++++++++++++++++- arch/arm64/kernel/mpam/mpam_device.h | 7 + 3 files changed, 240 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/mpam.h b/arch/arm64/include/asm/mpam.h index de00c141065f..b83f940e0432 100644 --- a/arch/arm64/include/asm/mpam.h +++ b/arch/arm64/include/asm/mpam.h @@ -8,6 +8,7 @@
#include <linux/seq_buf.h> #include <linux/seq_file.h> +#include <linux/resctrlfs.h>
/* MPAM register */ #define SYS_MPAM0_EL1 sys_reg(3, 0, 10, 5, 1) @@ -97,9 +98,11 @@ */ #define VPMR_MAX_BITS (3) #define PARTID_MAX_SHIFT (0) +#define PARTID_MAX_MASK (MPAM_MASK(PARTID_BITS) << PARTID_MAX_SHIFT) #define HAS_HCR_SHIFT (PARTID_MAX_SHIFT + PARTID_BITS + 1) #define VPMR_MAX_SHIFT (HAS_HCR_SHIFT + 1) #define PMG_MAX_SHIFT (VPMR_MAX_SHIFT + VPMR_MAX_BITS + 11) +#define PMG_MAX_MASK (MPAM_MASK(PMG_BITS) << PMG_MAX_SHIFT) #define VPMR_MASK MPAM_MASK(VPMR_MAX_BITS)
/* diff --git a/arch/arm64/kernel/mpam/mpam_device.c b/arch/arm64/kernel/mpam/mpam_device.c index 269a99695e6c..096ca71ff648 100644 --- a/arch/arm64/kernel/mpam/mpam_device.c +++ b/arch/arm64/kernel/mpam/mpam_device.c @@ -29,6 +29,9 @@ #include <linux/io.h> #include <linux/slab.h> #include <linux/types.h> +#include <linux/cpu.h> +#include <linux/cacheinfo.h> +#include <asm/mpam.h>
#include "mpam_device.h"
@@ -45,6 +48,75 @@ static LIST_HEAD(mpam_all_devices); /* Classes are the set of MSCs that make up components of the same type. */ LIST_HEAD(mpam_classes);
+static DEFINE_MUTEX(mpam_cpuhp_lock); +static int mpam_cpuhp_state; + + +static inline int mpam_cpu_online(unsigned int cpu); +static inline int mpam_cpu_offline(unsigned int cpu); + +static struct mpam_sysprops_prop mpam_sysprops; + +/* + * mpam is enabled once all devices have been probed from CPU online callbacks, + * scheduled via this work_struct. + */ +static struct work_struct mpam_enable_work; + +/* + * This gets set if something terrible happens, it prevents future attempts + * to configure devices. + */ +static int mpam_broken; +static struct work_struct mpam_failed_work; + +static int mpam_device_probe(struct mpam_device *dev) +{ + return 0; +} + +/* + * Enable mpam once all devices have been probed. + * Scheduled by mpam_discovery_complete() once all devices have been created. + * Also scheduled when new devices are probed when new CPUs come online. + */ +static void __init mpam_enable(struct work_struct *work) +{ + unsigned long flags; + struct mpam_device *dev; + bool all_devices_probed = true; + + /* Have we probed all the devices? */ + mutex_lock(&mpam_devices_lock); + list_for_each_entry(dev, &mpam_all_devices, glbl_list) { + spin_lock_irqsave(&dev->lock, flags); + if (!dev->probed) + all_devices_probed = false; + spin_unlock_irqrestore(&dev->lock, flags); + + if (!all_devices_probed) + break; + } + mutex_unlock(&mpam_devices_lock); + + if (!all_devices_probed) + return; +} + +static void mpam_failed(struct work_struct *work) +{ + /* + * Make it look like all CPUs are offline. This also resets the + * cpu default values and disables interrupts. + */ + mutex_lock(&mpam_cpuhp_lock); + if (mpam_cpuhp_state) { + cpuhp_remove_state(mpam_cpuhp_state); + mpam_cpuhp_state = 0; + } + mutex_unlock(&mpam_cpuhp_lock); +} + static struct mpam_device * __init mpam_device_alloc(struct mpam_component *comp) { @@ -242,6 +314,28 @@ static int mpam_cpus_have_feature(void) return 1; }
+/* + * get max partid from reading SYS_MPAMIDR_EL1. + */ +static inline u16 mpam_cpu_max_partid(void) +{ + u64 reg; + + reg = mpam_read_sysreg_s(SYS_MPAMIDR_EL1, "SYS_MPAMIDR_EL1"); + return reg & PARTID_MAX_MASK; +} + +/* + * get max pmg from reading SYS_MPAMIDR_EL1. + */ +static inline u16 mpam_cpu_max_pmg(void) +{ + u64 reg; + + reg = mpam_read_sysreg_s(SYS_MPAMIDR_EL1, "SYS_MPAMIDR_EL1"); + return (reg & PMG_MAX_MASK) >> PMG_MAX_SHIFT; +} + /* * prepare for initializing devices. */ @@ -250,14 +344,149 @@ int __init mpam_discovery_start(void) if (!mpam_cpus_have_feature()) return -EOPNOTSUPP;
+ mpam_sysprops.max_partid = mpam_cpu_max_partid(); + mpam_sysprops.max_pmg = mpam_cpu_max_pmg(); + + INIT_WORK(&mpam_enable_work, mpam_enable); + INIT_WORK(&mpam_failed_work, mpam_failed); + return 0; }
-int __init mpam_discovery_complete(void) +static int __online_devices(struct mpam_component *comp, int cpu) { + int err = 0; + unsigned long flags; + struct mpam_device *dev; + bool new_device_probed = false; + + list_for_each_entry(dev, &comp->devices, comp_list) { + if (!cpumask_test_cpu(cpu, &dev->fw_affinity)) + continue; + + spin_lock_irqsave(&dev->lock, flags); + if (!dev->probed) { + err = mpam_device_probe(dev); + if (!err) + new_device_probed = true; + } + + cpumask_set_cpu(cpu, &dev->online_affinity); + spin_unlock_irqrestore(&dev->lock, flags); + + if (err) + return err; + } + + if (new_device_probed) + return 1; + + return 0; +} + +/* + * Firmware didn't give us an affinity, but a cache-id, if this cpu has that + * cache-id, update the fw_affinity for this component. + */ +static void +mpam_sync_cpu_cache_component_fw_affinity(struct mpam_class *class, int cpu) +{ + int cpu_cache_id; + struct cacheinfo *leaf; + struct mpam_component *comp; + + lockdep_assert_held(&mpam_devices_lock); /* we modify mpam_sysprops */ + + if (class->type != MPAM_CLASS_CACHE) + return; + + cpu_cache_id = cpu_to_node(cpu); + comp = mpam_component_get(class, cpu_cache_id, false); + + /* This cpu does not have a component of this class */ + if (IS_ERR(comp)) + return; + + /* + * The resctrl rmid_threshold is based on cache size. Keep track of + * the biggest cache we've seen. + */ + leaf = get_cpu_cache_leaf(cpu, class->level); + if (leaf) + mpam_sysprops.mpam_llc_size = max(mpam_sysprops.mpam_llc_size, + leaf->size); + + cpumask_set_cpu(cpu, &comp->fw_affinity); + cpumask_set_cpu(cpu, &class->fw_affinity); +} + +static int mpam_cpu_online(unsigned int cpu) +{ + int err = 0; + struct mpam_class *class; + struct mpam_component *comp; + bool new_device_probed = false; + + mutex_lock(&mpam_devices_lock); + + list_for_each_entry(class, &mpam_classes, classes_list) { + mpam_sync_cpu_cache_component_fw_affinity(class, cpu); + + list_for_each_entry(comp, &class->components, class_list) { + if (!cpumask_test_cpu(cpu, &comp->fw_affinity)) + continue; + + err = __online_devices(comp, cpu); + if (err > 0) + new_device_probed = true; + if (err < 0) + break; // mpam_broken + } + } + + if (new_device_probed && err >= 0) + schedule_work(&mpam_enable_work); + + mutex_unlock(&mpam_devices_lock); + if (err < 0) { + if (!cmpxchg(&mpam_broken, err, 0)) + schedule_work(&mpam_failed_work); + return err; + } + return 0; }
+static int mpam_cpu_offline(unsigned int cpu) +{ + struct mpam_device *dev; + + mutex_lock(&mpam_devices_lock); + list_for_each_entry(dev, &mpam_all_devices, glbl_list) + cpumask_clear_cpu(cpu, &dev->online_affinity); + + mutex_unlock(&mpam_devices_lock); + + return 0; +} + +int __init mpam_discovery_complete(void) +{ + int ret = 0; + + mutex_lock(&mpam_cpuhp_lock); + mpam_cpuhp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, + "mpam:online", mpam_cpu_online, + mpam_cpu_offline); + if (mpam_cpuhp_state <= 0) { + pr_err("Failed to register 'dyn' cpuhp callbacks"); + ret = -EINVAL; + } + mutex_unlock(&mpam_cpuhp_lock); + + return ret; +} + void __init mpam_discovery_failed(void) { struct mpam_class *class, *tmp; diff --git a/arch/arm64/kernel/mpam/mpam_device.h b/arch/arm64/kernel/mpam/mpam_device.h index ab986fb1911f..7b8d9ae5a548 100644 --- a/arch/arm64/kernel/mpam/mpam_device.h +++ b/arch/arm64/kernel/mpam/mpam_device.h @@ -89,4 +89,11 @@ struct mpam_class { struct list_head classes_list; };
+/* System wide properties */ +struct mpam_sysprops_prop { + u32 mpam_llc_size; + u16 max_partid; + u16 max_pmg; +}; + #endif /* _ASM_ARM64_MPAM_DEVICE_H */