From: eillon <yezhenyu2(a)huawei.com>
We use QEMU to enable or disable the HDBSS feature during live
migration. We can use the migration-parameter to control the size
of the HDBSS buffer, such as:
migrate_set_parameter hdbss-buffer-size 3
info migrate_parameters
Signed-off-by: eillon <yezhenyu2(a)huawei.com>
---
linux-headers/linux/kvm.h | 2 ++
migration/migration-hmp-cmds.c | 9 +++++++++
migration/migration.h | 7 +++++++
migration/options.c | 21 +++++++++++++++++++++
migration/options.h | 1 +
migration/ram.c | 28 ++++++++++++++++++++++++++++
qapi/migration.json | 17 ++++++++++++++---
7 files changed, 82 insertions(+), 3 deletions(-)
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index b94c5fd90f..57d6e12744 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -1212,6 +1212,8 @@ struct kvm_ppc_resize_hpt {
/* support request to inject secret to CSV3 guest */
#define KVM_CAP_HYGON_COCO_EXT_CSV3_INJ_SECRET (1 << 2)
+#define KVM_CAP_ARM_HW_DIRTY_STATE_TRACK 502
+
#define KVM_CAP_ARM_VIRT_MSI_BYPASS 799
#define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE)
diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp-cmds.c
index aac5e7a73a..9857e2c97f 100644
--- a/migration/migration-hmp-cmds.c
+++ b/migration/migration-hmp-cmds.c
@@ -409,6 +409,11 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict)
monitor_printf(mon, "%s: %s\n",
MigrationParameter_str(MIGRATION_PARAMETER_SEV_AMD_CERT),
params->sev_amd_cert);
+
+ assert(params->has_hdbss_buffer_size);
+ monitor_printf(mon, "%s: %u\n",
+ MigrationParameter_str(MIGRATION_PARAMETER_HDBSS_BUFFER_SIZE),
+ params->hdbss_buffer_size);
}
qapi_free_MigrationParameters(params);
@@ -725,6 +730,10 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
p->sev_amd_cert->type = QTYPE_QSTRING;
visit_type_str(v, param, &p->sev_amd_cert->u.s, &err);
break;
+ case MIGRATION_PARAMETER_HDBSS_BUFFER_SIZE:
+ p->has_hdbss_buffer_size = true;
+ visit_type_uint8(v, param, &p->hdbss_buffer_size, &err);
+ break;
default:
assert(0);
}
diff --git a/migration/migration.h b/migration/migration.h
index eeddb7c0bd..4a95f00157 100644
--- a/migration/migration.h
+++ b/migration/migration.h
@@ -48,6 +48,13 @@ struct PostcopyBlocktimeContext;
*/
#define CLEAR_BITMAP_SHIFT_MAX 31
+/*
+ * The default HDBSS size. The value ranges [0, 9].
+ * Set to 0 to disable the HDBSS feature.
+ */
+#define DEFAULT_HDBSS_BUFFER_SIZE 0
+#define MAX_HDBSS_BUFFER_SIZE 9
+
/* This is an abstraction of a "temp huge page" for postcopy's purpose */
typedef struct {
/*
diff --git a/migration/options.c b/migration/options.c
index 71e71ea801..71645c8721 100644
--- a/migration/options.c
+++ b/migration/options.c
@@ -186,6 +186,9 @@ Property migration_properties[] = {
DEFINE_PROP_STRING("sev-pdh", MigrationState, parameters.sev_pdh),
DEFINE_PROP_STRING("sev-plat-cert", MigrationState, parameters.sev_plat_cert),
DEFINE_PROP_STRING("sev-amd-cert", MigrationState, parameters.sev_amd_cert),
+ DEFINE_PROP_UINT8("hdbss-buffer-size", MigrationState,
+ parameters.hdbss_buffer_size,
+ DEFAULT_HDBSS_BUFFER_SIZE),
/* Migration capabilities */
DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
@@ -853,6 +856,13 @@ MigMode migrate_mode(void)
return s->parameters.mode;
}
+int migrate_hdbss_buffer_size(void)
+{
+ MigrationState *s = migrate_get_current();
+
+ return s->parameters.hdbss_buffer_size;
+}
+
int migrate_multifd_channels(void)
{
MigrationState *s = migrate_get_current();
@@ -1032,6 +1042,8 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp)
params->vcpu_dirty_limit = s->parameters.vcpu_dirty_limit;
params->has_mode = true;
params->mode = s->parameters.mode;
+ params->has_hdbss_buffer_size = true;
+ params->hdbss_buffer_size = s->parameters.hdbss_buffer_size;
return params;
}
@@ -1069,6 +1081,7 @@ void migrate_params_init(MigrationParameters *params)
params->has_x_vcpu_dirty_limit_period = true;
params->has_vcpu_dirty_limit = true;
params->has_mode = true;
+ params->has_hdbss_buffer_size = true;
params->sev_pdh = g_strdup("");
params->sev_plat_cert = g_strdup("");
@@ -1415,6 +1428,10 @@ static void migrate_params_test_apply(MigrateSetParameters *params,
assert(params->sev_amd_cert->type == QTYPE_QSTRING);
dest->sev_amd_cert = params->sev_amd_cert->u.s;
}
+
+ if (params->has_hdbss_buffer_size) {
+ dest->hdbss_buffer_size = params->hdbss_buffer_size;
+ }
}
static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
@@ -1579,6 +1596,10 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp)
assert(params->sev_amd_cert->type == QTYPE_QSTRING);
s->parameters.sev_amd_cert = g_strdup(params->sev_amd_cert->u.s);
}
+
+ if (params->has_hdbss_buffer_size) {
+ s->parameters.hdbss_buffer_size = params->hdbss_buffer_size;
+ }
}
void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp)
diff --git a/migration/options.h b/migration/options.h
index 9aca5e41ad..987fc81a18 100644
--- a/migration/options.h
+++ b/migration/options.h
@@ -85,6 +85,7 @@ uint64_t migrate_max_bandwidth(void);
uint64_t migrate_avail_switchover_bandwidth(void);
uint64_t migrate_max_postcopy_bandwidth(void);
MigMode migrate_mode(void);
+int migrate_hdbss_buffer_size(void);
int migrate_multifd_channels(void);
MultiFDCompression migrate_multifd_compression(void);
int migrate_multifd_zlib_level(void);
diff --git a/migration/ram.c b/migration/ram.c
index 1f9348fd06..fcef466595 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -39,6 +39,7 @@
#include "migration-stats.h"
#include "migration/register.h"
#include "migration/misc.h"
+#include "migration/options.h"
#include "qemu-file.h"
#include "postcopy-ram.h"
#include "page_cache.h"
@@ -2790,6 +2791,31 @@ static void xbzrle_cleanup(void)
XBZRLE_cache_unlock();
}
+static void kvm_update_hdbss_cap(bool enable)
+{
+ KVMState *s = kvm_state;
+ int size, ret;
+
+ if (!kvm_check_extension(s, KVM_CAP_ARM_HW_DIRTY_STATE_TRACK)) {
+ return;
+ }
+
+ size = migrate_hdbss_buffer_size();
+ if (size < 0 || size > MAX_HDBSS_BUFFER_SIZE) {
+ fprintf(stderr, "Invalid hdbss buffer size: %d\n", size);
+ return;
+ }
+
+ ret = kvm_vm_enable_cap(s, KVM_CAP_ARM_HW_DIRTY_STATE_TRACK, 0,
+ enable ? size : 0);
+ if (ret) {
+ fprintf(stderr, "Could not %s KVM_CAP_ARM_HW_DIRTY_STATE_TRACK: %d\n",
+ enable ? "enable" : "disable", ret);
+ }
+
+ return;
+}
+
static void ram_save_cleanup(void *opaque)
{
RAMState **rsp = opaque;
@@ -2806,6 +2832,7 @@ static void ram_save_cleanup(void *opaque)
* memory_global_dirty_log_stop will assert that
* memory_global_dirty_log_start/stop used in pairs
*/
+ kvm_update_hdbss_cap(false);
memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION);
}
}
@@ -3209,6 +3236,7 @@ static void ram_init_bitmaps(RAMState *rs)
ram_list_init_bitmaps();
/* We don't use dirty log with background snapshots */
if (!migrate_background_snapshot()) {
+ kvm_update_hdbss_cap(true);
memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION);
migration_bitmap_sync_precopy(rs, false);
}
diff --git a/qapi/migration.json b/qapi/migration.json
index 3aed216c3b..f672da5c0d 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -902,6 +902,9 @@
# @sev-amd-cert: AMD certificate chain which include ASK and OCA encoded in
# base64, or vendor cert filename for hygon (Since 4.2)
#
+# @hdbss-buffer-size: Size of the HDBSS(Hardware Dirty state tracking Structure).
+# Defaults to 0. (Since 8.6)
+#
# Features:
#
# @deprecated: Member @block-incremental is deprecated. Use
@@ -937,7 +940,7 @@
{ 'name': 'x-vcpu-dirty-limit-period', 'features': ['unstable'] },
'vcpu-dirty-limit',
'mode',
- 'sev-pdh', 'sev-plat-cert', 'sev-amd-cert'] }
+ 'sev-pdh', 'sev-plat-cert', 'sev-amd-cert', 'hdbss-buffer-size'] }
##
# @MigrateSetParameters:
@@ -1106,6 +1109,9 @@
# @sev-amd-cert: AMD certificate chain which include ASK and OCA encoded in
# base64, or vendor cert filename for hygon (Since 4.2)
#
+# @hdbss-buffer-size: Size of the HDBSS(Hardware Dirty state tracking Structure).
+# Defaults to 0. (Since 8.6)
+#
# Features:
#
# @deprecated: Member @block-incremental is deprecated. Use
@@ -1165,7 +1171,8 @@
'*mode': 'MigMode',
'*sev-pdh': 'StrOrNull',
'*sev-plat-cert': 'StrOrNull',
- '*sev-amd-cert' : 'StrOrNull' } }
+ '*sev-amd-cert' : 'StrOrNull',
+ '*hdbss-buffer-size': 'uint8'} }
##
@@ -1355,6 +1362,9 @@
# @sev-amd-cert: AMD certificate chain which include ASK and OCA encoded in
# base64, or vendor cert filename for hygon (Since 4.2)
#
+# @hdbss-buffer-size: Size of the HDBSS(Hardware Dirty state tracking Structure).
+# Defaults to 0. (Since 8.6)
+#
# Features:
#
# @deprecated: Member @block-incremental is deprecated. Use
@@ -1410,7 +1420,8 @@
'*mode': 'MigMode',
'*sev-pdh': 'str',
'*sev-plat-cert': 'str',
- '*sev-amd-cert' : 'str'} }
+ '*sev-amd-cert' : 'str',
+ '*hdbss-buffer-size': 'uint8'} }
##
# @query-migrate-parameters:
--
2.39.3
