From: eillon <yezhenyu2@huawei.com> We use QEMU to enable or disable the HDBSS feature during live migration. We can use the migration-parameter to control the size of the HDBSS buffer, such as: migrate_set_parameter hdbss-buffer-size 3 info migrate_parameters Signed-off-by: eillon <yezhenyu2@huawei.com> --- linux-headers/linux/kvm.h | 2 ++ migration/migration-hmp-cmds.c | 9 +++++++++ migration/migration.h | 7 +++++++ migration/options.c | 21 +++++++++++++++++++++ migration/options.h | 1 + migration/ram.c | 28 ++++++++++++++++++++++++++++ qapi/migration.json | 17 ++++++++++++++--- 7 files changed, 82 insertions(+), 3 deletions(-) diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h index b94c5fd90f..57d6e12744 100644 --- a/linux-headers/linux/kvm.h +++ b/linux-headers/linux/kvm.h @@ -1212,6 +1212,8 @@ struct kvm_ppc_resize_hpt { /* support request to inject secret to CSV3 guest */ #define KVM_CAP_HYGON_COCO_EXT_CSV3_INJ_SECRET (1 << 2) +#define KVM_CAP_ARM_HW_DIRTY_STATE_TRACK 502 + #define KVM_CAP_ARM_VIRT_MSI_BYPASS 799 #define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE) diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp-cmds.c index aac5e7a73a..9857e2c97f 100644 --- a/migration/migration-hmp-cmds.c +++ b/migration/migration-hmp-cmds.c @@ -409,6 +409,11 @@ void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict) monitor_printf(mon, "%s: %s\n", MigrationParameter_str(MIGRATION_PARAMETER_SEV_AMD_CERT), params->sev_amd_cert); + + assert(params->has_hdbss_buffer_size); + monitor_printf(mon, "%s: %u\n", + MigrationParameter_str(MIGRATION_PARAMETER_HDBSS_BUFFER_SIZE), + params->hdbss_buffer_size); } qapi_free_MigrationParameters(params); @@ -725,6 +730,10 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict) p->sev_amd_cert->type = QTYPE_QSTRING; visit_type_str(v, param, &p->sev_amd_cert->u.s, &err); break; + case MIGRATION_PARAMETER_HDBSS_BUFFER_SIZE: + p->has_hdbss_buffer_size = true; + visit_type_uint8(v, param, &p->hdbss_buffer_size, &err); + break; default: assert(0); } diff --git a/migration/migration.h b/migration/migration.h index eeddb7c0bd..4a95f00157 100644 --- a/migration/migration.h +++ b/migration/migration.h @@ -48,6 +48,13 @@ struct PostcopyBlocktimeContext; */ #define CLEAR_BITMAP_SHIFT_MAX 31 +/* + * The default HDBSS size. The value ranges [0, 9]. + * Set to 0 to disable the HDBSS feature. + */ +#define DEFAULT_HDBSS_BUFFER_SIZE 0 +#define MAX_HDBSS_BUFFER_SIZE 9 + /* This is an abstraction of a "temp huge page" for postcopy's purpose */ typedef struct { /* diff --git a/migration/options.c b/migration/options.c index 71e71ea801..71645c8721 100644 --- a/migration/options.c +++ b/migration/options.c @@ -186,6 +186,9 @@ Property migration_properties[] = { DEFINE_PROP_STRING("sev-pdh", MigrationState, parameters.sev_pdh), DEFINE_PROP_STRING("sev-plat-cert", MigrationState, parameters.sev_plat_cert), DEFINE_PROP_STRING("sev-amd-cert", MigrationState, parameters.sev_amd_cert), + DEFINE_PROP_UINT8("hdbss-buffer-size", MigrationState, + parameters.hdbss_buffer_size, + DEFAULT_HDBSS_BUFFER_SIZE), /* Migration capabilities */ DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE), @@ -853,6 +856,13 @@ MigMode migrate_mode(void) return s->parameters.mode; } +int migrate_hdbss_buffer_size(void) +{ + MigrationState *s = migrate_get_current(); + + return s->parameters.hdbss_buffer_size; +} + int migrate_multifd_channels(void) { MigrationState *s = migrate_get_current(); @@ -1032,6 +1042,8 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp) params->vcpu_dirty_limit = s->parameters.vcpu_dirty_limit; params->has_mode = true; params->mode = s->parameters.mode; + params->has_hdbss_buffer_size = true; + params->hdbss_buffer_size = s->parameters.hdbss_buffer_size; return params; } @@ -1069,6 +1081,7 @@ void migrate_params_init(MigrationParameters *params) params->has_x_vcpu_dirty_limit_period = true; params->has_vcpu_dirty_limit = true; params->has_mode = true; + params->has_hdbss_buffer_size = true; params->sev_pdh = g_strdup(""); params->sev_plat_cert = g_strdup(""); @@ -1415,6 +1428,10 @@ static void migrate_params_test_apply(MigrateSetParameters *params, assert(params->sev_amd_cert->type == QTYPE_QSTRING); dest->sev_amd_cert = params->sev_amd_cert->u.s; } + + if (params->has_hdbss_buffer_size) { + dest->hdbss_buffer_size = params->hdbss_buffer_size; + } } static void migrate_params_apply(MigrateSetParameters *params, Error **errp) @@ -1579,6 +1596,10 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) assert(params->sev_amd_cert->type == QTYPE_QSTRING); s->parameters.sev_amd_cert = g_strdup(params->sev_amd_cert->u.s); } + + if (params->has_hdbss_buffer_size) { + s->parameters.hdbss_buffer_size = params->hdbss_buffer_size; + } } void qmp_migrate_set_parameters(MigrateSetParameters *params, Error **errp) diff --git a/migration/options.h b/migration/options.h index 9aca5e41ad..987fc81a18 100644 --- a/migration/options.h +++ b/migration/options.h @@ -85,6 +85,7 @@ uint64_t migrate_max_bandwidth(void); uint64_t migrate_avail_switchover_bandwidth(void); uint64_t migrate_max_postcopy_bandwidth(void); MigMode migrate_mode(void); +int migrate_hdbss_buffer_size(void); int migrate_multifd_channels(void); MultiFDCompression migrate_multifd_compression(void); int migrate_multifd_zlib_level(void); diff --git a/migration/ram.c b/migration/ram.c index 1f9348fd06..fcef466595 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -39,6 +39,7 @@ #include "migration-stats.h" #include "migration/register.h" #include "migration/misc.h" +#include "migration/options.h" #include "qemu-file.h" #include "postcopy-ram.h" #include "page_cache.h" @@ -2790,6 +2791,31 @@ static void xbzrle_cleanup(void) XBZRLE_cache_unlock(); } +static void kvm_update_hdbss_cap(bool enable) +{ + KVMState *s = kvm_state; + int size, ret; + + if (!kvm_check_extension(s, KVM_CAP_ARM_HW_DIRTY_STATE_TRACK)) { + return; + } + + size = migrate_hdbss_buffer_size(); + if (size < 0 || size > MAX_HDBSS_BUFFER_SIZE) { + fprintf(stderr, "Invalid hdbss buffer size: %d\n", size); + return; + } + + ret = kvm_vm_enable_cap(s, KVM_CAP_ARM_HW_DIRTY_STATE_TRACK, 0, + enable ? size : 0); + if (ret) { + fprintf(stderr, "Could not %s KVM_CAP_ARM_HW_DIRTY_STATE_TRACK: %d\n", + enable ? "enable" : "disable", ret); + } + + return; +} + static void ram_save_cleanup(void *opaque) { RAMState **rsp = opaque; @@ -2806,6 +2832,7 @@ static void ram_save_cleanup(void *opaque) * memory_global_dirty_log_stop will assert that * memory_global_dirty_log_start/stop used in pairs */ + kvm_update_hdbss_cap(false); memory_global_dirty_log_stop(GLOBAL_DIRTY_MIGRATION); } } @@ -3209,6 +3236,7 @@ static void ram_init_bitmaps(RAMState *rs) ram_list_init_bitmaps(); /* We don't use dirty log with background snapshots */ if (!migrate_background_snapshot()) { + kvm_update_hdbss_cap(true); memory_global_dirty_log_start(GLOBAL_DIRTY_MIGRATION); migration_bitmap_sync_precopy(rs, false); } diff --git a/qapi/migration.json b/qapi/migration.json index 3aed216c3b..f672da5c0d 100644 --- a/qapi/migration.json +++ b/qapi/migration.json @@ -902,6 +902,9 @@ # @sev-amd-cert: AMD certificate chain which include ASK and OCA encoded in # base64, or vendor cert filename for hygon (Since 4.2) # +# @hdbss-buffer-size: Size of the HDBSS(Hardware Dirty state tracking Structure). +# Defaults to 0. (Since 8.6) +# # Features: # # @deprecated: Member @block-incremental is deprecated. Use @@ -937,7 +940,7 @@ { 'name': 'x-vcpu-dirty-limit-period', 'features': ['unstable'] }, 'vcpu-dirty-limit', 'mode', - 'sev-pdh', 'sev-plat-cert', 'sev-amd-cert'] } + 'sev-pdh', 'sev-plat-cert', 'sev-amd-cert', 'hdbss-buffer-size'] } ## # @MigrateSetParameters: @@ -1106,6 +1109,9 @@ # @sev-amd-cert: AMD certificate chain which include ASK and OCA encoded in # base64, or vendor cert filename for hygon (Since 4.2) # +# @hdbss-buffer-size: Size of the HDBSS(Hardware Dirty state tracking Structure). +# Defaults to 0. (Since 8.6) +# # Features: # # @deprecated: Member @block-incremental is deprecated. Use @@ -1165,7 +1171,8 @@ '*mode': 'MigMode', '*sev-pdh': 'StrOrNull', '*sev-plat-cert': 'StrOrNull', - '*sev-amd-cert' : 'StrOrNull' } } + '*sev-amd-cert' : 'StrOrNull', + '*hdbss-buffer-size': 'uint8'} } ## @@ -1355,6 +1362,9 @@ # @sev-amd-cert: AMD certificate chain which include ASK and OCA encoded in # base64, or vendor cert filename for hygon (Since 4.2) # +# @hdbss-buffer-size: Size of the HDBSS(Hardware Dirty state tracking Structure). +# Defaults to 0. (Since 8.6) +# # Features: # # @deprecated: Member @block-incremental is deprecated. Use @@ -1410,7 +1420,8 @@ '*mode': 'MigMode', '*sev-pdh': 'str', '*sev-plat-cert': 'str', - '*sev-amd-cert' : 'str'} } + '*sev-amd-cert' : 'str', + '*hdbss-buffer-size': 'uint8'} } ## # @query-migrate-parameters: -- 2.39.3