From: Longfang Liu <liulongfang@huawei.com> In the previous UADK v2 framework, the entire UADK was bound to SVA mode, and checks were performed on device morphology and queue attributes according to SVA mode requirements. After adding the new No-SVA mode memory pool, these bindings need to be removed to ensure that UADK can normally execute business operations on an OS running in No-SVA mode Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- include/wd_alg.h | 25 ++++--------------- libwd.map | 4 +++ wd.c | 4 --- wd_alg.c | 64 +++++++++++++++++------------------------------- wd_util.c | 5 ---- 5 files changed, 31 insertions(+), 71 deletions(-) diff --git a/include/wd_alg.h b/include/wd_alg.h index 2fc350a..5ff73ca 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -115,26 +115,6 @@ struct wd_alg_driver { int (*get_extend_ops)(void *ops); }; -inline int wd_alg_driver_init(struct wd_alg_driver *drv, void *conf) -{ - return drv->init(drv, conf); -} - -inline void wd_alg_driver_exit(struct wd_alg_driver *drv) -{ - drv->exit(drv); -} - -inline int wd_alg_driver_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) -{ - return drv->send(drv, ctx, msg); -} - -inline int wd_alg_driver_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) -{ - return drv->recv(drv, ctx, msg); -} - /* * wd_alg_driver_register() - Register a device driver. * @wd_alg_driver: a device driver that supports an algorithm. @@ -194,6 +174,11 @@ bool wd_drv_alg_support(const char *alg_name, void wd_enable_drv(struct wd_alg_driver *drv); void wd_disable_drv(struct wd_alg_driver *drv); +int wd_alg_driver_init(struct wd_alg_driver *drv, void *conf); +void wd_alg_driver_exit(struct wd_alg_driver *drv); +int wd_alg_driver_send(struct wd_alg_driver *drv, handle_t ctx, void *msg); +int wd_alg_driver_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg); + struct wd_alg_list *wd_get_alg_head(void); #ifdef WD_STATIC_DRV diff --git a/libwd.map b/libwd.map index b1b90b3..90eb5c5 100644 --- a/libwd.map +++ b/libwd.map @@ -49,6 +49,10 @@ global: wd_enable_drv; wd_disable_drv; wd_get_alg_head; + wd_alg_driver_init; + wd_alg_driver_exit; + wd_alg_driver_send; + wd_alg_driver_recv; wd_find_ctx; wd_get_dev_id; diff --git a/wd.c b/wd.c index 3e867b6..657fbae 100644 --- a/wd.c +++ b/wd.c @@ -238,10 +238,6 @@ static int get_dev_info(struct uacce_dev *dev) ret = get_int_attr(dev, "flags", &dev->flags); if (ret < 0) return ret; - else if (!((unsigned int)dev->flags & UACCE_DEV_SVA)) { - WD_ERR("skip none sva uacce device!\n"); - return -WD_ENODEV; - } ret = get_int_attr(dev, "region_mmio_size", &value); if (ret < 0) diff --git a/wd_alg.c b/wd_alg.c index 08f0e2e..9c7c0fd 100644 --- a/wd_alg.c +++ b/wd_alg.c @@ -24,46 +24,6 @@ static struct wd_alg_list *alg_list_tail = &alg_list_head; static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; -static bool wd_check_dev_sva(const char *dev_name) -{ - char dev_path[PATH_MAX] = {'\0'}; - char buf[DEV_SVA_SIZE] = {'\0'}; - unsigned int val; - ssize_t ret; - int fd; - - ret = snprintf(dev_path, PATH_STR_SIZE, "%s/%s/%s", SYS_CLASS_DIR, - dev_name, SVA_FILE_NAME); - if (ret < 0) { - WD_ERR("failed to snprintf, device name: %s!\n", dev_name); - return false; - } - - /** - * The opened file is the specified device driver file. - * no need for realpath processing. - */ - fd = open(dev_path, O_RDONLY, 0); - if (fd < 0) { - WD_ERR("failed to open %s(%d)!\n", dev_path, -errno); - return false; - } - - ret = read(fd, buf, DEV_SVA_SIZE - 1); - if (ret <= 0) { - WD_ERR("failed to read anything at %s!\n", dev_path); - close(fd); - return false; - } - close(fd); - - val = strtol(buf, NULL, STR_DECIMAL); - if (val & UACCE_DEV_SVA) - return true; - - return false; -} - static bool wd_check_accel_dev(const char *dev_name) { struct dirent *dev_dir; @@ -80,8 +40,7 @@ static bool wd_check_accel_dev(const char *dev_name) !strncmp(dev_dir->d_name, "..", LINUX_PRTDIR_SIZE)) continue; - if (!strncmp(dev_dir->d_name, dev_name, strlen(dev_name)) && - wd_check_dev_sva(dev_dir->d_name)) { + if (!strncmp(dev_dir->d_name, dev_name, strlen(dev_name))) { closedir(wd_class); return true; } @@ -413,3 +372,24 @@ void wd_release_drv(struct wd_alg_driver *drv) select_node->refcnt--; pthread_mutex_unlock(&mutex); } + +int wd_alg_driver_init(struct wd_alg_driver *drv, void *conf) +{ + return drv->init(drv, conf); +} + +void wd_alg_driver_exit(struct wd_alg_driver *drv) +{ + drv->exit(drv); +} + +int wd_alg_driver_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +{ + return drv->send(drv, ctx, msg); +} + +int wd_alg_driver_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +{ + return drv->recv(drv, ctx, msg); +} + diff --git a/wd_util.c b/wd_util.c index d0d83eb..beb4131 100644 --- a/wd_util.c +++ b/wd_util.c @@ -1984,11 +1984,6 @@ int wd_init_param_check(struct wd_ctx_config *config, struct wd_sched *sched) return -WD_EINVAL; } - if (!wd_is_sva(config->ctxs[0].ctx)) { - WD_ERR("invalid: the mode is non sva, please check system!\n"); - return -WD_EINVAL; - } - return 0; } -- 2.33.0

From: Longfang Liu <liulongfang@huawei.com> In the uadk device scheduling algorithm, when the device ID numbering scheme differs from the device count numbering scheme, all accelerator devices are uniformly numbered when passed through to user space. This can result in device IDs being significantly larger than the actual number of devices. Therefore, a mapping mechanism is required to handle the relationship between the number of devices within a process and their corresponding device IDs, thereby resolving this issue. Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- include/wd_internal.h | 3 +- wd_sched.c | 149 +++++++++++++++++++++++++++++++----------- wd_util.c | 2 +- 3 files changed, 115 insertions(+), 39 deletions(-) diff --git a/include/wd_internal.h b/include/wd_internal.h index 769192d..d899555 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -13,7 +13,8 @@ #ifdef __cplusplus extern "C" { #endif -#define NOSVA_DEVICE_MAX 16 + +#define DEVICE_REGION_MAX 16 #define DECIMAL_NUMBER 10 #define MAX_FD_NUM 65535 diff --git a/wd_sched.c b/wd_sched.c index 46a8c00..ec1e7b6 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -65,6 +65,10 @@ struct wd_sched_info { bool valid; }; +struct dev_region_map { + __u32 dev_id; + __u32 region_id; +}; /* * wd_sched_ctx - define the context of the scheduler. * @policy: define the policy of the scheduler. @@ -81,6 +85,7 @@ struct wd_sched_ctx { __u16 dev_num; user_poll_func poll_func; int numa_map[NUMA_NUM_NODES]; + struct dev_region_map dev_id_map[DEVICE_REGION_MAX]; struct wd_sched_info sched_info[0]; }; @@ -453,13 +458,27 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, static bool sched_dev_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) { - if (key->dev_id >= sched_ctx->dev_num || key->mode >= SCHED_MODE_BUTT || - key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's dev: %u, mode: %u, type: %u!\n", + bool found = false; + int i; + + if (key->mode >= SCHED_MODE_BUTT || key->type >= sched_ctx->type_num) { + WD_ERR("invalid: sched key's device id: %u, mode: %u, type: %u!\n", key->dev_id, key->mode, key->type); return false; } + for (i = 0; i < sched_ctx->dev_num; i++) { + if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { + found = true; + break; + } + } + + if (!found) { + WD_ERR("invalid: dev_id %u is not registered!\n", key->dev_id); + return false; + } + return true; } @@ -470,11 +489,16 @@ static struct sched_ctx_region *sched_dev_get_region(struct wd_sched_ctx *sched_ const struct sched_key *key) { struct wd_sched_info *sched_info; - - sched_info = sched_ctx->sched_info; - if (key->dev_id < sched_ctx->dev_num && - sched_info[key->dev_id].ctx_region[key->mode][key->type].valid) - return &sched_info[key->dev_id].ctx_region[key->mode][key->type]; + int i, region_id; + + for (i = 0; i < sched_ctx->dev_num; i++) { + if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { + region_id = sched_ctx->dev_id_map[i].region_id; + sched_info = &sched_ctx->sched_info[region_id]; + if (sched_info->ctx_region[key->mode][key->type].valid) + return &sched_info->ctx_region[key->mode][key->type]; + } + } /* * If the scheduling domain of dev_id does not exist, @@ -528,23 +552,19 @@ static handle_t session_dev_sched_init(handle_t h_sched_ctx, void *sched_param) return (handle_t)(-WD_EINVAL); } + if (!param) { + WD_DEBUG("no-sva session don't set scheduler parameters!\n"); + return (handle_t)(-WD_EINVAL); + } + skey = malloc(sizeof(struct sched_key)); if (!skey) { WD_ERR("failed to alloc memory for session sched key!\n"); return (handle_t)(-WD_ENOMEM); } - if (!param) { - WD_DEBUG("no-sva session don't set scheduler parameters!\n"); - return (handle_t)(-WD_EINVAL); - } - skey->type = param->type; skey->dev_id = param->dev_id; - if (skey->dev_id > NOSVA_DEVICE_MAX) { - WD_ERR("failed to get valid sched device region!\n"); - goto out; - } skey->sync_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); skey->async_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); @@ -621,12 +641,68 @@ static void wd_sched_map_cpus_to_dev(struct wd_sched_ctx *sched_ctx) } } +static int wd_instance_dev_region(struct wd_sched_ctx *sched_ctx, + struct sched_params *param) +{ + struct wd_sched_info *sched_info; + __u32 region_idx = INVALID_POS; + __u8 type, mode; + __u32 dev_id; + int i; + + dev_id = param->dev_id; + type = param->type; + mode = param->mode; + + /* Check whether dev_id has already been registered. */ + for (i = 0; i < sched_ctx->dev_num; i++) { + if (sched_ctx->dev_id_map[i].dev_id == dev_id) { + region_idx = sched_ctx->dev_id_map[i].region_id; + break; + } + } + + /* If not registered, allocate a new region. */ + if (region_idx == INVALID_POS) { + if (sched_ctx->dev_num >= DEVICE_REGION_MAX) { + WD_ERR("too many devices registered!\n"); + return -WD_EINVAL; + } + + region_idx = sched_ctx->dev_num; + sched_ctx->dev_id_map[region_idx].dev_id = dev_id; + sched_ctx->dev_id_map[region_idx].region_id = region_idx; + sched_ctx->dev_num++; + + sched_info = &sched_ctx->sched_info[region_idx]; + } else { + sched_info = &sched_ctx->sched_info[region_idx]; + } + + /* Check whether the mode and type have already been registered. */ + if (sched_info->ctx_region[mode][type].valid) { + WD_INFO("device %u mode %u type %u already registered\n", + dev_id, mode, type); + return WD_SUCCESS; + } + + /* Initialize the scheduling region for this mode and type */ + sched_info->ctx_region[mode][type].begin = param->begin; + sched_info->ctx_region[mode][type].end = param->end; + sched_info->ctx_region[mode][type].last = param->begin; + sched_info->ctx_region[mode][type].valid = true; + sched_info->valid = true; + + pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); + + return WD_SUCCESS; +} + int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *param) { struct wd_sched_info *sched_info = NULL; struct wd_sched_ctx *sched_ctx = NULL; __u8 type, mode; - __u32 dev_id; int numa_id; if (!sched || !sched->h_sched_ctx || !param) { @@ -640,7 +716,6 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para } numa_id = param->numa_id; - dev_id = param->dev_id; type = param->type; mode = param->mode; sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; @@ -652,12 +727,6 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para return -WD_EINVAL; } - if (sched_ctx->dev_num > 0 && dev_id >= sched_ctx->dev_num) { - WD_ERR("invalid: sched_ctx's dev id is %u, device num is %u!\n", - dev_id, sched_ctx->dev_num); - return -WD_EINVAL; - } - if (type >= sched_ctx->type_num) { WD_ERR("invalid: sched_ctx's type is %u, type_num is %u!\n", type, sched_ctx->type_num); @@ -671,10 +740,9 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para } if (sched_ctx->policy == SCHED_POLICY_DEV) - sched_info = &sched_ctx->sched_info[dev_id]; - else - sched_info = &sched_ctx->sched_info[numa_id]; + return wd_instance_dev_region(sched_ctx, param); + sched_info = &sched_ctx->sched_info[numa_id]; if (!sched_info->ctx_region[mode]) { WD_ERR("invalid: ctx_region is NULL, numa: %d, mode: %u!\n", numa_id, mode); @@ -687,11 +755,10 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para sched_info->ctx_region[mode][type].valid = true; sched_info->valid = true; - if (sched_ctx->policy != SCHED_POLICY_DEV) - wd_sched_map_cpus_to_dev(sched_ctx); + wd_sched_map_cpus_to_dev(sched_ctx); pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); - return 0; + return WD_SUCCESS; } void wd_sched_rr_release(struct wd_sched *sched) @@ -709,7 +776,7 @@ void wd_sched_rr_release(struct wd_sched *sched) /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ if (sched_ctx->policy == SCHED_POLICY_DEV) - region_num = sched_ctx->dev_num; + region_num = DEVICE_REGION_MAX; else region_num = sched_ctx->numa_num; @@ -758,7 +825,8 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; struct wd_sched *sched; - int i, j, region_num; + int region_num; + int i, j; if (sched_type >= SCHED_POLICY_BUTT || !type_num) { WD_ERR("invalid: sched_type is %u or type_num is %u!\n", @@ -772,8 +840,13 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, return NULL; } + if (sched_type == SCHED_POLICY_DEV) + region_num = DEVICE_REGION_MAX; + else + region_num = numa_num; + sched_ctx = calloc(1, sizeof(struct wd_sched_ctx) + - sizeof(struct wd_sched_info) * numa_num); + sizeof(struct wd_sched_info) * region_num); if (!sched_ctx) { WD_ERR("failed to alloc memory for sched_ctx!\n"); goto err_out; @@ -782,12 +855,14 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ if (sched_type == SCHED_POLICY_DEV) { sched_ctx->numa_num = 0; - sched_ctx->dev_num = numa_num; - region_num = sched_ctx->dev_num; + sched_ctx->dev_num = 0; + for (i = 0; i < DEVICE_REGION_MAX; i++) { + sched_ctx->dev_id_map[i].dev_id = INVALID_POS; + sched_ctx->dev_id_map[i].region_id = INVALID_POS; + } } else { sched_ctx->numa_num = numa_num; sched_ctx->dev_num = 0; - region_num = sched_ctx->numa_num; if (numa_num_check(sched_ctx->numa_num)) goto err_out; } diff --git a/wd_util.c b/wd_util.c index b96e125..d0d62a1 100644 --- a/wd_util.c +++ b/wd_util.c @@ -2964,7 +2964,7 @@ int wd_alg_attrs_init(struct wd_init_attrs *attrs) if (sched_type == SCHED_POLICY_DEV) alg_sched = wd_sched_rr_alloc(sched_type, attrs->ctx_params->op_type_num, - NOSVA_DEVICE_MAX, alg_poll_func); + DEVICE_REGION_MAX, alg_poll_func); else alg_sched = wd_sched_rr_alloc(sched_type, attrs->ctx_params->op_type_num, numa_max_node() + 1, alg_poll_func); -- 2.33.0

From: Wenkai Lin <linwenkai6@hisilicon.com> Initialize the algorithm names supported by each module. During subsequent initialization, this will be used to determine whether the algorithm executed by the user is supported. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> --- wd_aead.c | 1 + wd_agg.c | 1 + wd_cipher.c | 1 + wd_comp.c | 1 + wd_dh.c | 1 + wd_digest.c | 1 + wd_ecc.c | 1 + wd_join_gather.c | 1 + wd_rsa.c | 1 + wd_udma.c | 1 + 10 files changed, 10 insertions(+) diff --git a/wd_aead.c b/wd_aead.c index b10890f..373b6fe 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -446,6 +446,7 @@ static int wd_aead_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc if (ret < 0) return ret; + wd_aead_setting.config.alg_name = "aead"; ret = wd_init_ctx_config(&wd_aead_setting.config, config); if (ret) return ret; diff --git a/wd_agg.c b/wd_agg.c index 85fb7f6..8c54d10 100644 --- a/wd_agg.c +++ b/wd_agg.c @@ -580,6 +580,7 @@ static int wd_agg_alg_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret < 0) return ret; + wd_agg_setting.config.alg_name = "hashagg"; ret = wd_init_ctx_config(&wd_agg_setting.config, config); if (ret < 0) return ret; diff --git a/wd_cipher.c b/wd_cipher.c index ef54dc8..92ca07b 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -328,6 +328,7 @@ static int wd_cipher_common_init(struct wd_ctx_config *config, if (ret < 0) return ret; + wd_cipher_setting.config.alg_name = "cipher"; ret = wd_init_ctx_config(&wd_cipher_setting.config, config); if (ret < 0) return ret; diff --git a/wd_comp.c b/wd_comp.c index ac0470f..435f5a8 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -154,6 +154,7 @@ static int wd_comp_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc if (ret < 0) return ret; + wd_comp_setting.config.alg_name = "zlib gzip deflate lz77_zstd lz4 lz77_only"; ret = wd_init_ctx_config(&wd_comp_setting.config, config); if (ret < 0) return ret; diff --git a/wd_dh.c b/wd_dh.c index 2c88ce2..221322f 100644 --- a/wd_dh.c +++ b/wd_dh.c @@ -125,6 +125,7 @@ static int wd_dh_common_init(struct wd_ctx_config *config, struct wd_sched *sche if (ret < 0) return ret; + wd_dh_setting.config.alg_name = "dh"; ret = wd_init_ctx_config(&wd_dh_setting.config, config); if (ret) return ret; diff --git a/wd_digest.c b/wd_digest.c index cb0a111..2d31176 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -260,6 +260,7 @@ static int wd_digest_init_nolock(struct wd_ctx_config *config, if (ret < 0) return ret; + wd_digest_setting.config.alg_name = "digest"; ret = wd_init_ctx_config(&wd_digest_setting.config, config); if (ret < 0) return ret; diff --git a/wd_ecc.c b/wd_ecc.c index 2a1228d..b1971b9 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -192,6 +192,7 @@ static int wd_ecc_common_init(struct wd_ctx_config *config, struct wd_sched *sch if (ret < 0) return ret; + wd_ecc_setting.config.alg_name = "sm2 x448 x25519 ecdsa ecdh"; ret = wd_init_ctx_config(&wd_ecc_setting.config, config); if (ret < 0) return ret; diff --git a/wd_join_gather.c b/wd_join_gather.c index a845e03..915c1b8 100644 --- a/wd_join_gather.c +++ b/wd_join_gather.c @@ -684,6 +684,7 @@ static int wd_join_gather_alg_init(struct wd_ctx_config *config, struct wd_sched if (ret < 0) return ret; + wd_join_gather_setting.config.alg_name = "hashjoin gather"; ret = wd_init_ctx_config(&wd_join_gather_setting.config, config); if (ret < 0) return ret; diff --git a/wd_rsa.c b/wd_rsa.c index 4cb72fc..cf9239c 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -166,6 +166,7 @@ static int wd_rsa_common_init(struct wd_ctx_config *config, struct wd_sched *sch if (ret < 0) return ret; + wd_rsa_setting.config.alg_name = "rsa"; ret = wd_init_ctx_config(&wd_rsa_setting.config, config); if (ret < 0) return ret; diff --git a/wd_udma.c b/wd_udma.c index 5f47291..eebe495 100644 --- a/wd_udma.c +++ b/wd_udma.c @@ -390,6 +390,7 @@ static int wd_udma_alg_init(struct wd_ctx_config *config, struct wd_sched *sched if (ret < 0) return ret; + wd_udma_setting.config.alg_name = "udma"; ret = wd_init_ctx_config(&wd_udma_setting.config, config); if (ret < 0) return ret; -- 2.33.0

From: Wenkai Lin <linwenkai6@hisilicon.com> When use sgl memory with uadk, we needs to convert the user SGL into a hardware SGL. For performance considerations, we creates an SGL pool for each queue, from which SGL are requested when performing tasks. In the nosva scenario, to obtain the mapping the relationship between virtual and physical addresses, SGL memory needs to be allocated from the memory pool. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- drv/hisi_comp.c | 4 +- drv/hisi_qm_udrv.c | 140 +++++++++++++++++++++++++++++++++++---------- drv/hisi_qm_udrv.h | 6 +- drv/hisi_sec.c | 6 +- 4 files changed, 118 insertions(+), 38 deletions(-) diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index 001150e..87d2103 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -369,7 +369,7 @@ static int get_sgl_from_pool(handle_t h_qp, struct comp_sgl *c_sgl) { handle_t h_sgl_pool; - h_sgl_pool = hisi_qm_get_sglpool(h_qp); + h_sgl_pool = hisi_qm_get_sglpool(h_qp, NULL); if (unlikely(!h_sgl_pool)) { WD_ERR("failed to get sglpool!\n"); return -WD_EINVAL; @@ -1426,7 +1426,7 @@ static void free_hw_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, void *hw_sgl_in, *hw_sgl_out; handle_t h_sgl_pool; - h_sgl_pool = hisi_qm_get_sglpool(h_qp); + h_sgl_pool = hisi_qm_get_sglpool(h_qp, NULL); if (unlikely(!h_sgl_pool)) { WD_ERR("failed to get sglpool to free hw sgl!\n"); return; diff --git a/drv/hisi_qm_udrv.c b/drv/hisi_qm_udrv.c index 70a6698..9251b4c 100644 --- a/drv/hisi_qm_udrv.c +++ b/drv/hisi_qm_udrv.c @@ -76,8 +76,7 @@ struct hisi_sge { void *page_ctrl; __le32 len; __le32 pad; - __le32 pad0; - __le32 pad1; + uintptr_t vbuff; }; /* use default hw sgl head size 64B, in little-endian */ @@ -91,7 +90,8 @@ struct hisi_sgl { /* the sge num in this sgl */ __le16 entry_length_in_sgl; __le16 pad0; - __le64 pad1[5]; + __le64 pad1[4]; + struct hisi_sgl *next; /* valid sge buffs total size */ __le64 entry_size_in_sgl; struct hisi_sge sge_entries[]; @@ -107,6 +107,7 @@ struct hisi_sgl_pool { __u32 top; __u32 sge_num; __u32 sgl_num; + struct wd_mm_ops *mm_ops; pthread_spinlock_t lock; }; @@ -357,8 +358,16 @@ static int hisi_qm_setup_info(struct hisi_qp *qp, struct hisi_qm_priv *config) goto err_destroy_lock; } + ret = pthread_spin_init(&q_info->sgl_lock, PTHREAD_PROCESS_SHARED); + if (ret) { + WD_DEV_ERR(qp->h_ctx, "failed to init qinfo sgl_lock!\n"); + goto err_destroy_sd_lock; + } + return 0; +err_destroy_sd_lock: + pthread_spin_destroy(&q_info->sd_lock); err_destroy_lock: pthread_spin_destroy(&q_info->rv_lock); err_out: @@ -370,6 +379,7 @@ static void hisi_qm_clear_info(struct hisi_qp *qp) { struct hisi_qm_queue_info *q_info = &qp->q_info; + pthread_spin_destroy(&q_info->sgl_lock); pthread_spin_destroy(&q_info->sd_lock); pthread_spin_destroy(&q_info->rv_lock); hisi_qm_unset_region(qp->h_ctx, q_info); @@ -413,7 +423,7 @@ handle_t hisi_qm_alloc_qp(struct hisi_qm_priv *config, handle_t ctx) goto out_qp; qp->h_sgl_pool = hisi_qm_create_sglpool(HISI_SGL_NUM_IN_BD, - HISI_SGE_NUM_IN_SGL); + HISI_SGE_NUM_IN_SGL, NULL); if (!qp->h_sgl_pool) goto free_info; @@ -451,6 +461,8 @@ void hisi_qm_free_qp(handle_t h_qp) wd_release_ctx_force(qp->h_ctx); hisi_qm_destroy_sglpool(qp->h_sgl_pool); + if (qp->h_nosva_sgl_pool) + hisi_qm_destroy_sglpool(qp->h_nosva_sgl_pool); hisi_qm_clear_info(qp); @@ -630,32 +642,46 @@ void hisi_set_msg_id(handle_t h_qp, __u32 *tag) } } -static void *hisi_qm_create_sgl(__u32 sge_num) +static void *hisi_qm_create_sgl(__u32 sge_num, struct wd_mm_ops *mm_ops) { void *sgl; int size; size = sizeof(struct hisi_sgl) + sge_num * (sizeof(struct hisi_sge)) + HISI_SGL_ALIGE; - sgl = calloc(1, size); - if (!sgl) { - WD_ERR("failed to create sgl!\n"); - return NULL; - } + if (mm_ops) + sgl = mm_ops->alloc(mm_ops->usr, size); + else + sgl = calloc(1, size); + + if (!sgl) + WD_ERR("failed to alloc memory for the hisi qm sgl!\n"); return sgl; } -static struct hisi_sgl *hisi_qm_align_sgl(const void *sgl, __u32 sge_num) +static struct hisi_sgl *hisi_qm_align_sgl(const void *sgl, __u32 sge_num, + struct wd_mm_ops *mm_ops) { struct hisi_sgl *sgl_align; + uintptr_t iova, iova_align; /* Hardware require the address must be 64 bytes aligned */ - sgl_align = (struct hisi_sgl *)ADDR_ALIGN_64(sgl); + if (mm_ops) { + iova = (uintptr_t)mm_ops->iova_map(mm_ops->usr, (struct hisi_sgl *)sgl, + sizeof(struct hisi_sgl)); + if (!iova) + return NULL; + iova_align = ADDR_ALIGN_64(iova); + sgl_align = (struct hisi_sgl *)((uintptr_t)sgl + iova_align - iova); + } else { + sgl_align = (struct hisi_sgl *)ADDR_ALIGN_64(sgl); + } sgl_align->entry_sum_in_chain = sge_num; sgl_align->entry_sum_in_sgl = 0; sgl_align->entry_length_in_sgl = sge_num; sgl_align->next_dma = 0; + sgl_align->next = 0; return sgl_align; } @@ -665,18 +691,27 @@ static void hisi_qm_free_sglpool(struct hisi_sgl_pool *pool) __u32 i; if (pool->sgl) { - for (i = 0; i < pool->sgl_num; i++) - free(pool->sgl[i]); + if (pool->mm_ops && !pool->mm_ops->sva_mode) { + for (i = 0; i < pool->sgl_num; i++) + pool->mm_ops->free(pool->mm_ops->usr, pool->sgl[i]); + } else { + for (i = 0; i < pool->sgl_num; i++) + free(pool->sgl[i]); + } free(pool->sgl); } if (pool->sgl_align) free(pool->sgl_align); + + if (pool->mm_ops) + free(pool->mm_ops); + free(pool); } -handle_t hisi_qm_create_sglpool(__u32 sgl_num, __u32 sge_num) +handle_t hisi_qm_create_sglpool(__u32 sgl_num, __u32 sge_num, struct wd_mm_ops *mm_ops) { struct hisi_sgl_pool *sgl_pool; int ret; @@ -694,6 +729,16 @@ handle_t hisi_qm_create_sglpool(__u32 sgl_num, __u32 sge_num) return 0; } + if (mm_ops && !mm_ops->sva_mode && mm_ops->alloc && mm_ops->free && + mm_ops->iova_map && mm_ops->usr) { + sgl_pool->mm_ops = malloc(sizeof(struct wd_mm_ops)); + if (!sgl_pool->mm_ops) { + WD_ERR("failed to alloc memory for sglpool mm_ops!\n"); + goto err_out; + } + memcpy(sgl_pool->mm_ops, mm_ops, sizeof(struct wd_mm_ops)); + } + sgl_pool->sgl = calloc(sgl_num, sizeof(void *)); if (!sgl_pool->sgl) { WD_ERR("failed to alloc memory for sgl!\n"); @@ -708,14 +753,14 @@ handle_t hisi_qm_create_sglpool(__u32 sgl_num, __u32 sge_num) /* base the sgl_num create the sgl chain */ for (i = 0; i < sgl_num; i++) { - sgl_pool->sgl[i] = hisi_qm_create_sgl(sge_num); + sgl_pool->sgl[i] = hisi_qm_create_sgl(sge_num, sgl_pool->mm_ops); if (!sgl_pool->sgl[i]) { sgl_pool->sgl_num = i; goto err_out; } sgl_pool->sgl_align[i] = hisi_qm_align_sgl(sgl_pool->sgl[i], - sge_num); + sge_num, sgl_pool->mm_ops); } sgl_pool->sgl_num = sgl_num; @@ -756,8 +801,8 @@ static struct hisi_sgl *hisi_qm_sgl_pop(struct hisi_sgl_pool *pool) pthread_spin_lock(&pool->lock); if (pool->top == 0) { - WD_ERR("invalid: the sgl pool is empty!\n"); pthread_spin_unlock(&pool->lock); + WD_ERR("invalid: the sgl pool is empty!\n"); return NULL; } @@ -771,12 +816,13 @@ static int hisi_qm_sgl_push(struct hisi_sgl_pool *pool, struct hisi_sgl *hw_sgl) { pthread_spin_lock(&pool->lock); if (pool->top >= pool->depth) { - WD_ERR("invalid: the sgl pool is full!\n"); pthread_spin_unlock(&pool->lock); + WD_ERR("invalid: the sgl pool is full!\n"); return -WD_EINVAL; } hw_sgl->next_dma = 0; + hw_sgl->next = 0; hw_sgl->entry_sum_in_sgl = 0; hw_sgl->entry_sum_in_chain = pool->sge_num; hw_sgl->entry_length_in_sgl = pool->sge_num; @@ -800,7 +846,7 @@ void hisi_qm_put_hw_sgl(handle_t sgl_pool, void *hw_sgl) return; while (cur) { - next = (struct hisi_sgl *)cur->next_dma; + next = (struct hisi_sgl *)cur->next; ret = hisi_qm_sgl_push(pool, cur); if (ret) break; @@ -832,7 +878,7 @@ static void hisi_qm_dump_sgl(void *sgl) WD_DEBUG("[sgl-%d]->sge_entries[%d].len: %u\n", k, i, tmp->sge_entries[i].len); - tmp = (struct hisi_sgl *)tmp->next_dma; + tmp = (struct hisi_sgl *)tmp->next; k++; if (!tmp) { @@ -847,6 +893,7 @@ void *hisi_qm_get_hw_sgl(handle_t sgl_pool, struct wd_datalist *sgl) struct hisi_sgl_pool *pool = (struct hisi_sgl_pool *)sgl_pool; struct wd_datalist *tmp = sgl; struct hisi_sgl *head, *next, *cur; + struct wd_mm_ops *mm_ops; __u32 i = 0; if (!pool || !sgl) { @@ -854,6 +901,12 @@ void *hisi_qm_get_hw_sgl(handle_t sgl_pool, struct wd_datalist *sgl) return NULL; } + if (pool->mm_ops && !pool->mm_ops->iova_map) { + WD_ERR("invalid: mm_ops iova_map function is NULL!\n"); + return NULL; + } + + mm_ops = pool->mm_ops; head = hisi_qm_sgl_pop(pool); if (!head) return NULL; @@ -872,7 +925,18 @@ void *hisi_qm_get_hw_sgl(handle_t sgl_pool, struct wd_datalist *sgl) goto err_out; } - cur->sge_entries[i].buff = (uintptr_t)tmp->data; + if (mm_ops) + cur->sge_entries[i].buff = (uintptr_t)mm_ops->iova_map(mm_ops->usr, + tmp->data, tmp->len); + else + cur->sge_entries[i].buff = (uintptr_t)tmp->data; + + if (!cur->sge_entries[i].buff) { + WD_ERR("invalid: the iova map addr of sge is NULL!\n"); + goto err_out; + } + + cur->sge_entries[i].vbuff = (uintptr_t)tmp->data; cur->sge_entries[i].len = tmp->len; cur->entry_sum_in_sgl++; cur->entry_size_in_sgl += tmp->len; @@ -890,7 +954,12 @@ void *hisi_qm_get_hw_sgl(handle_t sgl_pool, struct wd_datalist *sgl) WD_ERR("invalid: the sgl pool is not enough!\n"); goto err_out; } - cur->next_dma = (uintptr_t)next; + if (mm_ops) + cur->next_dma = (uintptr_t)mm_ops->iova_map(mm_ops->usr, + next, sizeof(*next)); + else + cur->next_dma = (uintptr_t)next; + cur->next = next; cur = next; head->entry_sum_in_chain += pool->sge_num; /* In the new sgl chain, the subscript must be reset */ @@ -912,10 +981,19 @@ err_out: return NULL; } -handle_t hisi_qm_get_sglpool(handle_t h_qp) +handle_t hisi_qm_get_sglpool(handle_t h_qp, struct wd_mm_ops *mm_ops) { struct hisi_qp *qp = (struct hisi_qp *)h_qp; + if (mm_ops && !mm_ops->sva_mode) { + pthread_spin_lock(&qp->q_info.sgl_lock); + if (!qp->h_nosva_sgl_pool) + qp->h_nosva_sgl_pool = hisi_qm_create_sglpool(HISI_SGL_NUM_IN_BD, + HISI_SGE_NUM_IN_SGL, mm_ops); + pthread_spin_unlock(&qp->q_info.sgl_lock); + return qp->h_nosva_sgl_pool; + } + return qp->h_sgl_pool; } @@ -927,7 +1005,7 @@ static void hisi_qm_sgl_copy_inner(void *pbuff, struct hisi_sgl *hw_sgl, __u32 offset; void *src; - src = (void *)tmp->sge_entries[begin_sge].buff + sge_offset; + src = (void *)tmp->sge_entries[begin_sge].vbuff + sge_offset; offset = tmp->sge_entries[begin_sge].len - sge_offset; /* the first one is enough for copy size, copy and return */ if (offset >= size) { @@ -939,7 +1017,7 @@ static void hisi_qm_sgl_copy_inner(void *pbuff, struct hisi_sgl *hw_sgl, while (tmp) { for (; i < tmp->entry_sum_in_sgl; i++) { - src = (void *)tmp->sge_entries[i].buff; + src = (void *)tmp->sge_entries[i].vbuff; if (offset + tmp->sge_entries[i].len >= size) { memcpy(pbuff + offset, src, size - offset); return; @@ -949,7 +1027,7 @@ static void hisi_qm_sgl_copy_inner(void *pbuff, struct hisi_sgl *hw_sgl, offset += tmp->sge_entries[i].len; } - tmp = (struct hisi_sgl *)tmp->next_dma; + tmp = (struct hisi_sgl *)tmp->next; i = 0; } } @@ -964,14 +1042,14 @@ static void hisi_qm_pbuff_copy_inner(void *pbuff, struct hisi_sgl *hw_sgl, void *dst; if (tmp->sge_entries[begin_sge].len - sge_offset >= size) { - dst = (void *)tmp->sge_entries[begin_sge].buff + sge_offset; + dst = (void *)tmp->sge_entries[begin_sge].vbuff + sge_offset; memcpy(dst, pbuff, size); return; } while (tmp) { for (; i < tmp->entry_sum_in_sgl; i++) { - dst = (void *)tmp->sge_entries[i].buff; + dst = (void *)tmp->sge_entries[i].vbuff; if (offset + tmp->sge_entries[i].len >= size) { memcpy(dst, pbuff + offset, size - offset); return; @@ -981,7 +1059,7 @@ static void hisi_qm_pbuff_copy_inner(void *pbuff, struct hisi_sgl *hw_sgl, offset += tmp->sge_entries[i].len; } - tmp = (struct hisi_sgl *)tmp->next_dma; + tmp = (struct hisi_sgl *)tmp->next; i = 0; } } @@ -1000,7 +1078,7 @@ void hisi_qm_sgl_copy(void *pbuff, void *hw_sgl, __u32 offset, __u32 size, while (len + tmp->entry_size_in_sgl <= offset) { len += tmp->entry_size_in_sgl; - tmp = (struct hisi_sgl *)tmp->next_dma; + tmp = (struct hisi_sgl *)tmp->next; if (!tmp) return; } diff --git a/drv/hisi_qm_udrv.h b/drv/hisi_qm_udrv.h index e787193..f066881 100644 --- a/drv/hisi_qm_udrv.h +++ b/drv/hisi_qm_udrv.h @@ -81,6 +81,7 @@ struct hisi_qm_queue_info { pthread_spinlock_t rv_lock; unsigned long region_size[UACCE_QFRT_MAX]; bool epoll_en; + pthread_spinlock_t sgl_lock; }; struct hisi_qp { @@ -89,6 +90,7 @@ struct hisi_qp { handle_t h_ctx; /* Private area for driver use, point to queue specifial data */ void *priv; + handle_t h_nosva_sgl_pool; }; /* Capabilities */ @@ -148,7 +150,7 @@ void hisi_set_msg_id(handle_t h_qp, __u32 *tag); * * Fixed me: the sge buff's size now is Fixed. */ -handle_t hisi_qm_create_sglpool(__u32 sgl_num, __u32 sge_num); +handle_t hisi_qm_create_sglpool(__u32 sgl_num, __u32 sge_num, struct wd_mm_ops *mm_ops); /** * hisi_qm_destroy_sglpool - Destroy sgl pool in qm. @@ -176,7 +178,7 @@ void hisi_qm_put_hw_sgl(handle_t sgl_pool, void *hw_sgl); * hisi_qm_get_sglpool - Get the qp's hw sgl pool handle * @h_qp: Handle of the qp. */ -handle_t hisi_qm_get_sglpool(handle_t h_qp); +handle_t hisi_qm_get_sglpool(handle_t h_qp, struct wd_mm_ops *mm_ops); /** * hisi_qm_sgl_copy: Buffer copying from hw sgl to pbuff or pbuff to sgl diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 1e95ce5..246edf4 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -1028,7 +1028,7 @@ static void hisi_sec_put_sgl(handle_t h_qp, __u8 alg_type, void *in, void *out) { handle_t h_sgl_pool; - h_sgl_pool = hisi_qm_get_sglpool(h_qp); + h_sgl_pool = hisi_qm_get_sglpool(h_qp, NULL); if (!h_sgl_pool) return; @@ -1045,7 +1045,7 @@ static int hisi_sec_fill_sgl(handle_t h_qp, __u8 **in, __u8 **out, void *hw_sgl_in; void *hw_sgl_out; - h_sgl_pool = hisi_qm_get_sglpool(h_qp); + h_sgl_pool = hisi_qm_get_sglpool(h_qp, NULL); if (!h_sgl_pool) { WD_ERR("failed to get sglpool for hw_v2!\n"); return -WD_EINVAL; @@ -1085,7 +1085,7 @@ static int hisi_sec_fill_sgl_v3(handle_t h_qp, __u8 **in, __u8 **out, void *hw_sgl_in; void *hw_sgl_out; - h_sgl_pool = hisi_qm_get_sglpool(h_qp); + h_sgl_pool = hisi_qm_get_sglpool(h_qp, NULL); if (!h_sgl_pool) { WD_ERR("failed to get sglpool for hw_v3!\n"); return -WD_EINVAL; -- 2.33.0

From: Wenkai Lin <linwenkai6@hisilicon.com> Modify GATHER_ROW_BATCH_EMPTY_SIZE from 2 to 4 for hardware memory alignment. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- drv/hisi_dae_join_gather.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drv/hisi_dae_join_gather.c b/drv/hisi_dae_join_gather.c index dcbeed4..f3eeb95 100644 --- a/drv/hisi_dae_join_gather.c +++ b/drv/hisi_dae_join_gather.c @@ -29,7 +29,7 @@ #define HASH_TABLE_MAX_INDEX_NUM 15 #define HASH_TABLE_INDEX_SIZE 12 #define HASH_TABLE_EMPTY_SIZE 4 -#define GATHER_ROW_BATCH_EMPTY_SIZE 2 +#define GATHER_ROW_BATCH_EMPTY_SIZE 4 /* DAE hardware protocol data */ enum dae_join_stage { -- 2.33.0

From: Zhushuai Yin <yinzhushuai@huawei.com> The SEC v2 interface needs to support the no-sva business model within the new evolution framework. This ensures that users switching to the v2 interface can utilize both the sva mode and the no-sva mode. Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- drv/hisi_sec.c | 1081 ++++++++++++++++++++++++++++++----- include/drv/wd_aead_drv.h | 23 +- include/drv/wd_cipher_drv.h | 2 + include/drv/wd_digest_drv.h | 2 + include/wd_aead.h | 2 + include/wd_cipher.h | 2 + include/wd_digest.h | 2 + wd_aead.c | 179 +++++- wd_cipher.c | 40 +- wd_digest.c | 42 +- 10 files changed, 1207 insertions(+), 168 deletions(-) diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 246edf4..0074952 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -91,6 +91,7 @@ #define AUTH_ALG_OFFSET 11 #define WD_CIPHER_THEN_DIGEST 0x0 #define WD_DIGEST_THEN_CIPHER 0x1 +#define AEAD_AIV_OFFSET 0x6 #define SEC_CTX_Q_NUM_DEF 1 @@ -602,6 +603,157 @@ static int hisi_sec_get_usage(void *param) return 0; } +static int eops_param_check(struct wd_alg_driver *drv, struct wd_mm_ops *mm_ops) +{ + if (!drv || !drv->priv) { + WD_ERR("invalid: aead drv or priv is NULL!\n"); + return -WD_EINVAL; + } + + if (!mm_ops) { + WD_ERR("invalid: mm_ops is NULL!\n"); + return -WD_EINVAL; + } + + return WD_SUCCESS; +} + +static int aead_sess_eops_init(struct wd_alg_driver *drv, + struct wd_mm_ops *mm_ops, void **params) +{ + struct wd_aead_aiv_addr *aiv_addr; + struct hisi_sec_ctx *sec_ctx; + struct hisi_qp *qp; + __u16 sq_depth; + int ret; + + ret = eops_param_check(drv, mm_ops); + if (ret) + return ret; + + if (!params) { + WD_ERR("invalid: extend ops init params address is NULL!\n"); + return -WD_EINVAL; + } + + if (*params) { + WD_ERR("invalid: extend ops init params repeatedly!\n"); + return -WD_EINVAL; + } + + aiv_addr = calloc(1, sizeof(struct wd_aead_aiv_addr)); + if (!aiv_addr) { + WD_ERR("aead failed to alloc aiv_addr memory!\n"); + return -WD_ENOMEM; + } + + sec_ctx = (struct hisi_sec_ctx *)drv->priv; + qp = (struct hisi_qp *)wd_ctx_get_priv(sec_ctx->config.ctxs[0].ctx); + sq_depth = qp->q_info.sq_depth; + aiv_addr->aiv = mm_ops->alloc(mm_ops->usr, (__u32)sq_depth << AEAD_AIV_OFFSET); + if (!aiv_addr->aiv) { + WD_ERR("aead failed to alloc aiv memory!\n"); + goto aiv_err; + } + memset(aiv_addr->aiv, 0, (__u32)sq_depth << AEAD_AIV_OFFSET); + if (!mm_ops->sva_mode) { + aiv_addr->aiv_nosva = mm_ops->iova_map(mm_ops->usr, aiv_addr->aiv, + (__u32)sq_depth << AEAD_AIV_OFFSET); + if (!aiv_addr->aiv_nosva) + goto aiv_nosva_err; + } + + aiv_addr->aiv_status = calloc(1, sq_depth); + if (!aiv_addr->aiv_status) { + WD_ERR("aead failed to alloc aiv_status memory!\n"); + goto aiv_status_err; + } + + *params = aiv_addr; + + return WD_SUCCESS; + +aiv_status_err: + if (!mm_ops->sva_mode) + mm_ops->iova_unmap(mm_ops->usr, aiv_addr->aiv, (void *)aiv_addr->aiv_nosva, + (__u32)sq_depth << AEAD_AIV_OFFSET); +aiv_nosva_err: + mm_ops->free(mm_ops->usr, aiv_addr->aiv); +aiv_err: + free(aiv_addr); + return -WD_ENOMEM; +} + +static void aead_sess_eops_uninit(struct wd_alg_driver *drv, + struct wd_mm_ops *mm_ops, void *params) +{ + struct wd_aead_aiv_addr *aiv_addr; + struct hisi_sec_ctx *sec_ctx; + struct hisi_qp *qp; + __u16 sq_depth; + int ret; + + ret = eops_param_check(drv, mm_ops); + if (ret) + return; + + if (!params) { + WD_ERR("invalid: extend ops uninit params address is NULL!\n"); + return; + } + + sec_ctx = (struct hisi_sec_ctx *)drv->priv; + qp = (struct hisi_qp *)wd_ctx_get_priv(sec_ctx->config.ctxs[0].ctx); + sq_depth = qp->q_info.sq_depth; + + aiv_addr = (struct wd_aead_aiv_addr *)params; + if (!mm_ops->sva_mode) + mm_ops->iova_unmap(mm_ops->usr, aiv_addr->aiv, (void *)aiv_addr->aiv_nosva, + (__u32)sq_depth << AEAD_AIV_OFFSET); + mm_ops->free(mm_ops->usr, aiv_addr->aiv); + free(aiv_addr->aiv_status); + free(params); +} + +static int aead_get_aiv_addr(struct hisi_qp *qp, struct wd_aead_msg *msg) +{ + struct wd_aead_aiv_addr *aiv_addr = (struct wd_aead_aiv_addr *)msg->drv_cfg; + __u16 sq_depth = qp->q_info.sq_depth; + int i; + + for (i = 0; i < sq_depth; i++) { + if (!__atomic_test_and_set(&aiv_addr->aiv_status[i], __ATOMIC_ACQUIRE)) { + msg->aiv = aiv_addr->aiv + i * AIV_STREAM_LEN; + return i; + } + } + + return -WD_EBUSY; +} + +static void aead_free_aiv_addr(struct wd_aead_msg *msg) +{ + struct wd_aead_aiv_addr *aiv_addr = (struct wd_aead_aiv_addr *)msg->drv_cfg; + __u32 aiv_idx; + + aiv_idx = (msg->aiv - aiv_addr->aiv) >> AEAD_AIV_OFFSET; + __atomic_clear(&aiv_addr->aiv_status[aiv_idx], __ATOMIC_RELEASE); +} + +static int sec_aead_get_extend_ops(void *ops) +{ + struct wd_aead_extend_ops *aead_ops = (struct wd_aead_extend_ops *)ops; + + if (!aead_ops) + return -WD_EINVAL; + + aead_ops->params = NULL; + aead_ops->eops_aiv_init = aead_sess_eops_init; + aead_ops->eops_aiv_uninit = aead_sess_eops_uninit; + + return WD_SUCCESS; +} + #define GEN_SEC_ALG_DRIVER(sec_alg_name, alg_type) \ {\ .drv_name = "hisi_sec2",\ @@ -616,6 +768,7 @@ static int hisi_sec_get_usage(void *param) .send = alg_type##_send,\ .recv = alg_type##_recv,\ .get_usage = hisi_sec_get_usage,\ + .get_extend_ops = sec_aead_get_extend_ops,\ } static struct wd_alg_driver cipher_alg_driver[] = { @@ -914,13 +1067,83 @@ static int fill_cipher_bd2_mode(struct wd_cipher_msg *msg, return 0; } -static void fill_cipher_bd2_addr(struct wd_cipher_msg *msg, - struct hisi_sec_sqe *sqe) +static void destroy_cipher_bd2_addr(struct wd_cipher_msg *msg, struct hisi_sec_sqe *sqe) { - sqe->type2.data_src_addr = (__u64)(uintptr_t)msg->in; - sqe->type2.data_dst_addr = (__u64)(uintptr_t)msg->out; - sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; - sqe->type2.c_key_addr = (__u64)(uintptr_t)msg->key; + struct wd_mm_ops *mm_ops = msg->mm_ops; + void *mempool; + + /* SVA mode and skip */ + if (!mm_ops || mm_ops->sva_mode) + return; + + if (!mm_ops->usr) { + WD_ERR("cipher failed to check memory pool!\n"); + return; + } + + mempool = mm_ops->usr; + if (sqe->type2.data_src_addr) + mm_ops->iova_unmap(mempool, msg->in, (void *)(uintptr_t)sqe->type2.data_src_addr, + msg->in_bytes); + + if (sqe->type2.data_dst_addr) + mm_ops->iova_unmap(mempool, msg->out, (void *)(uintptr_t)sqe->type2.data_dst_addr, + msg->out_bytes); + + if (sqe->type2.c_key_addr) + mm_ops->iova_unmap(mempool, msg->key, (void *)(uintptr_t)sqe->type2.c_key_addr, + msg->key_bytes); + + if (sqe->type2.c_ivin_addr) + mm_ops->iova_unmap(mempool, msg->iv, (void *)(uintptr_t)sqe->type2.c_ivin_addr, + msg->iv_bytes); +} + +static int fill_cipher_bd2_addr(struct wd_cipher_msg *msg, struct hisi_sec_sqe *sqe) +{ + struct wd_mm_ops *mm_ops = msg->mm_ops; + void *mempool, *phy_addr; + + if (mm_ops->sva_mode) { + sqe->type2.data_src_addr = (__u64)(uintptr_t)msg->in; + sqe->type2.data_dst_addr = (__u64)(uintptr_t)msg->out; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->type2.c_key_addr = (__u64)(uintptr_t)msg->key; + return 0; + } + if (msg->mm_type > UADK_MEM_PROXY) { + WD_ERR("cipher failed to check memory type!\n"); + return -WD_EINVAL; + } + + /* No-SVA mode and Memory is USER mode or PROXY mode */ + mempool = mm_ops->usr; + phy_addr = mm_ops->iova_map(mempool, msg->in, msg->in_bytes); + if (!phy_addr) + return -WD_ENOMEM; + sqe->type2.data_src_addr = (__u64)(uintptr_t)phy_addr; + phy_addr = mm_ops->iova_map(mempool, msg->out, msg->out_bytes); + if (!phy_addr) + goto map_err; + sqe->type2.data_dst_addr = (__u64)(uintptr_t)phy_addr; + if (msg->iv_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->iv, msg->iv_bytes); + if (!phy_addr) + goto map_err; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)phy_addr; + } + if (msg->key_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->key, msg->key_bytes); + if (!phy_addr) + goto map_err; + sqe->type2.c_key_addr = (__u64)(uintptr_t)phy_addr; + } + + return 0; + +map_err: + destroy_cipher_bd2_addr(msg, sqe); + return -WD_ENOMEM; } static void parse_cipher_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, @@ -945,8 +1168,6 @@ static void parse_cipher_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { recv_msg->alg_type = WD_CIPHER; recv_msg->data_fmt = get_data_fmt_v2(sqe->sds_sa_type); - recv_msg->in = (__u8 *)(uintptr_t)sqe->type2.data_src_addr; - recv_msg->out = (__u8 *)(uintptr_t)sqe->type2.data_dst_addr; temp_msg = wd_cipher_get_msg(qp->q_info.idx, tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; @@ -954,6 +1175,9 @@ static void parse_cipher_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, qp->q_info.idx, tag); return; } + recv_msg->in = temp_msg->in; + recv_msg->out = temp_msg->out; + recv_msg->mm_ops = temp_msg->mm_ops; } else { /* The synchronization mode uses the same message */ temp_msg = recv_msg; @@ -964,6 +1188,8 @@ static void parse_cipher_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, else update_iv_sgl(temp_msg); + destroy_cipher_bd2_addr(temp_msg, sqe); + if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "cipher"); } @@ -1024,11 +1250,12 @@ static int cipher_len_check(struct wd_cipher_msg *msg) return 0; } -static void hisi_sec_put_sgl(handle_t h_qp, __u8 alg_type, void *in, void *out) +static void hisi_sec_put_sgl(handle_t h_qp, __u8 alg_type, void *in, void *out, + struct wd_mm_ops *mm_ops) { handle_t h_sgl_pool; - h_sgl_pool = hisi_qm_get_sglpool(h_qp, NULL); + h_sgl_pool = hisi_qm_get_sglpool(h_qp, mm_ops); if (!h_sgl_pool) return; @@ -1038,19 +1265,12 @@ static void hisi_sec_put_sgl(handle_t h_qp, __u8 alg_type, void *in, void *out) hisi_qm_put_hw_sgl(h_sgl_pool, out); } -static int hisi_sec_fill_sgl(handle_t h_qp, __u8 **in, __u8 **out, - struct hisi_sec_sqe *sqe, __u8 type) +static int hisi_sec_fill_sgl(handle_t h_sgl_pool, __u8 **in, __u8 **out, + struct hisi_sec_sqe *sqe, __u8 type) { - handle_t h_sgl_pool; void *hw_sgl_in; void *hw_sgl_out; - h_sgl_pool = hisi_qm_get_sglpool(h_qp, NULL); - if (!h_sgl_pool) { - WD_ERR("failed to get sglpool for hw_v2!\n"); - return -WD_EINVAL; - } - hw_sgl_in = hisi_qm_get_hw_sgl(h_sgl_pool, (struct wd_datalist *)(*in)); if (!hw_sgl_in) { WD_ERR("failed to get sgl in for hw_v2!\n"); @@ -1078,19 +1298,12 @@ static int hisi_sec_fill_sgl(handle_t h_qp, __u8 **in, __u8 **out, return 0; } -static int hisi_sec_fill_sgl_v3(handle_t h_qp, __u8 **in, __u8 **out, +static int hisi_sec_fill_sgl_v3(handle_t h_sgl_pool, __u8 **in, __u8 **out, struct hisi_sec_sqe3 *sqe, __u8 type) { - handle_t h_sgl_pool; void *hw_sgl_in; void *hw_sgl_out; - h_sgl_pool = hisi_qm_get_sglpool(h_qp, NULL); - if (!h_sgl_pool) { - WD_ERR("failed to get sglpool for hw_v3!\n"); - return -WD_EINVAL; - } - hw_sgl_in = hisi_qm_get_hw_sgl(h_sgl_pool, (struct wd_datalist *)(*in)); if (!hw_sgl_in) { WD_ERR("failed to get sgl in for hw_v3!\n"); @@ -1165,6 +1378,7 @@ static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *w handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *msg = wd_msg; struct hisi_sec_sqe sqe; + handle_t h_sgl_pool; __u16 count = 0; int ret; @@ -1179,7 +1393,13 @@ static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *w return ret; if (msg->data_fmt == WD_SGL_BUF) { - ret = hisi_sec_fill_sgl(h_qp, &msg->in, &msg->out, &sqe, + h_sgl_pool = hisi_qm_get_sglpool(h_qp, msg->mm_ops); + if (!h_sgl_pool) { + WD_ERR("cipher failed to get sglpool for hw_v2!\n"); + return -WD_EINVAL; + } + + ret = hisi_sec_fill_sgl(h_sgl_pool, &msg->in, &msg->out, &sqe, msg->alg_type); if (ret) return ret; @@ -1188,7 +1408,11 @@ static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *w hisi_set_msg_id(h_qp, &msg->tag); sqe.type2.clen_ivhlen |= (__u32)msg->in_bytes; sqe.type2.tag = (__u16)msg->tag; - fill_cipher_bd2_addr(msg, &sqe); + ret = fill_cipher_bd2_addr(msg, &sqe); + if (ret < 0) { + WD_ERR("cipher map memory is err(%d)!\n", ret); + return ret; + } ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (ret < 0) { @@ -1197,8 +1421,8 @@ static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *w if (msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, - msg->out); - + msg->out, msg->mm_ops); + destroy_cipher_bd2_addr(msg, &sqe); return ret; } @@ -1225,7 +1449,7 @@ static int hisi_sec_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *w if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, - recv_msg->out); + recv_msg->out, recv_msg->mm_ops); return 0; } @@ -1316,13 +1540,83 @@ static int fill_cipher_bd3_mode(struct wd_cipher_msg *msg, return 0; } -static void fill_cipher_bd3_addr(struct wd_cipher_msg *msg, - struct hisi_sec_sqe3 *sqe) +static void destroy_cipher_bd3_addr(struct wd_cipher_msg *msg, struct hisi_sec_sqe3 *sqe) { - sqe->data_src_addr = (__u64)(uintptr_t)msg->in; - sqe->data_dst_addr = (__u64)(uintptr_t)msg->out; - sqe->no_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; - sqe->c_key_addr = (__u64)(uintptr_t)msg->key; + struct wd_mm_ops *mm_ops = msg->mm_ops; + void *mempool; + + /* SVA mode and skip */ + if (!mm_ops || mm_ops->sva_mode) + return; + + if (!mm_ops->usr) { + WD_ERR("cipher failed to check memory pool!\n"); + return; + } + + mempool = mm_ops->usr; + if (sqe->data_src_addr) + mm_ops->iova_unmap(mempool, msg->in, (void *)(uintptr_t)sqe->data_src_addr, + msg->in_bytes); + + if (sqe->data_dst_addr) + mm_ops->iova_unmap(mempool, msg->out, (void *)(uintptr_t)sqe->data_dst_addr, + msg->out_bytes); + + if (sqe->c_key_addr) + mm_ops->iova_unmap(mempool, msg->key, (void *)(uintptr_t)sqe->c_key_addr, + msg->key_bytes); + + if (sqe->no_scene.c_ivin_addr) + mm_ops->iova_unmap(mempool, msg->iv, + (void *)(uintptr_t)sqe->no_scene.c_ivin_addr, msg->iv_bytes); +} + +static int fill_cipher_bd3_addr(struct wd_cipher_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + struct wd_mm_ops *mm_ops = msg->mm_ops; + void *mempool, *phy_addr; + + if (mm_ops->sva_mode) { + sqe->data_src_addr = (__u64)(uintptr_t)msg->in; + sqe->data_dst_addr = (__u64)(uintptr_t)msg->out; + sqe->no_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->c_key_addr = (__u64)(uintptr_t)msg->key; + return 0; + } + if (msg->mm_type > UADK_MEM_PROXY) { + WD_ERR("cipher failed to check memory type!\n"); + return -WD_EINVAL; + } + + /* No-SVA mode and Memory is USER mode or PROXY mode */ + mempool = mm_ops->usr; + phy_addr = mm_ops->iova_map(mempool, msg->in, msg->in_bytes); + if (!phy_addr) + return -WD_ENOMEM; + sqe->data_src_addr = (__u64)(uintptr_t)phy_addr; + phy_addr = mm_ops->iova_map(mempool, msg->out, msg->out_bytes); + if (!phy_addr) + goto map_err; + sqe->data_dst_addr = (__u64)(uintptr_t)phy_addr; + if (msg->iv_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->iv, msg->iv_bytes); + if (!phy_addr) + goto map_err; + sqe->no_scene.c_ivin_addr = (__u64)(uintptr_t)phy_addr; + } + if (msg->key_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->key, msg->key_bytes); + if (!phy_addr) + goto map_err; + sqe->c_key_addr = (__u64)(uintptr_t)phy_addr; + } + + return 0; + +map_err: + destroy_cipher_bd3_addr(msg, sqe); + return -WD_ENOMEM; } static int fill_cipher_bd3(struct wd_cipher_msg *msg, struct hisi_sec_sqe3 *sqe) @@ -1361,8 +1655,12 @@ static int fill_cipher_bd3(struct wd_cipher_msg *msg, struct hisi_sec_sqe3 *sqe) return 0; } -static void fill_sec_prefetch(__u8 data_fmt, __u32 len, __u16 hw_type, struct hisi_sec_sqe3 *sqe) +static void fill_sec_prefetch(__u8 data_fmt, __u32 len, __u16 hw_type, struct hisi_sec_sqe3 *sqe, + bool sva_mode) { + if (!sva_mode) + return; + if (hw_type >= HISI_QM_API_VER5_BASE || (data_fmt == WD_FLAT_BUF && len <= SEC_SVA_PREFETCH_MAX_LEN)) sqe->auth_mac_key |= (__u32)SEC_ENABLE_SVA_PREFETCH << SEC_SVA_PREFETCH_OFFSET; @@ -1374,6 +1672,7 @@ static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct wd_cipher_msg *msg = wd_msg; struct hisi_sec_sqe3 sqe; + handle_t h_sgl_pool; __u16 count = 0; int ret; @@ -1387,10 +1686,16 @@ static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void if (ret) return ret; - fill_sec_prefetch(msg->data_fmt, msg->in_bytes, qp->q_info.hw_type, &sqe); + fill_sec_prefetch(msg->data_fmt, msg->in_bytes, qp->q_info.hw_type, &sqe, + msg->mm_ops->sva_mode); if (msg->data_fmt == WD_SGL_BUF) { - ret = hisi_sec_fill_sgl_v3(h_qp, &msg->in, &msg->out, &sqe, + h_sgl_pool = hisi_qm_get_sglpool(h_qp, msg->mm_ops); + if (!h_sgl_pool) { + WD_ERR("cipher failed to get sglpool for hw_v3!\n"); + return -WD_EINVAL; + } + ret = hisi_sec_fill_sgl_v3(h_sgl_pool, &msg->in, &msg->out, &sqe, msg->alg_type); if (ret) return ret; @@ -1399,7 +1704,11 @@ static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void hisi_set_msg_id(h_qp, &msg->tag); sqe.c_len_ivin = (__u32)msg->in_bytes; sqe.tag = (__u64)(uintptr_t)msg->tag; - fill_cipher_bd3_addr(msg, &sqe); + ret = fill_cipher_bd3_addr(msg, &sqe); + if (ret < 0) { + WD_ERR("cipher map memory is err(%d)!\n", ret); + return ret; + } ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (ret < 0) { @@ -1408,8 +1717,8 @@ static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void if (msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, - msg->out); - + msg->out, msg->mm_ops); + destroy_cipher_bd3_addr(msg, &sqe); return ret; } @@ -1438,8 +1747,6 @@ static void parse_cipher_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { recv_msg->alg_type = WD_CIPHER; recv_msg->data_fmt = get_data_fmt_v3(sqe->bd_param); - recv_msg->in = (__u8 *)(uintptr_t)sqe->data_src_addr; - recv_msg->out = (__u8 *)(uintptr_t)sqe->data_dst_addr; temp_msg = wd_cipher_get_msg(qp->q_info.idx, tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; @@ -1447,6 +1754,9 @@ static void parse_cipher_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, qp->q_info.idx, tag); return; } + recv_msg->in = temp_msg->in; + recv_msg->out = temp_msg->out; + recv_msg->mm_ops = temp_msg->mm_ops; } else { /* The synchronization mode uses the same message */ temp_msg = recv_msg; @@ -1457,6 +1767,8 @@ static void parse_cipher_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, else update_iv_sgl(temp_msg); + destroy_cipher_bd3_addr(temp_msg, sqe); + if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "cipher"); } @@ -1481,7 +1793,7 @@ static int hisi_sec_cipher_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, - recv_msg->out); + recv_msg->out, recv_msg->mm_ops); return 0; } @@ -1519,7 +1831,6 @@ static int fill_digest_bd2_alg(struct wd_digest_msg *msg, return -WD_EINVAL; } sqe->type2.mac_key_alg |= (__u32)BYTES_TO_WORDS(msg->key_bytes) << MAC_LEN_OFFSET; - sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->key; sqe->type2.mac_key_alg |= g_hmac_a_alg[msg->alg] << AUTH_ALG_OFFSET; @@ -1586,6 +1897,83 @@ static int fill_digest_long_hash(handle_t h_qp, struct wd_digest_msg *msg, return 0; } +static void destroy_digest_bd2_addr(struct wd_digest_msg *msg, struct hisi_sec_sqe *sqe) +{ + struct wd_mm_ops *mm_ops = msg->mm_ops; + void *mempool; + + /* SVA mode and skip */ + if (!mm_ops || mm_ops->sva_mode) + return; + + if (!mm_ops->usr) { + WD_ERR("digest failed to check memory pool!\n"); + return; + } + + mempool = mm_ops->usr; + + if (sqe->type2.data_src_addr) + mm_ops->iova_unmap(mempool, msg->in, (void *)(uintptr_t)sqe->type2.data_src_addr, + msg->in_bytes); + + if (sqe->type2.mac_addr) + mm_ops->iova_unmap(mempool, msg->out, (void *)(uintptr_t)sqe->type2.mac_addr, + msg->out_bytes); + + if (sqe->type2.a_key_addr && msg->mode == WD_DIGEST_HMAC) + mm_ops->iova_unmap(mempool, msg->key, (void *)(uintptr_t)sqe->type2.a_key_addr, + msg->key_bytes); +} + +static int fill_digest_bd2_addr(struct wd_digest_msg *msg, struct hisi_sec_sqe *sqe) +{ + struct wd_mm_ops *mm_ops = msg->mm_ops; + void *mempool, *phy_addr; + + if (mm_ops->sva_mode) { + /* avoid HW accessing address 0 when the pointer is NULL */ + if (msg->in) + sqe->type2.data_src_addr = (__u64)(uintptr_t)msg->in; + else + sqe->type2.data_src_addr = (__u64)(uintptr_t)msg->out; + sqe->type2.mac_addr = (__u64)(uintptr_t)msg->out; + if (msg->mode == WD_DIGEST_HMAC) + sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->key; + return 0; + } + if (msg->mm_type > UADK_MEM_PROXY) { + WD_ERR("digest failed to check memory type!\n"); + return -WD_EINVAL; + } + + /* No-SVA mode and Memory is USER mode or PROXY mode */ + mempool = mm_ops->usr; + if (msg->in_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->in, msg->in_bytes); + if (!phy_addr) + return -WD_ENOMEM; + sqe->type2.data_src_addr = (__u64)(uintptr_t)phy_addr; + } + phy_addr = mm_ops->iova_map(mempool, msg->out, msg->out_bytes); + if (!phy_addr) + goto map_err; + sqe->type2.mac_addr = (__u64)(uintptr_t)phy_addr; + + if (msg->key_bytes != 0 && msg->mode == WD_DIGEST_HMAC) { + phy_addr = mm_ops->iova_map(mempool, msg->key, msg->key_bytes); + if (!phy_addr) + goto map_err; + sqe->type2.a_key_addr = (__u64)(uintptr_t)phy_addr; + } + + return 0; + +map_err: + destroy_digest_bd2_addr(msg, sqe); + return -WD_ENOMEM; +} + static void parse_digest_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, struct wd_digest_msg *recv_msg) { @@ -1606,7 +1994,6 @@ static void parse_digest_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { recv_msg->alg_type = WD_DIGEST; recv_msg->data_fmt = get_data_fmt_v2(sqe->sds_sa_type); - recv_msg->in = (__u8 *)(uintptr_t)sqe->type2.data_src_addr; temp_msg = wd_digest_get_msg(qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; @@ -1614,11 +2001,15 @@ static void parse_digest_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, qp->q_info.idx, recv_msg->tag); return; } + recv_msg->in = temp_msg->in; + recv_msg->mm_ops = temp_msg->mm_ops; } else { /* The synchronization mode uses the same message */ temp_msg = recv_msg; } + destroy_digest_bd2_addr(temp_msg, sqe); + if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "digest"); } @@ -1719,6 +2110,7 @@ static int hisi_sec_digest_send(struct wd_alg_driver *drv, handle_t ctx, void *w handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *msg = wd_msg; struct hisi_sec_sqe sqe; + handle_t h_sgl_pool; __u16 count = 0; __u8 scene; __u8 de; @@ -1743,7 +2135,12 @@ static int hisi_sec_digest_send(struct wd_alg_driver *drv, handle_t ctx, void *w de = DATA_DST_ADDR_DISABLE << SEC_DE_OFFSET; if (msg->data_fmt == WD_SGL_BUF) { - ret = hisi_sec_fill_sgl(h_qp, &msg->in, &msg->out, &sqe, + h_sgl_pool = hisi_qm_get_sglpool(h_qp, msg->mm_ops); + if (!h_sgl_pool) { + WD_ERR("digest failed to get sglpool for hw_v2!\n"); + return -WD_EINVAL; + } + ret = hisi_sec_fill_sgl(h_sgl_pool, &msg->in, &msg->out, &sqe, msg->alg_type); if (ret) return ret; @@ -1751,16 +2148,19 @@ static int hisi_sec_digest_send(struct wd_alg_driver *drv, handle_t ctx, void *w sqe.sds_sa_type |= (__u8)(de | scene); sqe.type2.alen_ivllen |= (__u32)msg->in_bytes; - sqe.type2.data_src_addr = (__u64)(uintptr_t)msg->in; - sqe.type2.mac_addr = (__u64)(uintptr_t)msg->out; + ret = fill_digest_bd2_addr(msg, &sqe); + if (ret) { + WD_ERR("digest map memory is err(%d)!\n", ret); + goto put_sgl; + } ret = fill_digest_bd2_alg(msg, &sqe); if (ret) - goto put_sgl; + goto destroy_addr; ret = fill_digest_long_hash(h_qp, msg, &sqe); if (ret) - goto put_sgl; + goto destroy_addr; hisi_set_msg_id(h_qp, &msg->tag); sqe.type2.tag = (__u16)msg->tag; @@ -1769,15 +2169,16 @@ static int hisi_sec_digest_send(struct wd_alg_driver *drv, handle_t ctx, void *w if (ret != -WD_EBUSY) WD_ERR("digest send sqe is err(%d)!\n", ret); - goto put_sgl; + goto destroy_addr; } return 0; +destroy_addr: + destroy_digest_bd2_addr(msg, &sqe); put_sgl: if (msg->data_fmt == WD_SGL_BUF) - hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, msg->out); - + hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, msg->out, msg->mm_ops); return ret; } @@ -1801,7 +2202,7 @@ static int hisi_sec_digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *w if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, - recv_msg->out); + recv_msg->out, recv_msg->mm_ops); return 0; } @@ -1831,7 +2232,7 @@ static int hmac_key_len_check(struct wd_digest_msg *msg) } static int fill_digest_bd3_alg(struct wd_digest_msg *msg, - struct hisi_sec_sqe3 *sqe) + struct hisi_sec_sqe3 *sqe) { int ret; @@ -1866,13 +2267,11 @@ static int fill_digest_bd3_alg(struct wd_digest_msg *msg, return ret; sqe->auth_mac_key |= (__u32)BYTES_TO_WORDS(msg->key_bytes) << SEC_AKEY_OFFSET_V3; - sqe->a_key_addr = (__u64)(uintptr_t)msg->key; sqe->auth_mac_key |= g_hmac_a_alg[msg->alg] << SEC_AUTH_ALG_OFFSET_V3; if (msg->alg == WD_DIGEST_AES_GMAC) { sqe->auth_mac_key |= AI_GEN_IVIN_ADDR << SEC_AI_GEN_OFFSET_V3; - sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)msg->iv; } } else { WD_ERR("failed to check digest mode, mode = %u\n", msg->mode); @@ -1956,12 +2355,104 @@ static void fill_digest_v3_scene(struct hisi_sec_sqe3 *sqe, sqe->bd_param |= (__u16)(de | scene); } +static void destroy_digest_bd3_addr(struct wd_digest_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + struct wd_mm_ops *mm_ops = msg->mm_ops; + void *mempool; + + /* SVA mode and skip */ + if (!mm_ops || mm_ops->sva_mode) + return; + + if (!mm_ops->usr) { + WD_ERR("digest failed to check memory pool!\n"); + return; + } + + mempool = mm_ops->usr; + + if (sqe->data_src_addr) + mm_ops->iova_unmap(mempool, msg->in, (void *)(uintptr_t)sqe->data_src_addr, + msg->in_bytes); + + if (sqe->mac_addr) + mm_ops->iova_unmap(mempool, msg->out, (void *)(uintptr_t)sqe->mac_addr, + msg->out_bytes); + + if (sqe->a_key_addr && msg->mode == WD_DIGEST_HMAC) + mm_ops->iova_unmap(mempool, msg->key, (void *)(uintptr_t)sqe->a_key_addr, + msg->key_bytes); + + if (sqe->auth_ivin.a_ivin_addr && msg->mode == WD_DIGEST_HMAC && + msg->alg == WD_DIGEST_AES_GMAC) + mm_ops->iova_unmap(mempool, msg->iv, (void *)(uintptr_t)sqe->auth_ivin.a_ivin_addr, + MAX_IV_SIZE); +} + +static int fill_digest_bd3_addr(struct wd_digest_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + struct wd_mm_ops *mm_ops = msg->mm_ops; + void *mempool, *phy_addr; + + if (msg->mm_ops->sva_mode) { + /* avoid HW accessing address 0 when the pointer is NULL */ + if (msg->in) + sqe->data_src_addr = (__u64)(uintptr_t)msg->in; + else + sqe->data_src_addr = (__u64)(uintptr_t)msg->out; + sqe->mac_addr = (__u64)(uintptr_t)msg->out; + if (msg->mode == WD_DIGEST_HMAC) + sqe->a_key_addr = (__u64)(uintptr_t)msg->key; + if (msg->mode == WD_DIGEST_HMAC && msg->alg == WD_DIGEST_AES_GMAC) + sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)msg->iv; + return 0; + } + if (msg->mm_type > UADK_MEM_PROXY) { + WD_ERR("digest failed to check memory type!\n"); + return -WD_EINVAL; + } + + /* No-SVA mode and Memory is USER mode or PROXY mode */ + mempool = mm_ops->usr; + if (msg->in_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->in, msg->in_bytes); + if (!phy_addr) + return -WD_ENOMEM; + sqe->data_src_addr = (__u64)(uintptr_t)phy_addr; + } + phy_addr = mm_ops->iova_map(mempool, msg->out, msg->out_bytes); + if (!phy_addr) + goto map_err; + sqe->mac_addr = (__u64)(uintptr_t)phy_addr; + + if (msg->iv && msg->mode == WD_DIGEST_HMAC && + msg->alg == WD_DIGEST_AES_GMAC) { + phy_addr = mm_ops->iova_map(mempool, msg->iv, MAX_IV_SIZE); + if (!phy_addr) + goto map_err; + sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)phy_addr; + } + if (msg->key_bytes != 0 && msg->mode == WD_DIGEST_HMAC) { + phy_addr = mm_ops->iova_map(mempool, msg->key, msg->key_bytes); + if (!phy_addr) + goto map_err; + sqe->a_key_addr = (__u64)(uintptr_t)phy_addr; + } + + return 0; + +map_err: + destroy_digest_bd3_addr(msg, sqe); + return -WD_ENOMEM; +} + static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct wd_digest_msg *msg = wd_msg; struct hisi_sec_sqe3 sqe; + handle_t h_sgl_pool; __u16 count = 0; int ret; @@ -1980,43 +2471,53 @@ static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void sqe.auth_mac_key = AUTH_HMAC_CALCULATE; if (msg->data_fmt == WD_SGL_BUF) { - ret = hisi_sec_fill_sgl_v3(h_qp, &msg->in, &msg->out, &sqe, + h_sgl_pool = hisi_qm_get_sglpool(h_qp, msg->mm_ops); + if (!h_sgl_pool) { + WD_ERR("digest failed to get sglpool for hw_v3!\n"); + return -WD_EINVAL; + } + ret = hisi_sec_fill_sgl_v3(h_sgl_pool, &msg->in, &msg->out, &sqe, msg->alg_type); if (ret) return ret; } sqe.a_len_key = (__u32)msg->in_bytes; - sqe.data_src_addr = (__u64)(uintptr_t)msg->in; - sqe.mac_addr = (__u64)(uintptr_t)msg->out; + ret = fill_digest_bd3_addr(msg, &sqe); + if (ret < 0) { + WD_ERR("digest map memory is err(%d)!\n", ret); + goto put_sgl; + } ret = fill_digest_bd3_alg(msg, &sqe); if (ret) - goto put_sgl; + goto destroy_addr; ret = fill_digest_long_hash3(h_qp, msg, &sqe); if (ret) - goto put_sgl; + goto destroy_addr; hisi_set_msg_id(h_qp, &msg->tag); sqe.tag = (__u64)(uintptr_t)msg->tag; - fill_sec_prefetch(msg->data_fmt, msg->in_bytes, qp->q_info.hw_type, &sqe); + fill_sec_prefetch(msg->data_fmt, msg->in_bytes, qp->q_info.hw_type, &sqe, + msg->mm_ops->sva_mode); ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (ret < 0) { if (ret != -WD_EBUSY) WD_ERR("digest send sqe is err(%d)!\n", ret); - goto put_sgl; + goto destroy_addr; } return 0; +destroy_addr: + destroy_digest_bd3_addr(msg, &sqe); put_sgl: if (msg->data_fmt == WD_SGL_BUF) - hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, msg->out); - + hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, msg->out, msg->mm_ops); return ret; } @@ -2040,7 +2541,6 @@ static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { recv_msg->alg_type = WD_DIGEST; recv_msg->data_fmt = get_data_fmt_v3(sqe->bd_param); - recv_msg->in = (__u8 *)(uintptr_t)sqe->data_src_addr; temp_msg = wd_digest_get_msg(qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; @@ -2048,10 +2548,13 @@ static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, qp->q_info.idx, recv_msg->tag); return; } + recv_msg->in = temp_msg->in; + recv_msg->mm_ops = temp_msg->mm_ops; } else { /* The synchronization mode uses the same message */ temp_msg = recv_msg; } + destroy_digest_bd3_addr(temp_msg, sqe); if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "digest"); @@ -2077,7 +2580,7 @@ static int hisi_sec_digest_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, - recv_msg->out); + recv_msg->out, recv_msg->mm_ops); return 0; } @@ -2253,22 +2756,6 @@ static void set_aead_auth_iv(struct wd_aead_msg *msg) } } -static void fill_aead_bd2_addr(struct wd_aead_msg *msg, - struct hisi_sec_sqe *sqe) -{ - sqe->type2.data_src_addr = (__u64)(uintptr_t)msg->in; - sqe->type2.data_dst_addr = (__u64)(uintptr_t)msg->out; - sqe->type2.mac_addr = (__u64)(uintptr_t)msg->mac; - sqe->type2.c_key_addr = (__u64)(uintptr_t)msg->ckey; - sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->akey; - sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; - - /* CCM/GCM should init a_iv */ - set_aead_auth_iv(msg); - - sqe->type2.a_ivin_addr = (__u64)(uintptr_t)msg->aiv; -} - static int aead_len_check(struct wd_aead_msg *msg, enum sec_bd_type type) { if (msg->msg_state == AEAD_MSG_MIDDLE) { @@ -2291,6 +2778,11 @@ static int aead_len_check(struct wd_aead_msg *msg, enum sec_bd_type type) return -WD_EINVAL; } + if (unlikely(msg->in_bytes == 0 && msg->assoc_bytes == 0)) { + WD_ERR("aead input data length is 0\n"); + return -WD_EINVAL; + } + if (unlikely(msg->cmode == WD_CIPHER_CCM && msg->assoc_bytes > MAX_CCM_AAD_LEN)) { WD_ERR("aead ccm aad length is too long, size = %u\n", @@ -2323,14 +2815,14 @@ static void gcm_auth_ivin(struct wd_aead_msg *msg) __u32 final_counter = GCM_FINAL_COUNTER; /* auth_ivin = {cipher_ivin(16B), null(16B), auth_mac(16B), null(16B)} */ - memset(msg->aiv_stream, 0, AIV_STREAM_LEN); + memset(msg->aiv, 0, AIV_STREAM_LEN); - memcpy(msg->aiv_stream, msg->iv, GCM_IV_SIZE); + memcpy(msg->aiv, msg->iv, GCM_IV_SIZE); /* The last 4 bytes of c_ivin are counters */ - memcpy(msg->aiv_stream + GCM_IV_SIZE, &final_counter, GCM_FINAL_COUNTER_LEN); + memcpy(msg->aiv + GCM_IV_SIZE, &final_counter, GCM_FINAL_COUNTER_LEN); /* Fill auth_ivin with the mac of last MIDDLE BD */ - memcpy(msg->aiv_stream + GCM_STREAM_MAC_OFFSET, msg->mac, GCM_FULL_MAC_LEN); + memcpy(msg->aiv + GCM_STREAM_MAC_OFFSET, msg->mac, GCM_FULL_MAC_LEN); /* Use the user's origin mac for decrypt icv check */ if (msg->op_type == WD_CIPHER_DECRYPTION_DIGEST) @@ -2353,8 +2845,6 @@ static void fill_gcm_first_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe sqe->type2.c_alg = 0; sqe->type2.auth_src_offset = 0; sqe->type2.alen_ivllen = msg->assoc_bytes; - sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; - sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->ckey; } static void fill_gcm_middle_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) @@ -2367,8 +2857,6 @@ static void fill_gcm_middle_bd2(struct wd_aead_msg *msg, struct hisi_sec_sqe *sq fill_gcm_akey_len(msg, sqe, BD_TYPE2); sqe->type2.alen_ivllen = 0; sqe->type2.a_ivin_addr = sqe->type2.mac_addr; - sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; - sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->ckey; } static void get_galois_vector_s(struct wd_aead_msg *msg, __u8 *s) @@ -2385,7 +2873,7 @@ static void get_galois_vector_s(struct wd_aead_msg *msg, __u8 *s) /* Based the little-endian operation */ for (i = 0; i < GCM_BLOCK_SIZE; i++) - s[i] = a_c[i] ^ msg->aiv_stream[(__u8)(GCM_AUTH_MAC_OFFSET - i)]; + s[i] = a_c[i] ^ msg->aiv[(__u8)(GCM_AUTH_MAC_OFFSET - i)]; } static int gcm_do_soft_mac(struct wd_aead_msg *msg) @@ -2424,9 +2912,9 @@ static int gcm_do_soft_mac(struct wd_aead_msg *msg) */ for (i = 0; i < GCM_BLOCK_SIZE; i++) G[i] = data[GCM_BLOCK_OFFSET - i] ^ - msg->aiv_stream[(__u8)(GCM_AUTH_MAC_OFFSET - i)]; + msg->aiv[(__u8)(GCM_AUTH_MAC_OFFSET - i)]; - galois_compute(G, H, msg->aiv_stream + GCM_STREAM_MAC_OFFSET, GCM_BLOCK_SIZE); + galois_compute(G, H, msg->aiv + GCM_STREAM_MAC_OFFSET, GCM_BLOCK_SIZE); len -= block; offset += block; } @@ -2436,7 +2924,7 @@ static int gcm_do_soft_mac(struct wd_aead_msg *msg) galois_compute(S, H, g, GCM_BLOCK_SIZE); /* Encrypt ctr0 based on AES_ECB */ - aes_encrypt(msg->ckey, msg->ckey_bytes, msg->aiv_stream, ctr_r); + aes_encrypt(msg->ckey, msg->ckey_bytes, msg->aiv, ctr_r); /* Get the GMAC tag final */ for (i = 0; i < GCM_BLOCK_SIZE; i++) @@ -2556,11 +3044,159 @@ static int aead_msg_state_check(struct wd_aead_msg *msg) return 0; } +static void destroy_aead_bd2_addr(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe) +{ + struct wd_mm_ops *mm_ops = msg->mm_ops; + __u64 dma_addr; + void *mempool; + + aead_free_aiv_addr(msg); + /* SVA mode and skip */ + if (!mm_ops || mm_ops->sva_mode) + return; + + if (!mm_ops->usr) { + WD_ERR("aead failed to check memory pool!\n"); + return; + } + + mempool = mm_ops->usr; + if (sqe->type2.data_src_addr) + mm_ops->iova_unmap(mempool, msg->in, (void *)(uintptr_t)sqe->type2.data_src_addr, + msg->in_bytes); + + if (sqe->type2.data_dst_addr) + mm_ops->iova_unmap(mempool, msg->out, (void *)(uintptr_t)sqe->type2.data_dst_addr, + msg->out_bytes); + + if (sqe->type2.c_ivin_addr) + mm_ops->iova_unmap(mempool, msg->iv, (void *)(uintptr_t)sqe->type2.c_ivin_addr, + msg->iv_bytes); + + if (sqe->type2.a_key_addr) { + if ((msg->msg_state == AEAD_MSG_FIRST || msg->msg_state == AEAD_MSG_MIDDLE) + && msg->cmode == WD_CIPHER_GCM) + mm_ops->iova_unmap(mempool, msg->ckey, + (void *)(uintptr_t)sqe->type2.a_key_addr, + msg->ckey_bytes); + else + mm_ops->iova_unmap(mempool, msg->akey, + (void *)(uintptr_t)sqe->type2.a_key_addr, + msg->akey_bytes); + } + + if (sqe->type2.c_key_addr && !((msg->msg_state == AEAD_MSG_FIRST || + msg->msg_state == AEAD_MSG_MIDDLE) && msg->cmode == WD_CIPHER_GCM)) + mm_ops->iova_unmap(mempool, msg->ckey, (void *)(uintptr_t)sqe->type2.c_key_addr, + msg->ckey_bytes); + + if (sqe->type2.mac_addr) + mm_ops->iova_unmap(mempool, msg->mac, (void *)(uintptr_t)sqe->type2.mac_addr, + msg->auth_bytes); +} + +static int aead_mem_nosva_map(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe, int idx) +{ + struct wd_aead_aiv_addr *aiv_addr = (struct wd_aead_aiv_addr *)msg->drv_cfg; + struct wd_mm_ops *mm_ops = msg->mm_ops; + void *mempool, *phy_addr; + + /* No-SVA mode and Memory is USER mode or PROXY mode */ + mempool = mm_ops->usr; + + phy_addr = mm_ops->iova_map(mempool, msg->in, msg->in_bytes + msg->assoc_bytes); + if (!phy_addr) + return -WD_ENOMEM; + sqe->type2.data_src_addr = (__u64)(uintptr_t)phy_addr; + phy_addr = mm_ops->iova_map(mempool, msg->out, msg->out_bytes); + if (!phy_addr) + goto map_err; + sqe->type2.data_dst_addr = (__u64)(uintptr_t)phy_addr; + if (msg->iv_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->iv, msg->iv_bytes); + if (!phy_addr) + goto map_err; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)phy_addr; + } + if (msg->akey_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->akey, msg->akey_bytes); + if (!phy_addr) + goto map_err; + sqe->type2.a_key_addr = (__u64)(uintptr_t)phy_addr; + } + if (msg->ckey_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->ckey, msg->ckey_bytes); + if (!phy_addr) + goto map_err; + if ((msg->msg_state == AEAD_MSG_FIRST || msg->msg_state == AEAD_MSG_MIDDLE) + && msg->cmode == WD_CIPHER_GCM) + sqe->type2.a_key_addr = (__u64)(uintptr_t)phy_addr; + else + sqe->type2.c_key_addr = (__u64)(uintptr_t)phy_addr; + } + if (msg->auth_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->mac, msg->auth_bytes); + if (!phy_addr) + goto map_err; + sqe->type2.mac_addr = (__u64)(uintptr_t)phy_addr; + } + + /* CCM/GCM should init a_iv */ + set_aead_auth_iv(msg); + phy_addr = aiv_addr->aiv_nosva + (idx << AEAD_AIV_OFFSET); + sqe->type2.a_ivin_addr = (__u64)(uintptr_t)phy_addr; + + return 0; + +map_err: + destroy_aead_bd2_addr(msg, sqe); + return -WD_ENOMEM; +} + +static int fill_aead_bd2_addr(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe, + struct hisi_qp *qp) +{ + int idx; + + idx = aead_get_aiv_addr(qp, msg); + if (idx < 0) + return idx; + + /* sva mode */ + if (msg->mm_ops->sva_mode) { + sqe->type2.data_src_addr = (__u64)(uintptr_t)msg->in; + sqe->type2.data_dst_addr = (__u64)(uintptr_t)msg->out; + sqe->type2.mac_addr = (__u64)(uintptr_t)msg->mac; + sqe->type2.c_key_addr = (__u64)(uintptr_t)msg->ckey; + sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->akey; + sqe->type2.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + /* CCM/GCM should init a_iv */ + set_aead_auth_iv(msg); + sqe->type2.a_ivin_addr = (__u64)(uintptr_t)msg->aiv; + + if ((msg->msg_state == AEAD_MSG_FIRST || msg->msg_state == AEAD_MSG_MIDDLE) + && msg->cmode == WD_CIPHER_GCM) + sqe->type2.a_key_addr = (__u64)(uintptr_t)msg->ckey; + + return 0; + } + if (msg->mm_type > UADK_MEM_PROXY) { + WD_ERR("aead failed to check memory type!\n"); + aead_free_aiv_addr(msg); + return -WD_EINVAL; + } + + /* aiv addr is freed in destroy addr interface */ + return aead_mem_nosva_map(msg, sqe, idx); +} + static int hisi_sec_aead_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); + struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct wd_aead_msg *msg = wd_msg; struct hisi_sec_sqe sqe; + handle_t h_sgl_pool; __u16 count = 0; int ret; @@ -2583,17 +3219,27 @@ static int hisi_sec_aead_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_ return ret; if (msg->data_fmt == WD_SGL_BUF) { - ret = hisi_sec_fill_sgl(h_qp, &msg->in, &msg->out, + h_sgl_pool = hisi_qm_get_sglpool(h_qp, msg->mm_ops); + if (!h_sgl_pool) { + WD_ERR("aead failed to get sglpool for hw_v2!\n"); + return -WD_EINVAL; + } + ret = hisi_sec_fill_sgl(h_sgl_pool, &msg->in, &msg->out, &sqe, msg->alg_type); if (ret) return ret; } - fill_aead_bd2_addr(msg, &sqe); + ret = fill_aead_bd2_addr(msg, &sqe, qp); + if (ret < 0) { + if (ret != -WD_EBUSY) + WD_ERR("aead map memory is err(%d)!\n", ret); + goto put_sgl; + } ret = fill_stream_bd2(msg, &sqe); if (unlikely(ret)) - goto put_sgl; + goto destroy_addr; hisi_set_msg_id(h_qp, &msg->tag); sqe.type2.tag = (__u16)msg->tag; @@ -2603,15 +3249,16 @@ static int hisi_sec_aead_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_ if (ret != -WD_EBUSY) WD_ERR("aead send sqe is err(%d)!\n", ret); - goto put_sgl; + goto destroy_addr; } return 0; +destroy_addr: + destroy_aead_bd2_addr(msg, &sqe); put_sgl: if (msg->data_fmt == WD_SGL_BUF) - hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, msg->out); - + hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, msg->out, msg->mm_ops); return ret; } @@ -2652,8 +3299,6 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { recv_msg->alg_type = WD_AEAD; recv_msg->data_fmt = get_data_fmt_v2(sqe->sds_sa_type); - recv_msg->in = (__u8 *)(uintptr_t)sqe->type2.data_src_addr; - recv_msg->out = (__u8 *)(uintptr_t)sqe->type2.data_dst_addr; temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; @@ -2661,12 +3306,16 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, qp->q_info.idx, recv_msg->tag); return; } + recv_msg->in = temp_msg->in; + recv_msg->out = temp_msg->out; + recv_msg->mm_ops = temp_msg->mm_ops; } else { /* The synchronization mode uses the same message */ temp_msg = recv_msg; } update_stream_counter(temp_msg); + destroy_aead_bd2_addr(temp_msg, sqe); if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "aead"); @@ -2692,7 +3341,7 @@ static int hisi_sec_aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_ if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, recv_msg->in, - recv_msg->out); + recv_msg->out, recv_msg->mm_ops); return 0; } @@ -2789,23 +3438,6 @@ static int fill_aead_bd3_mode(struct wd_aead_msg *msg, return 0; } -static void fill_aead_bd3_addr(struct wd_aead_msg *msg, - struct hisi_sec_sqe3 *sqe) -{ - sqe->data_src_addr = (__u64)(uintptr_t)msg->in; - sqe->data_dst_addr = (__u64)(uintptr_t)msg->out; - - sqe->mac_addr = (__u64)(uintptr_t)msg->mac; - sqe->c_key_addr = (__u64)(uintptr_t)msg->ckey; - sqe->a_key_addr = (__u64)(uintptr_t)msg->akey; - sqe->no_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; - - /* CCM/GCM should init a_iv */ - set_aead_auth_iv(msg); - - sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)msg->aiv; -} - static void fill_gcm_first_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) { sqe->auth_mac_key |= AI_GEN_INNER << SEC_AI_GEN_OFFSET_V3; @@ -2822,8 +3454,6 @@ static void fill_gcm_first_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sq sqe->c_mode_alg &= ~(0x7 << SEC_CALG_OFFSET_V3); sqe->auth_src_offset = 0; sqe->a_len_key = msg->assoc_bytes; - sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; - sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; } static void fill_gcm_middle_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) @@ -2837,8 +3467,6 @@ static void fill_gcm_middle_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *s fill_gcm_akey_len(msg, sqe, BD_TYPE3); sqe->a_len_key = 0; sqe->auth_ivin.a_ivin_addr = sqe->mac_addr; - sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; - sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; } static void fill_gcm_final_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) @@ -2853,9 +3481,6 @@ static void fill_gcm_final_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sq sqe->a_len_key = 0; sqe->stream_scene.long_a_data_len = msg->assoc_bytes; sqe->stream_scene.long_a_data_len |= msg->long_data_len << LONG_AUTH_DATA_OFFSET; - sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; - sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; - sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)msg->aiv_stream; } static int fill_stream_bd3(handle_t h_qp, struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) @@ -2939,12 +3564,167 @@ static int fill_aead_bd3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) return 0; } +static void destroy_aead_bd3_addr(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe) +{ + struct wd_mm_ops *mm_ops = msg->mm_ops; + void *mempool; + + aead_free_aiv_addr(msg); + /* SVA mode and skip */ + if (!mm_ops || mm_ops->sva_mode) + return; + + if (!mm_ops->usr) { + WD_ERR("aead failed to check memory pool!\n"); + return; + } + + mempool = mm_ops->usr; + if (sqe->data_src_addr) + mm_ops->iova_unmap(mempool, msg->in, (void *)(uintptr_t)sqe->data_src_addr, + msg->in_bytes); + + if (sqe->data_dst_addr) + mm_ops->iova_unmap(mempool, msg->out, (void *)(uintptr_t)sqe->data_dst_addr, + msg->out_bytes); + + if (sqe->no_scene.c_ivin_addr) + mm_ops->iova_unmap(mempool, msg->iv, (void *)(uintptr_t)sqe->no_scene.c_ivin_addr, + msg->iv_bytes); + else if (sqe->stream_scene.c_ivin_addr) + mm_ops->iova_unmap(mempool, msg->iv, + (void *)(uintptr_t)sqe->stream_scene.c_ivin_addr, + msg->iv_bytes); + + if (sqe->a_key_addr) { + if ((msg->msg_state == AEAD_MSG_FIRST || msg->msg_state == AEAD_MSG_MIDDLE || + msg->msg_state == AEAD_MSG_END) && msg->cmode == WD_CIPHER_GCM) + mm_ops->iova_unmap(mempool, msg->ckey, (void *)(uintptr_t)sqe->a_key_addr, + msg->ckey_bytes); + else + mm_ops->iova_unmap(mempool, msg->akey, (void *)(uintptr_t)sqe->a_key_addr, + msg->akey_bytes); + } + + if (sqe->c_key_addr && !((msg->msg_state == AEAD_MSG_FIRST || + msg->msg_state == AEAD_MSG_MIDDLE || msg->msg_state == AEAD_MSG_END) && + msg->cmode == WD_CIPHER_GCM)) + mm_ops->iova_unmap(mempool, msg->ckey, (void *)(uintptr_t)sqe->c_key_addr, + msg->ckey_bytes); + + if (sqe->mac_addr) + mm_ops->iova_unmap(mempool, msg->mac, (void *)(uintptr_t)sqe->mac_addr, + msg->auth_bytes); +} + +static int aead_mem_nosva_map_v3(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe, int idx) +{ + struct wd_aead_aiv_addr *aiv_addr = (struct wd_aead_aiv_addr *)msg->drv_cfg; + struct wd_mm_ops *mm_ops = msg->mm_ops; + void *mempool = mm_ops->usr; + void *phy_addr; + + phy_addr = mm_ops->iova_map(mempool, msg->in, msg->in_bytes + msg->assoc_bytes); + if (!phy_addr) + return -WD_ENOMEM; + sqe->data_src_addr = (__u64)(uintptr_t)phy_addr; + + phy_addr = mm_ops->iova_map(mempool, msg->out, msg->out_bytes); + if (!phy_addr) + goto map_err; + sqe->data_dst_addr = (__u64)(uintptr_t)phy_addr; + + if (msg->iv_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->iv, msg->iv_bytes); + if (!phy_addr) + goto map_err; + sqe->no_scene.c_ivin_addr = (__u64)(uintptr_t)phy_addr; + if ((msg->msg_state == AEAD_MSG_FIRST || msg->msg_state == AEAD_MSG_MIDDLE || + msg->msg_state == AEAD_MSG_END) && msg->cmode == WD_CIPHER_GCM) + sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)phy_addr; + } + + if (msg->akey_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->akey, msg->akey_bytes); + if (!phy_addr) + goto map_err; + sqe->a_key_addr = (__u64)(uintptr_t)phy_addr; + } + + if (msg->ckey_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->ckey, msg->ckey_bytes); + if (!phy_addr) + goto map_err; + sqe->c_key_addr = (__u64)(uintptr_t)phy_addr; + if ((msg->msg_state == AEAD_MSG_FIRST || msg->msg_state == AEAD_MSG_MIDDLE || + msg->msg_state == AEAD_MSG_END) && msg->cmode == WD_CIPHER_GCM) + sqe->a_key_addr = (__u64)(uintptr_t)phy_addr; + } + + if (msg->auth_bytes) { + phy_addr = mm_ops->iova_map(mempool, msg->mac, msg->auth_bytes); + if (!phy_addr) + goto map_err; + sqe->mac_addr = (__u64)(uintptr_t)phy_addr; + } + + /* CCM/GCM should init a_iv */ + set_aead_auth_iv(msg); + phy_addr = aiv_addr->aiv_nosva + (idx << AEAD_AIV_OFFSET); + sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)phy_addr; + + return 0; + +map_err: + destroy_aead_bd3_addr(msg, sqe); + return -WD_ENOMEM; +} + +static int fill_aead_bd3_addr(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe, + struct hisi_qp *qp) +{ + int idx; + + idx = aead_get_aiv_addr(qp, msg); + if (idx < 0) + return idx; + + /* sva mode */ + if (msg->mm_ops->sva_mode) { + sqe->data_src_addr = (__u64)(uintptr_t)msg->in; + sqe->data_dst_addr = (__u64)(uintptr_t)msg->out; + sqe->no_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->c_key_addr = (__u64)(uintptr_t)msg->ckey; + sqe->a_key_addr = (__u64)(uintptr_t)msg->akey; + sqe->mac_addr = (__u64)(uintptr_t)msg->mac; + + /* CCM/GCM should init a_iv */ + set_aead_auth_iv(msg); + sqe->auth_ivin.a_ivin_addr = (__u64)(uintptr_t)msg->aiv; + if ((msg->msg_state == AEAD_MSG_FIRST || msg->msg_state == AEAD_MSG_MIDDLE || + msg->msg_state == AEAD_MSG_END) && msg->cmode == WD_CIPHER_GCM) { + sqe->stream_scene.c_ivin_addr = (__u64)(uintptr_t)msg->iv; + sqe->a_key_addr = (__u64)(uintptr_t)msg->ckey; + } + return 0; + } + if (msg->mm_type > UADK_MEM_PROXY) { + WD_ERR("aead failed to check memory type!\n"); + aead_free_aiv_addr(msg); + return -WD_EINVAL; + } + + /* aiv addr is freed in destroy addr interface */ + return aead_mem_nosva_map_v3(msg, sqe, idx); +} + static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; struct wd_aead_msg *msg = wd_msg; struct hisi_sec_sqe3 sqe; + handle_t h_sgl_pool; __u16 count = 0; int ret; @@ -2967,19 +3747,30 @@ static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void * return ret; fill_sec_prefetch(msg->data_fmt, msg->in_bytes + msg->assoc_bytes, - qp->q_info.hw_type, &sqe); + qp->q_info.hw_type, &sqe, msg->mm_ops->sva_mode); if (msg->data_fmt == WD_SGL_BUF) { - ret = hisi_sec_fill_sgl_v3(h_qp, &msg->in, &msg->out, &sqe, + h_sgl_pool = hisi_qm_get_sglpool(h_qp, msg->mm_ops); + if (!h_sgl_pool) { + WD_ERR("aead failed to get sglpool for hw_v3!\n"); + return -WD_EINVAL; + } + ret = hisi_sec_fill_sgl_v3(h_sgl_pool, &msg->in, &msg->out, &sqe, msg->alg_type); if (ret) return ret; } - fill_aead_bd3_addr(msg, &sqe); + ret = fill_aead_bd3_addr(msg, &sqe, qp); + if (ret < 0) { + if (ret != -WD_EBUSY) + WD_ERR("aead map memory is err(%d)!\n", ret); + goto put_sgl; + } + ret = fill_stream_bd3(h_qp, msg, &sqe); if (unlikely(ret)) - goto put_sgl; + goto destroy_addr; hisi_set_msg_id(h_qp, &msg->tag); sqe.tag = msg->tag; @@ -2988,15 +3779,16 @@ static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void * if (ret != -WD_EBUSY) WD_ERR("aead send sqe is err(%d)!\n", ret); - goto put_sgl; + goto destroy_addr; } return 0; +destroy_addr: + destroy_aead_bd3_addr(msg, &sqe); put_sgl: if (msg->data_fmt == WD_SGL_BUF) - hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, msg->out); - + hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, msg->out, msg->mm_ops); return ret; } @@ -3023,8 +3815,6 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, if (qp->q_info.qp_mode == CTX_MODE_ASYNC) { recv_msg->alg_type = WD_AEAD; recv_msg->data_fmt = get_data_fmt_v3(sqe->bd_param); - recv_msg->in = (__u8 *)(uintptr_t)sqe->data_src_addr; - recv_msg->out = (__u8 *)(uintptr_t)sqe->data_dst_addr; temp_msg = wd_aead_get_msg(qp->q_info.idx, recv_msg->tag); if (!temp_msg) { recv_msg->result = WD_IN_EPARA; @@ -3032,12 +3822,16 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, qp->q_info.idx, recv_msg->tag); return; } + recv_msg->in = temp_msg->in; + recv_msg->out = temp_msg->out; + recv_msg->mm_ops = temp_msg->mm_ops; } else { /* The synchronization mode uses the same message */ temp_msg = recv_msg; } update_stream_counter(temp_msg); + destroy_aead_bd3_addr(temp_msg, sqe); if (unlikely(recv_msg->result != WD_SUCCESS)) dump_sec_msg(temp_msg, "aead"); @@ -3063,7 +3857,7 @@ static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void * if (recv_msg->data_fmt == WD_SGL_BUF) hisi_sec_put_sgl(h_qp, recv_msg->alg_type, - recv_msg->in, recv_msg->out); + recv_msg->in, recv_msg->out, recv_msg->mm_ops); return 0; } @@ -3128,6 +3922,7 @@ static void hisi_sec_exit(struct wd_alg_driver *drv) priv = (struct hisi_sec_ctx *)drv->priv; config = &priv->config; + for (i = 0; i < config->ctx_num; i++) { h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); hisi_qm_free_qp(h_qp); diff --git a/include/drv/wd_aead_drv.h b/include/drv/wd_aead_drv.h index a9c0e7c..2c53217 100644 --- a/include/drv/wd_aead_drv.h +++ b/include/drv/wd_aead_drv.h @@ -54,9 +54,7 @@ struct wd_aead_msg { /* input iv pointer */ __u8 *iv; /* input auth iv pointer */ - __u8 aiv[MAX_IV_SIZE]; - /* input auth iv pointer for stream mode */ - __u8 aiv_stream[AIV_STREAM_LEN]; + __u8 *aiv; /* input data pointer */ __u8 *in; /* output data pointer */ @@ -68,6 +66,25 @@ struct wd_aead_msg { /* total of data for stream mode */ __u64 long_data_len; enum wd_aead_msg_state msg_state; + struct wd_mm_ops *mm_ops; + enum wd_mem_type mm_type; + void *drv_cfg; /* internal driver configuration */ +}; + +struct wd_aead_aiv_addr { + __u8 *aiv; + __u8 *aiv_status; + __u8 *aiv_nosva; +}; + +struct wd_aead_extend_ops { + void *params; + int (*eops_aiv_init)(struct wd_alg_driver *drv, + struct wd_mm_ops *mm_ops, + void **params); + void (*eops_aiv_uninit)(struct wd_alg_driver *drv, + struct wd_mm_ops *mm_ops, + void *params); }; struct wd_aead_msg *wd_aead_get_msg(__u32 idx, __u32 tag); diff --git a/include/drv/wd_cipher_drv.h b/include/drv/wd_cipher_drv.h index c6d8ddf..c0be0c3 100644 --- a/include/drv/wd_cipher_drv.h +++ b/include/drv/wd_cipher_drv.h @@ -48,6 +48,8 @@ struct wd_cipher_msg { __u8 *in; /* output data pointer */ __u8 *out; + struct wd_mm_ops *mm_ops; + enum wd_mem_type mm_type; }; struct wd_cipher_msg *wd_cipher_get_msg(__u32 idx, __u32 tag); diff --git a/include/drv/wd_digest_drv.h b/include/drv/wd_digest_drv.h index a55ef5b..12398f2 100644 --- a/include/drv/wd_digest_drv.h +++ b/include/drv/wd_digest_drv.h @@ -59,6 +59,8 @@ struct wd_digest_msg { __u8 *partial_block; /* total of data for stream mode */ __u64 long_data_len; + struct wd_mm_ops *mm_ops; + enum wd_mem_type mm_type; }; static inline enum hash_block_type get_hash_block_type(struct wd_digest_msg *msg) diff --git a/include/wd_aead.h b/include/wd_aead.h index 01f6980..4b5095f 100644 --- a/include/wd_aead.h +++ b/include/wd_aead.h @@ -41,6 +41,8 @@ struct wd_aead_sess_setup { enum wd_digest_type dalg; enum wd_digest_mode dmode; void *sched_param; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; /** diff --git a/include/wd_cipher.h b/include/wd_cipher.h index d54f7fe..1d82eac 100644 --- a/include/wd_cipher.h +++ b/include/wd_cipher.h @@ -73,6 +73,8 @@ struct wd_cipher_sess_setup { enum wd_cipher_alg alg; enum wd_cipher_mode mode; void *sched_param; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; struct wd_cipher_req; diff --git a/include/wd_digest.h b/include/wd_digest.h index 6ce31f2..42a95db 100644 --- a/include/wd_digest.h +++ b/include/wd_digest.h @@ -100,6 +100,8 @@ struct wd_digest_sess_setup { enum wd_digest_type alg; enum wd_digest_mode mode; void *sched_param; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; typedef void *wd_digest_cb_t(void *cb_param); diff --git a/wd_aead.c b/wd_aead.c index 373b6fe..8467409 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -44,19 +44,22 @@ struct wd_aead_sess { enum wd_cipher_mode cmode; enum wd_digest_type dalg; enum wd_digest_mode dmode; - unsigned char ckey[MAX_CIPHER_KEY_SIZE]; - unsigned char akey[MAX_HMAC_KEY_SIZE]; + unsigned char *ckey; + unsigned char *akey; /* Mac data pointer for decrypto as stream mode */ - unsigned char mac_bak[WD_AEAD_CCM_GCM_MAX]; + unsigned char *mac_bak; __u16 ckey_bytes; __u16 akey_bytes; __u16 auth_bytes; void *priv; void *sched_key; /* Stored the counter for gcm stream mode */ - __u8 iv[MAX_IV_SIZE]; + __u8 *iv; /* Total of data for stream mode */ __u64 long_data_len; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; + struct wd_aead_extend_ops eops; }; struct wd_env_config wd_aead_env_config; @@ -302,26 +305,26 @@ int wd_aead_get_maxauthsize(handle_t h_sess) return g_aead_mac_len[sess->dalg]; } -handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) +static struct wd_aead_sess *check_and_init_sess(struct wd_aead_sess_setup *setup) { - struct wd_aead_sess *sess = NULL; + struct wd_aead_sess *sess; bool ret; if (unlikely(!setup)) { WD_ERR("failed to check session input parameter!\n"); - return (handle_t)0; + return NULL; } if (setup->calg >= WD_CIPHER_ALG_TYPE_MAX || - setup->cmode >= WD_CIPHER_MODE_TYPE_MAX) { + setup->cmode >= WD_CIPHER_MODE_TYPE_MAX) { WD_ERR("failed to check algorithm setup!\n"); - return (handle_t)0; + return NULL; } sess = malloc(sizeof(struct wd_aead_sess)); if (!sess) { WD_ERR("failed to alloc session memory!\n"); - return (handle_t)0; + return NULL; } memset(sess, 0, sizeof(struct wd_aead_sess)); @@ -330,24 +333,160 @@ handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) sess->cmode = setup->cmode; sess->dalg = setup->dalg; sess->dmode = setup->dmode; + ret = wd_drv_alg_support(sess->alg_name, wd_aead_setting.driver); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); - goto err_sess; + free(sess); + return NULL; + } + + return sess; +} + +static int aead_setup_memory_and_buffers(struct wd_aead_sess *sess, + struct wd_aead_sess_setup *setup) +{ + wd_alloc aead_alloc_func; + wd_free aead_free_func; + void *mempool; + int ret; + + ret = wd_mem_ops_init(wd_aead_setting.config.ctxs[0].ctx, + &setup->mm_ops, setup->mm_type); + if (ret) { + WD_ERR("failed to init memory ops!\n"); + return -WD_EINVAL; + } + + memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops)); + sess->mm_type = setup->mm_type; + + aead_alloc_func = sess->mm_ops.alloc; + aead_free_func = sess->mm_ops.free; + mempool = sess->mm_ops.usr; + + sess->mac_bak = aead_alloc_func(mempool, WD_AEAD_CCM_GCM_MAX); + if (!sess->mac_bak) { + WD_ERR("aead failed to calloc mac_bak memory!\n"); + return -WD_ENOMEM; + } + memset(sess->mac_bak, 0, WD_AEAD_CCM_GCM_MAX); + + sess->iv = aead_alloc_func(mempool, MAX_IV_SIZE); + if (!sess->iv) { + WD_ERR("failed to alloc iv memory!\n"); + goto iv_err; + } + memset(sess->iv, 0, MAX_IV_SIZE); + + sess->ckey = aead_alloc_func(mempool, MAX_CIPHER_KEY_SIZE); + if (!sess->ckey) { + WD_ERR("failed to alloc ckey memory!\n"); + goto ckey_err; + } + memset(sess->ckey, 0, MAX_CIPHER_KEY_SIZE); + + sess->akey = aead_alloc_func(mempool, MAX_HMAC_KEY_SIZE); + if (!sess->akey) { + WD_ERR("failed to alloc akey memory!\n"); + goto akey_err; + } + memset(sess->akey, 0, MAX_HMAC_KEY_SIZE); + + return 0; + +akey_err: + aead_free_func(mempool, sess->ckey); +ckey_err: + aead_free_func(mempool, sess->iv); +iv_err: + aead_free_func(mempool, sess->mac_bak); + + return -WD_ENOMEM; +} + +static void cleanup_session(struct wd_aead_sess *sess) +{ + sess->mm_ops.free(sess->mm_ops.usr, sess->mac_bak); + sess->mm_ops.free(sess->mm_ops.usr, sess->iv); + sess->mm_ops.free(sess->mm_ops.usr, sess->ckey); + sess->mm_ops.free(sess->mm_ops.usr, sess->akey); + + if (sess) + free(sess); +} + +static int wd_aead_sess_eops_init(struct wd_aead_sess *sess) +{ + int ret; + + if (sess->eops.eops_aiv_init) { + if (!sess->eops.eops_aiv_uninit) { + WD_ERR("failed to get aead extend ops free in session!\n"); + return -WD_EINVAL; + } + ret = sess->eops.eops_aiv_init(wd_aead_setting.driver, &sess->mm_ops, + &sess->eops.params); + if (ret) { + WD_ERR("failed to init aead extend ops params in session!\n"); + return ret; + } + } + + return WD_SUCCESS; +} + +static void wd_aead_sess_eops_uninit(struct wd_aead_sess *sess) +{ + if (sess->eops.eops_aiv_uninit) { + sess->eops.eops_aiv_uninit(wd_aead_setting.driver, &sess->mm_ops, + sess->eops.params); + sess->eops.params = NULL; + } +} + +handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) +{ + struct wd_aead_sess *sess; + int ret; + + sess = check_and_init_sess(setup); + if (!sess) + return (handle_t)0; + + if (aead_setup_memory_and_buffers(sess, setup)) { + free(sess); + return (handle_t)0; + } + + if (wd_aead_setting.driver->get_extend_ops) { + ret = wd_aead_setting.driver->get_extend_ops(&sess->eops); + if (ret) { + WD_ERR("failed to get aead sess extend ops!\n"); + goto sess_err; + } + } + + ret = wd_aead_sess_eops_init(sess); + if (ret) { + WD_ERR("failed to init aead sess extend eops!\n"); + goto sess_err; } - /* Some simple scheduler don't need scheduling parameters */ sess->sched_key = (void *)wd_aead_setting.sched.sched_init( - wd_aead_setting.sched.h_sched_ctx, setup->sched_param); + wd_aead_setting.sched.h_sched_ctx, setup->sched_param); if (WD_IS_ERR(sess->sched_key)) { WD_ERR("failed to init session schedule key!\n"); - goto err_sess; + goto sched_key_err; } return (handle_t)sess; -err_sess: - free(sess); +sched_key_err: + wd_aead_sess_eops_uninit(sess); +sess_err: + cleanup_session(sess); return (handle_t)0; } @@ -365,7 +504,8 @@ void wd_aead_free_sess(handle_t h_sess) if (sess->sched_key) free(sess->sched_key); - free(sess); + wd_aead_sess_eops_uninit(sess); + cleanup_session(sess); } static int wd_aead_param_check(struct wd_aead_sess *sess, @@ -717,8 +857,11 @@ static void fill_request_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, msg->mac = req->mac; msg->auth_bytes = sess->auth_bytes; msg->data_fmt = req->data_fmt; - msg->msg_state = req->msg_state; + + msg->mm_ops = &sess->mm_ops; + msg->mm_type = sess->mm_type; + msg->drv_cfg = sess->eops.params; fill_stream_msg(msg, req, sess); } diff --git a/wd_cipher.c b/wd_cipher.c index 92ca07b..58656dc 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -63,9 +63,11 @@ struct wd_cipher_sess { enum wd_cipher_mode mode; wd_dev_mask_t *dev_mask; void *priv; - unsigned char key[MAX_CIPHER_KEY_SIZE]; + unsigned char *key; __u32 key_bytes; void *sched_key; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; struct wd_env_config wd_cipher_env_config; @@ -250,6 +252,31 @@ int wd_cipher_set_key(handle_t h_sess, const __u8 *key, __u32 key_len) return 0; } +static int cipher_setup_memory_and_buffers(struct wd_cipher_sess *sess, + struct wd_cipher_sess_setup *setup) +{ + int ret; + + ret = wd_mem_ops_init(wd_cipher_setting.config.ctxs[0].ctx, + &setup->mm_ops, setup->mm_type); + if (ret) { + WD_ERR("cipher failed to init memory ops!\n"); + return ret; + } + + memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops)); + sess->mm_type = setup->mm_type; + + sess->key = sess->mm_ops.alloc(sess->mm_ops.usr, MAX_CIPHER_KEY_SIZE); + if (!sess->key) { + WD_ERR("cipher failed to alloc key memory!\n"); + return -WD_ENOMEM; + } + memset(sess->key, 0, MAX_CIPHER_KEY_SIZE); + + return 0; +} + handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) { struct wd_cipher_sess *sess = NULL; @@ -282,16 +309,22 @@ handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) sess->alg = setup->alg; sess->mode = setup->mode; + /* Memory type set */ + if (cipher_setup_memory_and_buffers(sess, setup)) + goto free_sess; + /* Some simple scheduler don't need scheduling parameters */ sess->sched_key = (void *)wd_cipher_setting.sched.sched_init( wd_cipher_setting.sched.h_sched_ctx, setup->sched_param); if (WD_IS_ERR(sess->sched_key)) { WD_ERR("failed to init session schedule key!\n"); - goto free_sess; + goto free_key; } return (handle_t)sess; +free_key: + sess->mm_ops.free(sess->mm_ops.usr, sess->key); free_sess: free(sess); return (handle_t)0; @@ -307,6 +340,7 @@ void wd_cipher_free_sess(handle_t h_sess) } wd_memset_zero(sess->key, sess->key_bytes); + sess->mm_ops.free(sess->mm_ops.usr, sess->key); if (sess->sched_key) free(sess->sched_key); @@ -545,6 +579,8 @@ static void fill_request_msg(struct wd_cipher_msg *msg, msg->iv = req->iv; msg->iv_bytes = req->iv_bytes; msg->data_fmt = req->data_fmt; + msg->mm_ops = &sess->mm_ops; + msg->mm_type = sess->mm_type; } static int cipher_iv_len_check(struct wd_cipher_req *req, diff --git a/wd_digest.c b/wd_digest.c index 2d31176..0b37f8b 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -64,10 +64,12 @@ struct wd_digest_sess { enum wd_digest_type alg; enum wd_digest_mode mode; void *priv; - unsigned char key[MAX_HMAC_KEY_SIZE]; + unsigned char *key; __u32 key_bytes; void *sched_key; struct wd_digest_stream_data stream_data; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; struct wd_env_config wd_digest_env_config; @@ -187,6 +189,31 @@ int wd_digest_set_key(handle_t h_sess, const __u8 *key, __u32 key_len) return 0; } +static int digest_setup_memory_and_buffers(struct wd_digest_sess *sess, + struct wd_digest_sess_setup *setup) +{ + int ret; + + ret = wd_mem_ops_init(wd_digest_setting.config.ctxs[0].ctx, + &setup->mm_ops, setup->mm_type); + if (ret) { + WD_ERR("failed to init memory ops!\n"); + return ret; + } + + memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops)); + sess->mm_type = setup->mm_type; + + sess->key = sess->mm_ops.alloc(sess->mm_ops.usr, MAX_HMAC_KEY_SIZE); + if (!sess->key) { + WD_ERR("digest failed to alloc key memory!\n"); + return -WD_ENOMEM; + } + memset(sess->key, 0, MAX_HMAC_KEY_SIZE); + + return 0; +} + handle_t wd_digest_alloc_sess(struct wd_digest_sess_setup *setup) { struct wd_digest_sess *sess = NULL; @@ -215,16 +242,23 @@ handle_t wd_digest_alloc_sess(struct wd_digest_sess_setup *setup) WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); goto err_sess; } + + /* Memory type set */ + if (digest_setup_memory_and_buffers(sess, setup)) + goto err_sess; + /* Some simple scheduler don't need scheduling parameters */ sess->sched_key = (void *)wd_digest_setting.sched.sched_init( wd_digest_setting.sched.h_sched_ctx, setup->sched_param); if (WD_IS_ERR(sess->sched_key)) { WD_ERR("failed to init session schedule key!\n"); - goto err_sess; + goto err_key; } return (handle_t)sess; +err_key: + sess->mm_ops.free(sess->mm_ops.usr, sess->key); err_sess: free(sess); return (handle_t)0; @@ -240,6 +274,7 @@ void wd_digest_free_sess(handle_t h_sess) } wd_memset_zero(sess->key, sess->key_bytes); + sess->mm_ops.free(sess->mm_ops.usr, sess->key); if (sess->sched_key) free(sess->sched_key); free(sess); @@ -603,6 +638,9 @@ static void fill_request_msg(struct wd_digest_msg *msg, msg->partial_block = sess->stream_data.partial_block; msg->partial_bytes = sess->stream_data.partial_bytes; + msg->mm_ops = &sess->mm_ops; + msg->mm_type = sess->mm_type; + /* Use iv_bytes to store the stream message state */ msg->iv_bytes = sess->stream_data.msg_state; } -- 2.33.0

From: Zhushuai Yin <yinzhushuai@huawei.com> The newly added tool interface can support unified verification scenarios for SEC sva and nosva interfaces. Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- uadk_tool/benchmark/sec_uadk_benchmark.c | 527 ++++++++++++++++++++++- 1 file changed, 518 insertions(+), 9 deletions(-) diff --git a/uadk_tool/benchmark/sec_uadk_benchmark.c b/uadk_tool/benchmark/sec_uadk_benchmark.c index cbd2a99..141b161 100644 --- a/uadk_tool/benchmark/sec_uadk_benchmark.c +++ b/uadk_tool/benchmark/sec_uadk_benchmark.c @@ -8,6 +8,7 @@ #include "include/wd_digest.h" #include "include/wd_aead.h" #include "include/wd_sched.h" +#include "include/wd_bmm.h" #define SEC_TST_PRT printf #define MAX_IVK_LENTH 64 @@ -17,6 +18,8 @@ #define SEC_MAX_MAC_LEN 64 #define SEC_SAVE_FILE_LEN 64 #define SEC_PERF_AUTH_SIZE 16 +#define SQE_SIZE 128 +#define SEC_OP_TYPE_MAX 2 char aead_key[] = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf"; @@ -28,7 +31,7 @@ char g_save_mac[SEC_MAX_MAC_LEN]; struct uadk_bd { u8 *src; u8 *dst; - u8 mac[SEC_MAX_MAC_LEN]; + u8 *mac; }; struct bd_pool { @@ -40,6 +43,7 @@ struct thread_pool { u8 **iv; u8 **key; u8 **hash; + void *rsv_pool; } g_uadk_pool; typedef struct uadk_thread_res { @@ -54,6 +58,7 @@ typedef struct uadk_thread_res { u32 dalg; u32 dmode; u32 d_outbytes; + int mm_type; } thread_data; static struct wd_ctx_config g_ctx_cfg; @@ -66,6 +71,7 @@ static unsigned int g_alg; static unsigned int g_algtype; static unsigned int g_optype; static unsigned int g_maclen; +static unsigned int g_dev_id; struct aead_alg_info { int index; @@ -635,6 +641,7 @@ static int specified_device_request_ctx(struct acc_option *options) g_ctx_cfg.ctxs[i].op_type = 0; g_ctx_cfg.ctxs[i].ctx_mode = (__u8)mode; } + g_dev_id = uadk_parse_dev_id(dev->char_dev_path); wd_free_list_accels(list); return 0; @@ -677,6 +684,7 @@ static int non_specified_device_request_ctx(struct acc_option *options) g_ctx_cfg.ctxs[i].op_type = 0; g_ctx_cfg.ctxs[i].ctx_mode = (__u8)mode; } + g_dev_id = uadk_parse_dev_id(dev->char_dev_path); free(dev); } @@ -718,15 +726,31 @@ static int init_ctx_config(struct acc_option *options) goto free_ctxs; } + switch(subtype) { case CIPHER_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_cipher_poll_ctx); + if (options->mem_type == UADK_AUTO) + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, SEC_OP_TYPE_MAX, + max_node, wd_cipher_poll_ctx); + else + g_sched = wd_sched_rr_alloc(SCHED_POLICY_DEV, SEC_OP_TYPE_MAX, + max_node, wd_cipher_poll_ctx); break; case AEAD_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_aead_poll_ctx); + if (options->mem_type == UADK_AUTO) + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, SEC_OP_TYPE_MAX, + max_node, wd_aead_poll_ctx); + else + g_sched = wd_sched_rr_alloc(SCHED_POLICY_DEV, SEC_OP_TYPE_MAX, + max_node, wd_aead_poll_ctx); break; case DIGEST_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_digest_poll_ctx); + if (options->mem_type == UADK_AUTO) + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, SEC_OP_TYPE_MAX, + max_node, wd_digest_poll_ctx); + else + g_sched = wd_sched_rr_alloc(SCHED_POLICY_DEV, SEC_OP_TYPE_MAX, + max_node, wd_digest_poll_ctx); break; default: SEC_TST_PRT("failed to parse alg subtype!\n"); @@ -743,6 +767,7 @@ static int init_ctx_config(struct acc_option *options) param.mode = mode; param.begin = 0; param.end = g_ctxnum - 1; + param.dev_id = g_dev_id; ret = wd_sched_rr_instance(g_sched, ¶m); if (ret) { SEC_TST_PRT("failed to fill sched data!\n"); @@ -867,7 +892,10 @@ static int init_ctx_config2(struct acc_option *options) /* init */ switch(subtype) { case CIPHER_TYPE: - ret = wd_cipher_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + if (options->mem_type == UADK_AUTO) + ret = wd_cipher_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + else + ret = wd_cipher_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); if (ret) SEC_TST_PRT("failed to do cipher init2!\n"); break; @@ -877,12 +905,18 @@ static int init_ctx_config2(struct acc_option *options) SEC_TST_PRT("failed to do cipher intruction init2!\n"); break; case AEAD_TYPE: - ret = wd_aead_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + if (options->mem_type == UADK_AUTO) + ret = wd_aead_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + else + ret = wd_aead_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); if (ret) SEC_TST_PRT("failed to do aead init2!\n"); break; case DIGEST_TYPE: - ret = wd_digest_init2_(alg_name, options->sched_type, options->task_type, &cparams); + if (options->mem_type == UADK_AUTO) + ret = wd_digest_init2_(alg_name, SCHED_POLICY_RR, options->task_type, &cparams); + else + ret = wd_digest_init2_(alg_name, SCHED_POLICY_DEV, options->task_type, &cparams); if (ret) SEC_TST_PRT("failed to do digest init2!\n"); break; @@ -1071,6 +1105,10 @@ static int init_uadk_bd_pool(void) memset(g_uadk_pool.pool[i].bds[j].dst, 0, step); if (!g_uadk_pool.pool[i].bds[j].dst) goto malloc_error3; + g_uadk_pool.pool[i].bds[j].mac = malloc(SEC_MAX_MAC_LEN); + memset(g_uadk_pool.pool[i].bds[j].mac, 0, SEC_MAX_MAC_LEN); + if (!g_uadk_pool.pool[i].bds[j].mac) + goto malloc_error4; if (g_alg != AEAD_TYPE) { get_rand_data(g_uadk_pool.pool[i].bds[j].src, g_pktlen); @@ -1093,18 +1131,22 @@ static int init_uadk_bd_pool(void) return 0; +malloc_error4: + free(g_uadk_pool.pool[i].bds[j].dst); malloc_error3: free(g_uadk_pool.pool[i].bds[j].src); malloc_error2: for (j--; j >= 0; j--) { free(g_uadk_pool.pool[i].bds[j].src); free(g_uadk_pool.pool[i].bds[j].dst); + free(g_uadk_pool.pool[i].bds[j].mac); } malloc_error1: for (i--; i >= 0; i--) { for (j = 0; j < MAX_POOL_LENTH; j++) { free(g_uadk_pool.pool[i].bds[j].src); free(g_uadk_pool.pool[i].bds[j].dst); + free(g_uadk_pool.pool[i].bds[j].mac); } free(g_uadk_pool.pool[i].bds); g_uadk_pool.pool[i].bds = NULL; @@ -1133,6 +1175,7 @@ static void free_uadk_bd_pool(void) for (j = 0; j < MAX_POOL_LENTH; j++) { free(g_uadk_pool.pool[i].bds[j].src); free(g_uadk_pool.pool[i].bds[j].dst); + free(g_uadk_pool.pool[i].bds[j].mac); } } free(g_uadk_pool.pool[i].bds); @@ -1144,6 +1187,322 @@ static void free_uadk_bd_pool(void) free_ivkey_source(); } +static void free_rsv_ivkey(void *pool) +{ + int i; + + /* release the hash block for each thread */ + for (i = 0; i < g_thread_num; i++) { + if (g_uadk_pool.hash[i]) + wd_mem_free(pool, g_uadk_pool.hash[i]); + } + + /* release the key block for each thread */ + for (i = 0; i < g_thread_num; i++) { + if (g_uadk_pool.key[i]) + wd_mem_free(pool, g_uadk_pool.key[i]); + } + + /* release the IV block for each thread */ + for (i = 0; i < g_thread_num; i++) { + if (g_uadk_pool.iv[i]) + wd_mem_free(pool, g_uadk_pool.iv[i]); + } + + /* release a pointer array */ + free(g_uadk_pool.hash); + free(g_uadk_pool.key); + free(g_uadk_pool.iv); + + /* the memory pool is managed externally */ +} + +static int init_rsv_ivkey(handle_t h_ctx, void *pool) +{ + int i, j, m, idx; + + /* Check if the incoming memory pool is valid */ + if (!pool) { + SEC_TST_PRT("Invalid pool parameter\n"); + return -EINVAL; + } + + /* + * Allocate a pointer array + * (still using malloc, as the structure is small) + */ + g_uadk_pool.iv = malloc(g_thread_num * sizeof(char *)); + if (!g_uadk_pool.iv) { + SEC_TST_PRT("Failed to alloc IV pointers\n"); + goto error; + } + memset(g_uadk_pool.iv, 0, g_thread_num * sizeof(char *)); + + g_uadk_pool.key = malloc(g_thread_num * sizeof(char *)); + if (!g_uadk_pool.key) { + SEC_TST_PRT("Failed to alloc Key pointers\n"); + goto free_iv_pointers; + } + memset(g_uadk_pool.key, 0, g_thread_num * sizeof(char *)); + + g_uadk_pool.hash = malloc(g_thread_num * sizeof(char *)); + if (!g_uadk_pool.hash) { + SEC_TST_PRT("Failed to alloc Hash pointers\n"); + goto free_key_pointers; + } + memset(g_uadk_pool.hash, 0, g_thread_num * sizeof(char *)); + + /* Allocate IV blocks for each thread */ + for (i = 0; i < g_thread_num; i++) { + g_uadk_pool.iv[i] = wd_mem_alloc(pool, g_pktlen); + if (!g_uadk_pool.iv[i]) { + SEC_TST_PRT("Failed to alloc IV block\n"); + goto free_iv_blocks; + } + memset(g_uadk_pool.iv[i], 0, MAX_IVK_LENTH); + } + + /* Allocate KEY blocks for each thread */ + for (j = 0; j < g_thread_num; j++) { + g_uadk_pool.key[j] = wd_mem_alloc(pool, g_pktlen); + if (!g_uadk_pool.key[j]) { + SEC_TST_PRT("Failed to alloc Key block\n"); + goto free_key_blocks; + } + memcpy(g_uadk_pool.key[j], aead_key, SEC_PERF_KEY_LEN); + } + + /* Allocate HASH blocks for each thread */ + for (m = 0; m < g_thread_num; m++) { + g_uadk_pool.hash[m] = wd_mem_alloc(pool, g_pktlen); + if (!g_uadk_pool.hash[m]) { + SEC_TST_PRT("Failed to alloc Hash block\n"); + goto free_hash_blocks; + } + memcpy(g_uadk_pool.hash[m], aead_key, SEC_PERF_KEY_LEN); + } + + return 0; + +free_hash_blocks: + for (idx = m - 1; idx >= 0; idx--) { + if (g_uadk_pool.hash[idx]) + wd_mem_free(pool, g_uadk_pool.hash[idx]); + } +free_key_blocks: + for (idx = j - 1; idx >= 0; idx--) { + if (g_uadk_pool.key[idx]) + wd_mem_free(pool, g_uadk_pool.key[idx]); + } +free_iv_blocks: + for (idx = i - 1; idx >= 0; idx--) { + if (g_uadk_pool.iv[idx]) + wd_mem_free(pool, g_uadk_pool.iv[idx]); + } + free(g_uadk_pool.hash); +free_key_pointers: + free(g_uadk_pool.key); +free_iv_pointers: + free(g_uadk_pool.iv); +error: + return -ENOMEM; +} + +static int init_uadk_rsv_pool(struct acc_option *option) +{ + struct wd_mempool_setup pool_setup; + char *alg = option->algclass; + unsigned long step, len; + /* + * Assuming that h_ctx exists globally, + * it needs to be obtained based on actual conditions. + */ + handle_t h_ctx; + int i, j; + int ret; + + h_ctx = wd_find_ctx(alg); + if (!h_ctx) { + SEC_TST_PRT("Failed to find a ctx for alg:%s\n", option->algname); + return -EINVAL; + } + g_ctx_cfg.priv = (void *)h_ctx; + + if (g_alg != AEAD_TYPE) + step = sizeof(char) * g_pktlen; + else + step = sizeof(char) * g_pktlen * 2; + + /* Create a memory pool for managing memory blocks of src and dst */ + pool_setup.block_size = step; + /* + * Each thread requires two blocks (src and dst) for each uadk_bd, + * along with the IV, KEY, and MAC sections, + * thus necessitating five times the data. + */ + pool_setup.block_num = g_thread_num * MAX_POOL_LENTH * 8; + pool_setup.align_size = SQE_SIZE; + pool_setup.ops.alloc = NULL; + pool_setup.ops.free = NULL; + + g_uadk_pool.rsv_pool = wd_mempool_alloc(h_ctx, &pool_setup); + if (!g_uadk_pool.rsv_pool) { + SEC_TST_PRT("Failed to create block pool\n"); + return -ENOMEM; + } + + /* Initialize the memory for iv and key */ + ret = init_rsv_ivkey(h_ctx, g_uadk_pool.rsv_pool); + if (ret) { + SEC_TST_PRT("init uadk ivkey resource failed!\n"); + goto free_pool; + } + + if (g_alg != AEAD_TYPE) + len = sizeof(char) * g_pktlen; + else + len = sizeof(char) * g_pktlen * 2; + /* + * Allocate thread pool structure + * (still using malloc, as the structure is small) + */ + g_uadk_pool.pool = calloc(1, g_thread_num * sizeof(struct bd_pool)); + if (!g_uadk_pool.pool) { + SEC_TST_PRT("init uadk pool alloc thread failed!\n"); + goto free_ivkey; + } + + for (i = 0; i < g_thread_num; i++) { + g_uadk_pool.pool[i].bds = calloc(1, MAX_POOL_LENTH * sizeof(struct uadk_bd)); + if (!g_uadk_pool.pool[i].bds) { + SEC_TST_PRT("init uadk bds alloc failed!\n"); + goto malloc_error1; + } + + for (j = 0; j < MAX_POOL_LENTH; j++) { + /* Allocate memory blocks for src and dst from the memory pool */ + g_uadk_pool.pool[i].bds[j].src = wd_mem_alloc(g_uadk_pool.rsv_pool, len); + if (!g_uadk_pool.pool[i].bds[j].src) { + SEC_TST_PRT("Failed to alloc src block\n"); + goto malloc_error3; + } + + g_uadk_pool.pool[i].bds[j].dst = wd_mem_alloc(g_uadk_pool.rsv_pool, len); + if (!g_uadk_pool.pool[i].bds[j].dst) { + SEC_TST_PRT("Failed to alloc dst block\n"); + goto malloc_error3; + } + + g_uadk_pool.pool[i].bds[j].mac = wd_mem_alloc(g_uadk_pool.rsv_pool, SEC_MAX_MAC_LEN); + if (!g_uadk_pool.pool[i].bds[j].mac) { + SEC_TST_PRT("Failed to alloc mac block\n"); + goto malloc_error3; + } + + memset(g_uadk_pool.pool[i].bds[j].src, 0, len); + memset(g_uadk_pool.pool[i].bds[j].dst, 0, len); + memset(g_uadk_pool.pool[i].bds[j].mac, 0, step); + + if (g_alg != AEAD_TYPE) { + get_rand_data(g_uadk_pool.pool[i].bds[j].src, g_pktlen); + if (g_prefetch) + get_rand_data(g_uadk_pool.pool[i].bds[j].dst, g_pktlen); + } else { + if (!g_optype) + get_aead_data(g_uadk_pool.pool[i].bds[j].src, g_pktlen + SEC_AEAD_LEN); + else { + read_aead_dst_data(g_uadk_pool.pool[i].bds[j].src, g_pktlen + SEC_AEAD_LEN); + memcpy(g_uadk_pool.pool[i].bds[j].mac, g_save_mac, SEC_MAX_MAC_LEN); + } + } + } + } + SEC_TST_PRT("Init uadk rsv block pool OK.\n"); + + return 0; + +malloc_error3: + /* Release the allocated src and dst blocks */ + if (g_uadk_pool.pool[i].bds[j].mac) + wd_mem_free(g_uadk_pool.rsv_pool, g_uadk_pool.pool[i].bds[j].mac); + if (g_uadk_pool.pool[i].bds[j].src) + wd_mem_free(g_uadk_pool.rsv_pool, g_uadk_pool.pool[i].bds[j].src); + if (g_uadk_pool.pool[i].bds[j].dst) + wd_mem_free(g_uadk_pool.rsv_pool, g_uadk_pool.pool[i].bds[j].dst); + goto malloc_error2; + +malloc_error2: + for (j--; j >= 0; j--) { + if (g_uadk_pool.pool[i].bds[j].src) + wd_mem_free(g_uadk_pool.rsv_pool, g_uadk_pool.pool[i].bds[j].src); + if (g_uadk_pool.pool[i].bds[j].dst) + wd_mem_free(g_uadk_pool.rsv_pool, g_uadk_pool.pool[i].bds[j].dst); + if (g_uadk_pool.pool[i].bds[j].mac) + wd_mem_free(g_uadk_pool.rsv_pool, g_uadk_pool.pool[i].bds[j].mac); + } +malloc_error1: + for (i--; i >= 0; i--) { + for (j = 0; j < MAX_POOL_LENTH; j++) { + if (g_uadk_pool.pool[i].bds[j].src) + wd_mem_free(g_uadk_pool.rsv_pool, g_uadk_pool.pool[i].bds[j].src); + if (g_uadk_pool.pool[i].bds[j].dst) + wd_mem_free(g_uadk_pool.rsv_pool, g_uadk_pool.pool[i].bds[j].dst); + if (g_uadk_pool.pool[i].bds[j].mac) + wd_mem_free(g_uadk_pool.rsv_pool, g_uadk_pool.pool[i].bds[j].mac); + } + free(g_uadk_pool.pool[i].bds); + g_uadk_pool.pool[i].bds = NULL; + } + free(g_uadk_pool.pool); + g_uadk_pool.pool = NULL; + +free_ivkey: + free_rsv_ivkey(g_uadk_pool.rsv_pool); + +free_pool: + wd_mempool_free(h_ctx, g_uadk_pool.rsv_pool); + g_uadk_pool.rsv_pool = NULL; + + SEC_TST_PRT("init uadk bd pool alloc failed!\n"); + return -ENOMEM; +} + +static void free_uadk_rsv_pool(struct acc_option *option) +{ + handle_t h_ctx = (handle_t)g_ctx_cfg.priv; + int i, j; + + /* save aad + ctext + mac */ + if (g_alg == AEAD_TYPE && !g_optype) + save_aead_dst_data(g_uadk_pool.pool[0].bds[0].dst, + g_pktlen + SEC_AEAD_LEN); + + for (i = 0; i < g_thread_num; i++) { + if (g_uadk_pool.pool[i].bds) { + for (j = 0; j < MAX_POOL_LENTH; j++) { + /* Use wd_mem_free to release the block */ + if (g_uadk_pool.pool[i].bds[j].src) + wd_mem_free(g_uadk_pool.rsv_pool, g_uadk_pool.pool[i].bds[j].src); + if (g_uadk_pool.pool[i].bds[j].dst) + wd_mem_free(g_uadk_pool.rsv_pool, g_uadk_pool.pool[i].bds[j].dst); + if (g_uadk_pool.pool[i].bds[j].mac) + wd_mem_free(g_uadk_pool.rsv_pool, g_uadk_pool.pool[i].bds[j].mac); + } + } + free(g_uadk_pool.pool[i].bds); + g_uadk_pool.pool[i].bds = NULL; + } + free(g_uadk_pool.pool); + g_uadk_pool.pool = NULL; + + /* Destroy IV and key */ + free_rsv_ivkey(g_uadk_pool.rsv_pool); + + /* Destroy memory pool */ + if (g_uadk_pool.rsv_pool) + wd_mempool_free(h_ctx, g_uadk_pool.rsv_pool); + g_uadk_pool.rsv_pool = NULL; +} /*-------------------------------uadk benchmark main code-------------------------------------*/ static void *sec_uadk_poll(void *data) @@ -1245,6 +1604,7 @@ static void *sec_uadk_cipher_async(void *arg) { thread_data *pdata = (thread_data *)arg; struct wd_cipher_sess_setup cipher_setup = {0}; + struct sched_params sc_param = {0}; struct wd_cipher_req creq; struct bd_pool *uadk_pool; u8 *priv_iv, *priv_key; @@ -1265,6 +1625,20 @@ static void *sec_uadk_cipher_async(void *arg) cipher_setup.alg = pdata->alg; cipher_setup.mode = pdata->mode; + sc_param.numa_id = 0; + sc_param.type = 0; + sc_param.mode = 0; + if (g_uadk_pool.rsv_pool) + sc_param.dev_id = wd_get_dev_id(g_uadk_pool.rsv_pool); + cipher_setup.sched_param = (void *)&sc_param; + + cipher_setup.mm_type = pdata->mm_type; + cipher_setup.mm_ops.usr = g_uadk_pool.rsv_pool; + cipher_setup.mm_ops.alloc = (void *)wd_mem_alloc; + cipher_setup.mm_ops.free = (void *)wd_mem_free; + cipher_setup.mm_ops.iova_map = (void *)wd_mem_map; + cipher_setup.mm_ops.iova_unmap = (void *)wd_mem_unmap; + cipher_setup.mm_ops.get_bufsize = (void *)wd_get_bufsize; h_sess = wd_cipher_alloc_sess(&cipher_setup); if (!h_sess) return NULL; @@ -1305,6 +1679,22 @@ static void *sec_uadk_cipher_async(void *arg) } count++; } + + /* Release memory after all tasks are complete. */ + if (count) { + i = 0; + while (get_recv_time() != g_ctxnum) { + if (i++ >= MAX_TRY_CNT) { + SEC_TST_PRT("failed to wait poll thread finish!\n"); + break; + } + + usleep(SEND_USLEEP); + } + } + /* Wait for the device to complete the tasks. */ + usleep(SEND_USLEEP * MAX_TRY_CNT); + wd_cipher_free_sess(h_sess); add_send_complete(); @@ -1316,6 +1706,7 @@ static void *sec_uadk_aead_async(void *arg) { thread_data *pdata = (thread_data *)arg; struct wd_aead_sess_setup aead_setup = {0}; + struct sched_params sc_param = {0}; u8 *priv_iv, *priv_key, *priv_hash; u32 auth_size = SEC_PERF_AUTH_SIZE; struct wd_aead_req areq = {0}; @@ -1338,6 +1729,21 @@ static void *sec_uadk_aead_async(void *arg) aead_setup.calg = pdata->alg; aead_setup.cmode = pdata->mode; + aead_setup.mm_type = pdata->mm_type; + sc_param.numa_id = 0; + sc_param.type = 0; + sc_param.mode = 0; // sync mode + if (g_uadk_pool.rsv_pool) + sc_param.dev_id = wd_get_dev_id(g_uadk_pool.rsv_pool); + aead_setup.sched_param = (void *)&sc_param; + + aead_setup.mm_ops.usr = g_uadk_pool.rsv_pool; + aead_setup.mm_ops.alloc = (void *)wd_mem_alloc; + aead_setup.mm_ops.free = (void *)wd_mem_free; + aead_setup.mm_ops.iova_map = (void *)wd_mem_map; + aead_setup.mm_ops.iova_unmap = (void *)wd_mem_unmap; + aead_setup.mm_ops.get_bufsize = (void *)wd_get_bufsize; + if (pdata->is_union) { aead_setup.dalg = pdata->dalg; aead_setup.dmode = pdata->dmode; @@ -1412,6 +1818,22 @@ static void *sec_uadk_aead_async(void *arg) } count++; } + + /* Release memory after all tasks are complete. */ + if (count) { + i = 0; + while (get_recv_time() != g_ctxnum) { + if (i++ >= MAX_TRY_CNT) { + SEC_TST_PRT("failed to wait poll thread finish!\n"); + break; + } + + usleep(SEND_USLEEP); + } + } + /* Wait for the device to complete the tasks. */ + usleep(SEND_USLEEP * MAX_TRY_CNT); + wd_aead_free_sess(h_sess); add_send_complete(); @@ -1423,6 +1845,7 @@ static void *sec_uadk_digest_async(void *arg) { thread_data *pdata = (thread_data *)arg; struct wd_digest_sess_setup digest_setup = {0}; + struct sched_params sc_param = {0}; struct wd_digest_req dreq; struct bd_pool *uadk_pool; u8 *priv_iv, *priv_key; @@ -1443,6 +1866,21 @@ static void *sec_uadk_digest_async(void *arg) digest_setup.alg = pdata->alg; digest_setup.mode = pdata->mode; // digest mode is optype + digest_setup.mm_type = pdata->mm_type; + sc_param.numa_id = 0; + sc_param.type = 0; + sc_param.mode = 0; // sync mode + if (g_uadk_pool.rsv_pool) + sc_param.dev_id = wd_get_dev_id(g_uadk_pool.rsv_pool); + digest_setup.sched_param = (void *)&sc_param; + + digest_setup.mm_ops.usr = g_uadk_pool.rsv_pool; + digest_setup.mm_ops.alloc = (void *)wd_mem_alloc; + digest_setup.mm_ops.free = (void *)wd_mem_free; + digest_setup.mm_ops.iova_map = (void *)wd_mem_map; + digest_setup.mm_ops.iova_unmap = (void *)wd_mem_unmap; + digest_setup.mm_ops.get_bufsize = (void *)wd_get_bufsize; + h_sess = wd_digest_alloc_sess(&digest_setup); if (!h_sess) return NULL; @@ -1482,6 +1920,22 @@ static void *sec_uadk_digest_async(void *arg) } count++; } + + /* Release memory after all tasks are complete. */ + if (count) { + i = 0; + while (get_recv_time() != g_ctxnum) { + if (i++ >= MAX_TRY_CNT) { + SEC_TST_PRT("failed to wait poll thread finish!\n"); + break; + } + + usleep(SEND_USLEEP); + } + } + /* Wait for the device to complete the tasks. */ + usleep(SEND_USLEEP * MAX_TRY_CNT); + wd_digest_free_sess(h_sess); add_send_complete(); @@ -1493,6 +1947,7 @@ static void *sec_uadk_cipher_sync(void *arg) { thread_data *pdata = (thread_data *)arg; struct wd_cipher_sess_setup cipher_setup = {0}; + struct sched_params sc_param = {0}; struct wd_cipher_req creq; struct bd_pool *uadk_pool; u8 *priv_iv, *priv_key; @@ -1512,6 +1967,20 @@ static void *sec_uadk_cipher_sync(void *arg) cipher_setup.alg = pdata->alg; cipher_setup.mode = pdata->mode; + sc_param.numa_id = 0; + sc_param.type = 0; + sc_param.mode = 0; // sync mode + if (g_uadk_pool.rsv_pool) + sc_param.dev_id = wd_get_dev_id(g_uadk_pool.rsv_pool); + cipher_setup.sched_param = (void *)&sc_param; + + cipher_setup.mm_type = pdata->mm_type; + cipher_setup.mm_ops.usr = g_uadk_pool.rsv_pool; + cipher_setup.mm_ops.alloc = (void *)wd_mem_alloc; + cipher_setup.mm_ops.free = (void *)wd_mem_free; + cipher_setup.mm_ops.iova_map = (void *)wd_mem_map; + cipher_setup.mm_ops.iova_unmap = (void *)wd_mem_unmap; + cipher_setup.mm_ops.get_bufsize = (void *)wd_get_bufsize; h_sess = wd_cipher_alloc_sess(&cipher_setup); if (!h_sess) return NULL; @@ -1554,6 +2023,7 @@ static void *sec_uadk_aead_sync(void *arg) { thread_data *pdata = (thread_data *)arg; struct wd_aead_sess_setup aead_setup = {0}; + struct sched_params sc_param = {0}; u8 *priv_iv, *priv_key, *priv_hash; u32 auth_size = SEC_PERF_AUTH_SIZE; struct wd_aead_req areq = {0}; @@ -1576,6 +2046,20 @@ static void *sec_uadk_aead_sync(void *arg) aead_setup.calg = pdata->alg; aead_setup.cmode = pdata->mode; + aead_setup.mm_type = pdata->mm_type; + sc_param.numa_id = 0; + sc_param.type = 0; + sc_param.mode = 0; // sync mode + if (g_uadk_pool.rsv_pool) + sc_param.dev_id = wd_get_dev_id(g_uadk_pool.rsv_pool); + aead_setup.sched_param = (void *)&sc_param; + + aead_setup.mm_ops.usr = g_uadk_pool.rsv_pool; + aead_setup.mm_ops.alloc = (void *)wd_mem_alloc; + aead_setup.mm_ops.free = (void *)wd_mem_free; + aead_setup.mm_ops.iova_map = (void *)wd_mem_map; + aead_setup.mm_ops.iova_unmap = (void *)wd_mem_unmap; + aead_setup.mm_ops.get_bufsize = (void *)wd_get_bufsize; if (pdata->is_union) { aead_setup.dalg = pdata->dalg; aead_setup.dmode = pdata->dmode; @@ -1644,6 +2128,7 @@ static void *sec_uadk_digest_sync(void *arg) { thread_data *pdata = (thread_data *)arg; struct wd_digest_sess_setup digest_setup = {0}; + struct sched_params sc_param = {0}; struct wd_digest_req dreq; struct bd_pool *uadk_pool; u8 *priv_iv, *priv_key; @@ -1663,6 +2148,20 @@ static void *sec_uadk_digest_sync(void *arg) digest_setup.alg = pdata->alg; digest_setup.mode = pdata->mode; // digest mode is optype + digest_setup.mm_type = pdata->mm_type; + sc_param.numa_id = 0; + sc_param.type = 0; + sc_param.mode = 0; // sync mode + if (g_uadk_pool.rsv_pool) + sc_param.dev_id = wd_get_dev_id(g_uadk_pool.rsv_pool); + digest_setup.sched_param = (void *)&sc_param; + + digest_setup.mm_ops.usr = g_uadk_pool.rsv_pool; + digest_setup.mm_ops.alloc = (void *)wd_mem_alloc; + digest_setup.mm_ops.free = (void *)wd_mem_free; + digest_setup.mm_ops.iova_map = (void *)wd_mem_map; + digest_setup.mm_ops.iova_unmap = (void *)wd_mem_unmap; + digest_setup.mm_ops.get_bufsize = (void *)wd_get_bufsize; h_sess = wd_digest_alloc_sess(&digest_setup); if (!h_sess) return NULL; @@ -1741,6 +2240,7 @@ int sec_uadk_sync_threads(struct acc_option *options) threads_args[i].optype = threads_option.optype; threads_args[i].td_id = i; threads_args[i].d_outbytes = threads_option.d_outbytes; + threads_args[i].mm_type = options->mem_type; ret = pthread_create(&tdid[i], NULL, uadk_sec_sync_run, &threads_args[i]); if (ret) { SEC_TST_PRT("Create sync thread fail!\n"); @@ -1813,6 +2313,7 @@ int sec_uadk_async_threads(struct acc_option *options) threads_args[i].optype = threads_option.optype; threads_args[i].td_id = i; threads_args[i].d_outbytes = threads_option.d_outbytes; + threads_args[i].mm_type = options->mem_type; ret = pthread_create(&tdid[i], NULL, uadk_sec_async_run, &threads_args[i]); if (ret) { SEC_TST_PRT("Create async thread fail!\n"); @@ -1875,7 +2376,11 @@ int sec_uadk_benchmark(struct acc_option *options) if (ret) return ret; - ret = init_uadk_bd_pool(); + if (options->mem_type == UADK_AUTO) // SVA memory + ret = init_uadk_bd_pool(); + else + ret = init_uadk_rsv_pool(options); // In the test scenario, the user uses the uadk interface to apply for memory + if (ret) return ret; @@ -1889,7 +2394,11 @@ int sec_uadk_benchmark(struct acc_option *options) if (ret) return ret; - free_uadk_bd_pool(); + if (options->mem_type == UADK_AUTO) + free_uadk_bd_pool(); + else + free_uadk_rsv_pool(options); + if (options->inittype == INIT2_TYPE) uninit_ctx_config2(options->subtype); else -- 2.33.0

From: lizhi <lizhi206@huawei.com> The newly added tool interface can support unified verification scenarios for hpre sva and nosva interfaces. Signed-off-by: lizhi <lizhi206@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- uadk_tool/benchmark/hpre_protocol_data.h | 7 + uadk_tool/benchmark/hpre_uadk_benchmark.c | 323 +++++++++++++++++++--- uadk_tool/benchmark/hpre_wd_benchmark.c | 15 +- 3 files changed, 304 insertions(+), 41 deletions(-) diff --git a/uadk_tool/benchmark/hpre_protocol_data.h b/uadk_tool/benchmark/hpre_protocol_data.h index 7bdb942..dcb5c85 100644 --- a/uadk_tool/benchmark/hpre_protocol_data.h +++ b/uadk_tool/benchmark/hpre_protocol_data.h @@ -1257,6 +1257,13 @@ static char ecdh_da_secp256k1[] = { 0x64, 0xfc, 0xbe, 0x4e, 0x44, 0x4a, 0xc6, 0x54, 0x71, 0x69, 0x2e, 0x4a, 0x46, 0xa7, 0x2d, 0xa3 }; +static char x448_da[] = { + 0x71, 0x71, 0xb4, 0x5b, 0x79, 0x51, 0x94, 0x70, 0x53, 0xf9, 0x77, 0x02, 0x64, 0xef, 0xc4, 0xdb, + 0x64, 0xfc, 0xbe, 0x4e, 0x44, 0x4a, 0xc6, 0x54, 0x71, 0x69, 0x2e, 0x4a, 0x46, 0xa7, 0x2d, 0xa3, + 0x68, 0xae, 0x87, 0x7c, 0x45, 0xb3, 0x8b, 0xa8, 0xa8, 0x8e, 0x4b, 0xe5, 0x1f, 0x4e, 0xe6, 0x89, + 0xe2, 0x74, 0x69, 0xc8, 0x17, 0x6c, 0x0d, 0xca +}; + #if 0 static char ecdh_db_secp256k1[] = { 0xe2, 0x74, 0x69, 0xc8, 0x17, 0x6c, 0x0d, 0xca, 0xdd, 0x9e, 0xf6, 0x2f, 0x30, 0x9f, 0xad, 0xf3, diff --git a/uadk_tool/benchmark/hpre_uadk_benchmark.c b/uadk_tool/benchmark/hpre_uadk_benchmark.c index 1457236..fa26a61 100644 --- a/uadk_tool/benchmark/hpre_uadk_benchmark.c +++ b/uadk_tool/benchmark/hpre_uadk_benchmark.c @@ -10,12 +10,17 @@ #include "include/wd_dh.h" #include "include/wd_ecc.h" #include "include/wd_sched.h" +#include "include/wd_bmm.h" +#include "include/wd_sched.h" #define ECC_CURVE_SECP256R1 0x3 /* default set with secp256r1 */ #define HPRE_TST_PRT printf #define ERR_OPTYPE 0xFF #define SM2_DG_SZ 1024 #define WD_SECP256R1 0x18 /* consistent with wd_ecc.c */ +#define SQE_SIZE 128 +#define POOL_MULTIPLY_FACTOR 2 +#define HPRE_OP_TYPE_MAX 6 struct hpre_rsa_key_in { void *e; @@ -87,6 +92,10 @@ struct hpre_ecc_setup { //----------------------------------ECC param-------------------------------------// +struct thread_pool { + void *rsv_pool; +} hpre_uadk_pool; + typedef struct uadk_thread_res { u32 subtype; u32 keybits; @@ -94,12 +103,14 @@ typedef struct uadk_thread_res { u32 optype; u32 td_id; u32 algtype; + int mm_type; } thread_data; static struct wd_ctx_config g_ctx_cfg; static struct wd_sched *g_sched; static unsigned int g_thread_num; static unsigned int g_ctxnum; +static unsigned int g_dev_id; static const char* const alg_operations[] = { "GenKey", "ShareKey", "Encrypt", "Decrypt", "Sign", "Verify", @@ -340,6 +351,7 @@ static int hpre_uadk_param_parse(thread_data *tddata, struct acc_option *options tddata->kmode = mode; tddata->optype = optype; tddata->algtype = algtype; + tddata->mm_type = options->mem_type; HPRE_TST_PRT("%s to run %s task!\n", options->algclass, alg_operations[options->optype]); @@ -404,6 +416,7 @@ static int specified_device_request_ctx(struct acc_option *options) g_ctx_cfg.ctxs[i].op_type = 0; g_ctx_cfg.ctxs[i].ctx_mode = (__u8)mode; } + g_dev_id = uadk_parse_dev_id(dev->char_dev_path); wd_free_list_accels(list); return 0; @@ -446,7 +459,7 @@ static int non_specified_device_request_ctx(struct acc_option *options) g_ctx_cfg.ctxs[i].op_type = 0; g_ctx_cfg.ctxs[i].ctx_mode = (__u8)mode; } - + g_dev_id = uadk_parse_dev_id(dev->char_dev_path); free(dev); } @@ -489,17 +502,32 @@ static int init_hpre_ctx_config(struct acc_option *options) switch(subtype) { case RSA_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_rsa_poll_ctx); + if (options->mem_type == UADK_AUTO) + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, HPRE_OP_TYPE_MAX, + max_node, wd_rsa_poll_ctx); + else + g_sched = wd_sched_rr_alloc(SCHED_POLICY_DEV, HPRE_OP_TYPE_MAX, + max_node, wd_rsa_poll_ctx); break; case DH_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_dh_poll_ctx); + if (options->mem_type == UADK_AUTO) + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, HPRE_OP_TYPE_MAX, + max_node, wd_dh_poll_ctx); + else + g_sched = wd_sched_rr_alloc(SCHED_POLICY_DEV, HPRE_OP_TYPE_MAX, + max_node, wd_dh_poll_ctx); break; case ECDH_TYPE: case ECDSA_TYPE: case SM2_TYPE: case X25519_TYPE: case X448_TYPE: - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, 1, max_node, wd_ecc_poll_ctx); + if (options->mem_type == UADK_AUTO) + g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, HPRE_OP_TYPE_MAX, + max_node, wd_ecc_poll_ctx); + else + g_sched = wd_sched_rr_alloc(SCHED_POLICY_DEV, HPRE_OP_TYPE_MAX, + max_node, wd_ecc_poll_ctx); break; default: HPRE_TST_PRT("failed to parse alg subtype!\n"); @@ -516,6 +544,7 @@ static int init_hpre_ctx_config(struct acc_option *options) param.mode = mode; param.begin = 0; param.end = g_ctxnum - 1; + param.dev_id = g_dev_id; ret = wd_sched_rr_instance(g_sched, ¶m); if (ret) { HPRE_TST_PRT("failed to fill hpre sched data!\n"); @@ -651,15 +680,24 @@ static int init_hpre_ctx_config2(struct acc_option *options) /* init2 */ switch (subtype) { case RSA_TYPE: - return wd_rsa_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + if (options->mem_type == UADK_AUTO) + return wd_rsa_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + else + return wd_rsa_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); case DH_TYPE: - return wd_dh_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + if (options->mem_type == UADK_AUTO) + return wd_dh_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + else + return wd_dh_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); case ECDH_TYPE: case ECDSA_TYPE: case SM2_TYPE: case X25519_TYPE: case X448_TYPE: - return wd_ecc_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + if (options->mem_type == UADK_AUTO) + return wd_ecc_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + else + return wd_ecc_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); default: HPRE_TST_PRT("failed to parse alg subtype on uninit2!\n"); return -EINVAL; @@ -671,6 +709,108 @@ out_freectx: return ret; } +static int hpre_uadk_get_block(u32 algtype) +{ + int block_size = 512; + + switch (algtype) { + case RSA_1024: + block_size = 1280; + break; + case RSA_2048: + block_size = 2560; + break; + case RSA_3072: + block_size = 3840; + break; + case RSA_4096: + block_size = 5120; + break; + case RSA_1024_CRT: + block_size = 1280; + break; + case RSA_2048_CRT: + block_size = 2560; + break; + case RSA_3072_CRT: + block_size = 3840; + break; + case RSA_4096_CRT: + block_size = 5120; + break; + case DH_768: + block_size = 1536; + break; + case DH_1024: + block_size = 2048; + break; + case DH_1536: + block_size = 3072; + break; + case DH_2048: + block_size = 4096; + break; + case DH_3072: + block_size = 6144; + break; + case DH_4096: + block_size = 8192; + break; + default: + block_size = 576; + break; + } + + return block_size; +} + +static int init_uadk_rsv_pool(struct acc_option *options) +{ + struct wd_mempool_setup pool_setup; + char *alg = options->algclass; + handle_t h_ctx; + unsigned long step; + + /* ctxs is NULL */ + h_ctx = wd_find_ctx(alg); + if (!h_ctx) { + HPRE_TST_PRT("Failed to find a ctx for alg:%s\n", options->algname); + return -EINVAL; + } + g_ctx_cfg.priv = (void *)h_ctx; + + step = hpre_uadk_get_block(options->algtype); + + pool_setup.block_size = step; + pool_setup.block_num = g_thread_num * MAX_POOL_LENTH * POOL_MULTIPLY_FACTOR; + pool_setup.align_size = SQE_SIZE; + pool_setup.ops.alloc = NULL; + pool_setup.ops.free = NULL; + + hpre_uadk_pool.rsv_pool = wd_mempool_alloc(h_ctx, &pool_setup); + if (!hpre_uadk_pool.rsv_pool) { + HPRE_TST_PRT("Failed to create block pool\n"); + return -ENOMEM; + } + + pool_setup.ops.alloc = (void *)wd_mem_alloc; + pool_setup.ops.free = (void *)wd_mem_free; + pool_setup.ops.iova_map = (void *)wd_mem_map; + pool_setup.ops.iova_unmap = (void *)wd_mem_unmap; + pool_setup.ops.get_bufsize = (void *)wd_get_bufsize; + pool_setup.ops.usr = hpre_uadk_pool.rsv_pool; + + return 0; +} + +static void free_uadk_rsv_pool(struct acc_option *option) +{ + handle_t h_ctx = (handle_t)g_ctx_cfg.priv; + + if (hpre_uadk_pool.rsv_pool) + wd_mempool_free(h_ctx, hpre_uadk_pool.rsv_pool); + hpre_uadk_pool.rsv_pool = NULL; +} /*-------------------------------uadk benchmark main code-------------------------------------*/ void *hpre_uadk_poll(void *data) @@ -1049,7 +1189,7 @@ static int get_ecc_curve(struct hpre_ecc_setup *setup, u32 cid) return 0; } -static int get_ecc_key_param(struct wd_ecc_curve *param, u32 key_bits) +static int get_ecc_key_param(struct wd_ecc_curve *param, u32 key_bits) { u32 key_size = (key_bits + 7) / 8; @@ -1188,12 +1328,22 @@ static int get_ecc_param_from_sample(struct hpre_ecc_setup *setup, setup->sign_size = sizeof(sm2_sign_data); } else { - /* x448, x25519 and ecdh-256 can share same private key of ecdh_da_secp256k1*/ - setup->priv_key = ecdh_da_secp256k1; + /* + * x25519 and ecdh-256 can share same 32-bytes private key of + * ecdh_da_secp256k1, while x448 should use 56-byte private key + * to get accurate performance. + */ + if (subtype == X448_TYPE) + setup->priv_key = x448_da; + else + setup->priv_key = ecdh_da_secp256k1; setup->except_pub_key = ecdh_except_b_pubkey_secp256k1; setup->pub_key = ecdh_cp_pubkey_secp256k1; setup->share_key = ecdh_cp_sharekey_secp256k1; - setup->priv_key_size = sizeof(ecdh_da_secp256k1); + if (subtype == X448_TYPE) + setup->priv_key_size = sizeof(x448_da); + else + setup->priv_key_size = sizeof(ecdh_da_secp256k1); setup->except_pub_key_size = sizeof(ecdh_except_b_pubkey_secp256k1); setup->pub_key_size = sizeof(ecdh_cp_pubkey_secp256k1); setup->share_key_size = sizeof(ecdh_cp_sharekey_secp256k1); @@ -1340,6 +1490,7 @@ static void *rsa_uadk_sync_run(void *arg) { thread_data *pdata = (thread_data *)arg; int key_size = pdata->keybits >> 3; + struct sched_params sc_param = {0}; struct wd_rsa_sess_setup setup; struct wd_rsa_req req; void *key_info = NULL; @@ -1349,8 +1500,21 @@ static void *rsa_uadk_sync_run(void *arg) memset(&setup, 0, sizeof(setup)); memset(&req, 0, sizeof(req)); + sc_param.numa_id = 0; + sc_param.type = 0; + sc_param.mode = 0; + if (hpre_uadk_pool.rsv_pool) + sc_param.dev_id = wd_get_dev_id(hpre_uadk_pool.rsv_pool); + setup.sched_param = (void *)&sc_param; setup.key_bits = pdata->keybits; setup.is_crt = pdata->kmode; + setup.mm_type = pdata->mm_type; + setup.mm_ops.usr = hpre_uadk_pool.rsv_pool; + setup.mm_ops.alloc = (void *)wd_mem_alloc; + setup.mm_ops.free = (void *)wd_mem_free; + setup.mm_ops.iova_map = (void *)wd_mem_map; + setup.mm_ops.iova_unmap = (void *)wd_mem_unmap; + setup.mm_ops.get_bufsize = (void *)wd_get_bufsize; h_sess = wd_rsa_alloc_sess(&setup); if (!h_sess) @@ -1372,7 +1536,7 @@ static void *rsa_uadk_sync_run(void *arg) rsa_key_in->p = rsa_key_in->e + key_size; rsa_key_in->q = rsa_key_in->p + (key_size >> 1); - ret = get_rsa_key_from_sample(h_sess, key_info, key_info, + ret = get_rsa_key_from_sample(h_sess, key_info, key_info, pdata->keybits, pdata->kmode); if (ret) { HPRE_TST_PRT("failed to get sample key data!\n"); @@ -1389,14 +1553,14 @@ static void *rsa_uadk_sync_run(void *arg) goto sample_release; } } else { - req.src = malloc(key_size); + req.src = setup.mm_ops.alloc(setup.mm_ops.usr, key_size); if (!req.src) { HPRE_TST_PRT("failed to alloc rsa in buffer!\n"); goto sample_release; } memset(req.src, 0, req.src_bytes); - memcpy(req.src + key_size - sizeof(rsa_m), rsa_m, sizeof(rsa_m)); - req.dst = malloc(key_size); + memcpy(req.src + key_size - sizeof(rsa_m), rsa_m, sizeof(rsa_m)); + req.dst = setup.mm_ops.alloc(setup.mm_ops.usr, key_size); if (!req.dst) { HPRE_TST_PRT("failed to alloc rsa out buffer!\n"); goto src_release; @@ -1435,10 +1599,10 @@ static void *rsa_uadk_sync_run(void *arg) dst_release: if (req.dst) - free(req.dst); + setup.mm_ops.free(setup.mm_ops.usr, req.dst); src_release: if (req.src) - free(req.src); + setup.mm_ops.free(setup.mm_ops.usr, req.src); sample_release: free(rsa_key_in); key_release: @@ -1460,6 +1624,7 @@ static void *rsa_uadk_async_run(void *arg) { thread_data *pdata = (thread_data *)arg; int key_size = pdata->keybits >> 3; + struct sched_params sc_param = {0}; struct rsa_async_tag *tag; struct wd_rsa_sess_setup setup; struct wd_rsa_req req; @@ -1471,8 +1636,21 @@ static void *rsa_uadk_async_run(void *arg) memset(&setup, 0, sizeof(setup)); memset(&req, 0, sizeof(req)); + sc_param.numa_id = 0; + sc_param.type = 0; + sc_param.mode = 0; + if (hpre_uadk_pool.rsv_pool) + sc_param.dev_id = wd_get_dev_id(hpre_uadk_pool.rsv_pool); + setup.sched_param = (void *)&sc_param; setup.key_bits = pdata->keybits; setup.is_crt = pdata->kmode; + setup.mm_type = pdata->mm_type; + setup.mm_ops.usr = hpre_uadk_pool.rsv_pool; + setup.mm_ops.alloc = (void *)wd_mem_alloc; + setup.mm_ops.free = (void *)wd_mem_free; + setup.mm_ops.iova_map = (void *)wd_mem_map; + setup.mm_ops.iova_unmap = (void *)wd_mem_unmap; + setup.mm_ops.get_bufsize = (void *)wd_get_bufsize; h_sess = wd_rsa_alloc_sess(&setup); if (!h_sess) @@ -1518,14 +1696,14 @@ static void *rsa_uadk_async_run(void *arg) goto tag_release; } } else { - req.src = malloc(key_size); + req.src = setup.mm_ops.alloc(setup.mm_ops.usr, key_size); if (!req.src) { HPRE_TST_PRT("failed to alloc rsa in buffer!\n"); goto tag_release; } memset(req.src, 0, req.src_bytes); - memcpy(req.src + key_size - sizeof(rsa_m), rsa_m, sizeof(rsa_m)); - req.dst = malloc(key_size); + memcpy(req.src + key_size - sizeof(rsa_m), rsa_m, sizeof(rsa_m)); + req.dst = setup.mm_ops.alloc(setup.mm_ops.usr, key_size); if (!req.dst) { HPRE_TST_PRT("failed to alloc rsa out buffer!\n"); goto src_release; @@ -1581,10 +1759,10 @@ static void *rsa_uadk_async_run(void *arg) } if (req.dst) - free(req.dst); + setup.mm_ops.free(setup.mm_ops.usr, req.dst); src_release: if (req.src) - free(req.src); + setup.mm_ops.free(setup.mm_ops.usr, req.src); tag_release: free(tag); key_in_release: @@ -1691,27 +1869,27 @@ static int get_dh_param_from_sample(struct hpre_dh_param *setup, return 0; } -static int get_dh_opdata_param(handle_t h_sess, struct wd_dh_req *req, - struct hpre_dh_param *setup, int key_size) +static int get_dh_opdata_param(struct wd_dh_sess_setup *dh_setup, handle_t h_sess, + struct wd_dh_req *req, struct hpre_dh_param *setup, int key_size) { unsigned char *ag_bin = NULL; struct wd_dtb ctx_g; int ret; - ag_bin = malloc(2 * key_size); + ag_bin = dh_setup->mm_ops.alloc(dh_setup->mm_ops.usr, 2 * key_size); if (!ag_bin) return -ENOMEM; memset(ag_bin, 0, 2 * key_size); req->pv = ag_bin; - req->x_p = malloc(2 * key_size); + req->x_p = dh_setup->mm_ops.alloc(dh_setup->mm_ops.usr, 2 * key_size); if (!req->x_p) goto ag_error; memset(req->x_p, 0, 2 * key_size); - req->pri = malloc(2 * key_size); + req->pri = dh_setup->mm_ops.alloc(dh_setup->mm_ops.usr, 2 * key_size); if (!req->pri) goto xp_error; @@ -1748,11 +1926,11 @@ static int get_dh_opdata_param(handle_t h_sess, struct wd_dh_req *req, return 0; ctx_release: - free(req->pri); + dh_setup->mm_ops.free(dh_setup->mm_ops.usr, req->pri); xp_error: - free(req->x_p); + dh_setup->mm_ops.free(dh_setup->mm_ops.usr, req->x_p); ag_error: - free(req->pv); + dh_setup->mm_ops.free(dh_setup->mm_ops.usr, req->pv); return -ENOMEM; } @@ -1766,6 +1944,7 @@ static void *dh_uadk_async_run(void *arg) { thread_data *pdata = (thread_data *)arg; int key_size = pdata->keybits >> 3; + struct sched_params sc_param = {0}; struct wd_dh_sess_setup dh_setup; struct rsa_async_tag *tag; struct hpre_dh_param param; @@ -1777,6 +1956,19 @@ static void *dh_uadk_async_run(void *arg) memset(&dh_setup, 0, sizeof(dh_setup)); memset(&req, 0, sizeof(req)); + sc_param.numa_id = 0; + sc_param.type = 0; + sc_param.mode = 0; + if (hpre_uadk_pool.rsv_pool) + sc_param.dev_id = wd_get_dev_id(hpre_uadk_pool.rsv_pool); + dh_setup.sched_param = (void *)&sc_param; + dh_setup.mm_type = pdata->mm_type; + dh_setup.mm_ops.usr = hpre_uadk_pool.rsv_pool; + dh_setup.mm_ops.alloc = (void *)wd_mem_alloc; + dh_setup.mm_ops.free = (void *)wd_mem_free; + dh_setup.mm_ops.iova_map = (void *)wd_mem_map; + dh_setup.mm_ops.iova_unmap = (void *)wd_mem_unmap; + dh_setup.mm_ops.get_bufsize = (void *)wd_get_bufsize; dh_setup.key_bits = pdata->keybits; if (pdata->optype == WD_DH_PHASE2) dh_setup.is_g2 = true; // G1 is 0; G2 is 1; @@ -1791,7 +1983,7 @@ static void *dh_uadk_async_run(void *arg) param.optype = pdata->optype; req.op_type = pdata->optype; - ret = get_dh_opdata_param(h_sess, &req, ¶m, key_size); + ret = get_dh_opdata_param(&dh_setup, h_sess, &req, ¶m, key_size); if (ret){ HPRE_TST_PRT("failed to fill dh key gen req!\n"); goto sess_release; @@ -1847,9 +2039,9 @@ static void *dh_uadk_async_run(void *arg) free(tag); param_release: - free(req.x_p); - free(req.pv); - free(req.pri); + dh_setup.mm_ops.free(dh_setup.mm_ops.usr, req.pri); + dh_setup.mm_ops.free(dh_setup.mm_ops.usr, req.x_p); + dh_setup.mm_ops.free(dh_setup.mm_ops.usr, req.pv); sess_release: wd_dh_free_sess(h_sess); add_send_complete(); @@ -1861,6 +2053,7 @@ static void *dh_uadk_sync_run(void *arg) { thread_data *pdata = (thread_data *)arg; int key_size = pdata->keybits >> 3; + struct sched_params sc_param = {0}; struct wd_dh_sess_setup dh_setup; struct hpre_dh_param setup; struct wd_dh_req req; @@ -1870,6 +2063,19 @@ static void *dh_uadk_sync_run(void *arg) memset(&dh_setup, 0, sizeof(dh_setup)); memset(&req, 0, sizeof(req)); + sc_param.numa_id = 0; + sc_param.type = 0; + sc_param.mode = 0; + if (hpre_uadk_pool.rsv_pool) + sc_param.dev_id = wd_get_dev_id(hpre_uadk_pool.rsv_pool); + dh_setup.sched_param = (void *)&sc_param; + dh_setup.mm_type = pdata->mm_type; + dh_setup.mm_ops.usr = hpre_uadk_pool.rsv_pool; + dh_setup.mm_ops.alloc = (void *)wd_mem_alloc; + dh_setup.mm_ops.free = (void *)wd_mem_free; + dh_setup.mm_ops.iova_map = (void *)wd_mem_map; + dh_setup.mm_ops.iova_unmap = (void *)wd_mem_unmap; + dh_setup.mm_ops.get_bufsize = (void *)wd_get_bufsize; dh_setup.key_bits = pdata->keybits; if (pdata->optype == WD_DH_PHASE2) dh_setup.is_g2 = true; // G1 is 0; G2 is 1; @@ -1884,7 +2090,7 @@ static void *dh_uadk_sync_run(void *arg) setup.optype = pdata->optype; req.op_type = pdata->optype; - ret = get_dh_opdata_param(h_sess, &req, &setup, key_size); + ret = get_dh_opdata_param(&dh_setup, h_sess, &req, &setup, key_size); if (ret){ HPRE_TST_PRT("failed to fill dh key gen req!\n"); goto param_release; @@ -1903,9 +2109,9 @@ static void *dh_uadk_sync_run(void *arg) } while(true); param_release: - free(req.x_p); - free(req.pv); - free(req.pri); + dh_setup.mm_ops.free(dh_setup.mm_ops.usr, req.pri); + dh_setup.mm_ops.free(dh_setup.mm_ops.usr, req.x_p); + dh_setup.mm_ops.free(dh_setup.mm_ops.usr, req.pv); sess_release: wd_dh_free_sess(h_sess); cal_avg_latency(count); @@ -2247,6 +2453,7 @@ static void *ecc_uadk_sync_run(void *arg) { thread_data *pdata = (thread_data *)arg; int key_size = pdata->keybits >> 3; + struct sched_params sc_param = {0}; u32 cid = ECC_CURVE_SECP256R1; u32 subtype = pdata->subtype; struct wd_ecc_sess_setup sess_setup; @@ -2304,8 +2511,21 @@ static void *ecc_uadk_sync_run(void *arg) } // set def setting; + sc_param.numa_id = 0; + sc_param.type = 0; + sc_param.mode = 0; + if (hpre_uadk_pool.rsv_pool) + sc_param.dev_id = wd_get_dev_id(hpre_uadk_pool.rsv_pool); + sess_setup.sched_param = (void *)&sc_param; sess_setup.hash.cb = hpre_compute_hash; - sess_setup.hash.type = WD_HASH_SHA256; + sess_setup.hash.type = 0; + sess_setup.mm_type = pdata->mm_type; + sess_setup.mm_ops.usr = hpre_uadk_pool.rsv_pool; + sess_setup.mm_ops.alloc = (void *)wd_mem_alloc; + sess_setup.mm_ops.free = (void *)wd_mem_free; + sess_setup.mm_ops.iova_map = (void *)wd_mem_map; + sess_setup.mm_ops.iova_unmap = (void *)wd_mem_unmap; + sess_setup.mm_ops.get_bufsize = (void *)wd_get_bufsize; ret = get_ecc_param_from_sample(&setup, subtype, pdata->keybits); if (ret) @@ -2403,6 +2623,7 @@ static void *ecc_uadk_async_run(void *arg) { thread_data *pdata = (thread_data *)arg; int key_size = pdata->keybits >> 3; + struct sched_params sc_param = {0}; u32 cid = ECC_CURVE_SECP256R1; u32 subtype = pdata->subtype; struct wd_ecc_sess_setup sess_setup; @@ -2464,6 +2685,19 @@ static void *ecc_uadk_async_run(void *arg) // set def setting; sess_setup.hash.cb = hpre_compute_hash; sess_setup.hash.type = WD_HASH_SHA256; + sc_param.numa_id = 0; + sc_param.type = 0; + sc_param.mode = 0; + if (hpre_uadk_pool.rsv_pool) + sc_param.dev_id = wd_get_dev_id(hpre_uadk_pool.rsv_pool); + sess_setup.sched_param = (void *)&sc_param; + sess_setup.mm_type = pdata->mm_type; + sess_setup.mm_ops.usr = hpre_uadk_pool.rsv_pool; + sess_setup.mm_ops.alloc = (void *)wd_mem_alloc; + sess_setup.mm_ops.free = (void *)wd_mem_free; + sess_setup.mm_ops.iova_map = (void *)wd_mem_map; + sess_setup.mm_ops.iova_unmap = (void *)wd_mem_unmap; + sess_setup.mm_ops.get_bufsize = (void *)wd_get_bufsize; ret = get_ecc_param_from_sample(&setup, subtype, pdata->keybits); if (ret) @@ -2630,6 +2864,7 @@ static int hpre_uadk_sync_threads(struct acc_option *options) threads_args[i].optype = threads_option.optype; threads_args[i].td_id = i; threads_args[i].algtype = threads_option.algtype; + threads_args[i].mm_type = threads_option.mm_type; ret = pthread_create(&tdid[i], NULL, uadk_hpre_sync_run, &threads_args[i]); if (ret) { HPRE_TST_PRT("Create sync thread fail!\n"); @@ -2707,6 +2942,7 @@ static int hpre_uadk_async_threads(struct acc_option *options) threads_args[i].optype = threads_option.optype; threads_args[i].td_id = i; threads_args[i].algtype = threads_option.algtype; + threads_args[i].mm_type = threads_option.mm_type; ret = pthread_create(&tdid[i], NULL, uadk_hpre_async_run, &threads_args[i]); if (ret) { HPRE_TST_PRT("Create async thread fail!\n"); @@ -2758,6 +2994,12 @@ int hpre_uadk_benchmark(struct acc_option *options) return ret; } + if (options->mem_type != UADK_AUTO) { + ret = init_uadk_rsv_pool(options); + if (ret) + return ret; + } + get_pid_cpu_time(&ptime); time_start(options->times); if (options->syncmode) @@ -2768,6 +3010,9 @@ int hpre_uadk_benchmark(struct acc_option *options) if (ret) return ret; + if (options->mem_type != UADK_AUTO) + free_uadk_rsv_pool(options); + if (options->inittype == INIT2_TYPE) uninit_hpre_ctx_config2(options->subtype); else diff --git a/uadk_tool/benchmark/hpre_wd_benchmark.c b/uadk_tool/benchmark/hpre_wd_benchmark.c index 60ed3bd..38c45b1 100644 --- a/uadk_tool/benchmark/hpre_wd_benchmark.c +++ b/uadk_tool/benchmark/hpre_wd_benchmark.c @@ -1613,11 +1613,22 @@ static int get_ecc_param_from_sample(struct hpre_ecc_setup *setup, setup->sign_size = sizeof(sm2_sign_data); } else { - setup->priv_key = ecdh_da_secp256k1; + /* + * x25519 and ecdh-256 can share same 32-bytes private key of + * ecdh_da_secp256k1, while x448 should use 56-byte private key + * to get accurate performance. + */ + if (subtype == X448_TYPE) + setup->priv_key = x448_da; + else + setup->priv_key = ecdh_da_secp256k1; setup->except_pub_key = ecdh_except_b_pubkey_secp256k1; setup->pub_key = ecdh_cp_pubkey_secp256k1; setup->share_key = ecdh_cp_sharekey_secp256k1; - setup->priv_key_size = sizeof(ecdh_da_secp256k1); + if (subtype == X448_TYPE) + setup->priv_key_size = sizeof(x448_da); + else + setup->priv_key_size = sizeof(ecdh_da_secp256k1); setup->except_pub_key_size = sizeof(ecdh_except_b_pubkey_secp256k1); setup->pub_key_size = sizeof(ecdh_cp_pubkey_secp256k1); setup->share_key_size = sizeof(ecdh_cp_sharekey_secp256k1); -- 2.33.0

From: Chenghai Huang <huangchenghai2@huawei.com> When the hardware configures the lit_length register, the stream mode LZ77_ZSTD algorithm needs to clear the lit_length in the context of each block input. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- drv/hisi_comp.c | 5 +++++ v1/drv/hisi_zip_udrv.c | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index ab2b5e7..4dc0c6a 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -89,6 +89,7 @@ #define PRICE_MIN_OUT_SIZE 4096 #define ZSTD_LIT_RESV_SIZE 16 #define REPCODE_SIZE 12 +#define SEQ_LIT_LEN_SIZE 4 #define BUF_TYPE 2 @@ -829,6 +830,10 @@ static int fill_buf_lz77_zstd(handle_t h_qp, struct hisi_zip_sqe *sqe, else memcpy(msg->ctx_buf + CTX_REPCODE2_OFFSET, msg->ctx_buf + CTX_REPCODE1_OFFSET, REPCODE_SIZE); + + /* The literal length info of each bd needs to be cleared. */ + memset(ctx_buf + CTX_HW_REPCODE_OFFSET + REPCODE_SIZE, 0, + SEQ_LIT_LEN_SIZE); } } diff --git a/v1/drv/hisi_zip_udrv.c b/v1/drv/hisi_zip_udrv.c index 8908e27..3045e48 100644 --- a/v1/drv/hisi_zip_udrv.c +++ b/v1/drv/hisi_zip_udrv.c @@ -58,6 +58,7 @@ #define ZSTD_LIT_RSV_SIZE 16 #define ZSTD_FREQ_DATA_SIZE 784 #define REPCODE_SIZE 12 +#define SEQ_LIT_LEN_SIZE 4 #define OVERFLOW_DATA_SIZE 8 /* Error status 0xe indicates that dest_avail_out insufficient */ @@ -694,6 +695,10 @@ static void fill_zip_sqe_hw_info_lz77_zstd(void *ssqe, struct wcrypto_comp_msg * else memcpy(msg->ctx_buf + CTX_REPCODE2_OFFSET, msg->ctx_buf + CTX_REPCODE1_OFFSET, REPCODE_SIZE); + + /* The literal length info of each bd needs to be cleared. */ + memset(msg->ctx_buf + CTX_HW_REPCODE_OFFSET + CTX_BUFFER_OFFSET + + REPCODE_SIZE, 0, SEQ_LIT_LEN_SIZE); } } -- 2.33.0

From: Chenghai Huang <huangchenghai2@huawei.com> Add a condition to check if the tail packet is not fully processed. When the bfinal end flag of the tail packet is in the second-to-last segment of data, it may cause the driver to incorrectly assume that the current data is the tail packet. In fact, there is still one more segment of data to be transmitted. Therefore, it is necessary to add a check for the data buffer status to notify the user to continue sending the packet. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- drv/hisi_comp.c | 3 ++- v1/drv/hisi_zip_udrv.c | 4 ++-- v1/drv/hisi_zip_udrv.h | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index bcc4f17..3607ea2 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -57,6 +57,7 @@ #define HZ_CTX_ST_MASK 0x000f #define HZ_CTX_BFINAL_MASK 0x80 +#define HZ_CTX_STORE_MASK 0x7ffff #define HZ_LSTBLK_MASK 0x0100 #define HZ_STATUS_MASK 0xff #define HZ_REQ_TYPE_MASK 0xff @@ -1595,7 +1596,7 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, /* last block no space when decomping, need resend null size req */ if (ctx_st == HZ_DECOMPING_NO_SPACE && recv_msg->req.src_len == recv_msg->in_cons && - (sqe->ctx_dw0 & HZ_CTX_BFINAL_MASK)) + (sqe->ctx_dw0 & HZ_CTX_BFINAL_MASK) && (sqe->ctx_dw1 & HZ_CTX_STORE_MASK)) recv_msg->req.status = WD_EAGAIN; /* diff --git a/v1/drv/hisi_zip_udrv.c b/v1/drv/hisi_zip_udrv.c index 4e5ed80..44e1545 100644 --- a/v1/drv/hisi_zip_udrv.c +++ b/v1/drv/hisi_zip_udrv.c @@ -440,7 +440,7 @@ int qm_parse_zip_sqe(void *hw_msg, const struct qm_queue_info *info, qm_parse_zip_sqe_set_status(recv_msg, status, lstblk, ctx_st); if (ctx_st == HW_DECOMPING_NO_SPACE && recv_msg->in_size == recv_msg->in_cons && - ctx_bfinal) + ctx_bfinal && (sqe->ctx_dw1 & HZ_CTX_STORE_MASK)) recv_msg->status = WCRYPTO_DECOMP_END_NOSPACE; return 1; @@ -907,7 +907,7 @@ int qm_parse_zip_sqe_v3(void *hw_msg, const struct qm_queue_info *info, qm_parse_zip_sqe_set_status(recv_msg, status, lstblk, ctx_st); if (ctx_st == HW_DECOMPING_NO_SPACE && recv_msg->in_size == recv_msg->in_cons && - ctx_bfinal) + ctx_bfinal && (sqe->ctx_dw1 & HZ_CTX_STORE_MASK)) recv_msg->status = WCRYPTO_DECOMP_END_NOSPACE; /* diff --git a/v1/drv/hisi_zip_udrv.h b/v1/drv/hisi_zip_udrv.h index 28a9c0f..1037f43 100644 --- a/v1/drv/hisi_zip_udrv.h +++ b/v1/drv/hisi_zip_udrv.h @@ -121,6 +121,7 @@ struct hisi_zip_sqe_v3 { #define HZ_BLK_SIZE_SHIFT 16 #define HZ_CTX_ST_MASK 0x000f #define HZ_CTX_BFINAL_MASK 0x80 +#define HZ_CTX_STORE_MASK 0x7ffff #define HZ_LSTBLK_MASK 0x0100 #define HZ_STATUS_MASK 0xff #define HZ_REQ_TYPE_MASK 0xff -- 2.33.0

From: Chenghai Huang <huangchenghai2@huawei.com> When the requested memory pool size is greater than 1M, a certain amount of redundancy is reserved to avoid memory pool allocation failure. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- v1/wd_bmm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v1/wd_bmm.c b/v1/wd_bmm.c index c58484f..dd20ff7 100644 --- a/v1/wd_bmm.c +++ b/v1/wd_bmm.c @@ -122,7 +122,7 @@ static int wd_pool_pre_layout(struct wd_queue *q, * ensure that the allocated memory is an integer multiple of 1M. */ if (!sp->br.alloc && !qinfo->iommu_type) - p->act_mem_sz = (p->act_mem_sz + BLK_BALANCE_SZ - 1) & ~(BLK_BALANCE_SZ - 1); + p->act_mem_sz = ((p->act_mem_sz + BLK_BALANCE_SZ - 1) & ~(BLK_BALANCE_SZ - 1)) << 1; return WD_SUCCESS; } -- 2.33.0

From: Chenghai Huang <huangchenghai2@huawei.com> We could use uadk.cnf to specify which UADK driver shared libraries (.so) should be loaded. Usage Guidelines: 1. Place this file (uadk.cnf) in the same directory as your driver .so files 2. List one driver library per line (e.g., "libhisi_zip.so") Example Configuration: libhisi_zip.so libhisi_sec.so Note: UADK will not load any .so files that are not explicitly listed here. If a non-existent so is specified, it will be skipped, and info user by log. If uadk.cnf is not present, load the libraries (.so) by default methods. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- uadk.cnf | 26 ++++++ wd_util.c | 264 ++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 231 insertions(+), 59 deletions(-) create mode 100644 uadk.cnf diff --git a/uadk.cnf b/uadk.cnf new file mode 100644 index 0000000..23b8f9d --- /dev/null +++ b/uadk.cnf @@ -0,0 +1,26 @@ +# UADK Driver Configuration File +# ============================= +# +# This file specifies which UADK driver shared libraries (.so) should be loaded. +# +# Usage Guidelines: +# 1. Place this file (uadk.cnf) in the same directory as your driver .so files +# 2. List one driver library per line (e.g., "libhisi_zip.so") +# +# Example Configuration: +# libhisi_zip.so +# libhisi_sec.so +# # libhisi_hpre.so (commented out - will not load) +# +# Note: +# UADK will not load any .so files that are not explicitly listed here. +# If a non-existent so is specified, it will be skipped, and info user by log. +# If uadk.cnf is not present, load the libraries (.so) by default methods. + +libhisi_zip.so +libhisi_sec.so +libhisi_hpre.so +libisa_ce.so +libisa_sve.so +libhisi_dae.so +libhisi_udma.so \ No newline at end of file diff --git a/wd_util.c b/wd_util.c index d0d62a1..daa7309 100644 --- a/wd_util.c +++ b/wd_util.c @@ -35,6 +35,7 @@ #define WD_SOFT_ASYNC_CTX 1 #define WD_DRV_LIB_DIR "uadk" +#define WD_DRV_CONF_FILE "uadk.cnf" #define WD_PATH_DIR_NUM 2 @@ -2279,11 +2280,64 @@ static void dladdr_empty(void) { } +static int line_check_valid(char *line) +{ + line[strcspn(line, "\n")] = 0; + if (line[0] == '\0' || line[0] == '#') + return 0; + + if (!strstr(line, ".so")) + return 0; + + return 1; +} + +static int check_uadk_config_file(const char *wd_dir, const char *lib_file) +{ + char *path_buf, *uadk_cnf_path, *line; + int ret = -WD_EINVAL; + FILE *fp; + + path_buf = calloc(WD_PATH_DIR_NUM, PATH_MAX); + if (!path_buf) { + WD_ERR("fail to alloc memery for path_buf.\n"); + return -WD_ENOMEM; + } + + uadk_cnf_path = path_buf; + line = path_buf + PATH_MAX; + + snprintf(uadk_cnf_path, PATH_MAX, "%s/%s/%s", wd_dir, WD_DRV_LIB_DIR, + WD_DRV_CONF_FILE); + fp = fopen(uadk_cnf_path, "r"); + if (!fp) { + ret = 0; + goto free_buf; + } + + while (fgets(line, PATH_MAX, fp)) { + if (!line_check_valid(line)) + continue; + + if (strstr(line, lib_file)) { + ret = 0; + goto close_fp; + } + } + +close_fp: + fclose(fp); +free_buf: + free(path_buf); + return ret; +} + int wd_get_lib_file_path(const char *lib_file, char *lib_path, bool is_dir) { char *path_buf, *path, *file_path; Dl_info file_info; int len, rc, i; + int ret = 0; /* Get libwd.so file's system path */ rc = dladdr(dladdr_empty, &file_info); @@ -2292,7 +2346,7 @@ int wd_get_lib_file_path(const char *lib_file, char *lib_path, bool is_dir) return -WD_EINVAL; } - path_buf = calloc(WD_PATH_DIR_NUM, sizeof(char) * PATH_MAX); + path_buf = calloc(WD_PATH_DIR_NUM, PATH_MAX); if (!path_buf) { WD_ERR("fail to calloc path_buf.\n"); return -WD_ENOMEM; @@ -2312,28 +2366,32 @@ int wd_get_lib_file_path(const char *lib_file, char *lib_path, bool is_dir) if (is_dir) { len = snprintf(lib_path, PATH_MAX, "%s/%s", file_path, WD_DRV_LIB_DIR); - if (len >= PATH_MAX) - goto free_path; } else { - len = snprintf(lib_path, PATH_MAX, "%s/%s/%s", file_path, WD_DRV_LIB_DIR, lib_file); - if (len >= PATH_MAX) + /* Confirm whether the corresponding file exists in uadk.cnf */ + ret = check_uadk_config_file(file_path, lib_file); + if (ret) goto free_path; + + len = snprintf(lib_path, PATH_MAX, "%s/%s/%s", + file_path, WD_DRV_LIB_DIR, lib_file); } - if (realpath(lib_path, path) == NULL) { - WD_ERR("invalid: %s: no such file or directory!\n", path); + if (len >= PATH_MAX) { + ret = -WD_EINVAL; goto free_path; } - free(path_buf); - return 0; + if (!realpath(lib_path, path)) { + WD_ERR("invalid: %s: no such file or directory!\n", path); + ret = -WD_EINVAL; + } free_path: free(path_buf); - return -WD_EINVAL; + return ret; } -/** +/* * There are many other .so files in this file directory (/root/lib/), * and it is necessary to screen out valid uadk driver files * through this function. @@ -2361,36 +2419,93 @@ static int file_check_valid(const char *lib_file) return 0; } -void *wd_dlopen_drv(const char *cust_lib_dir) +static void create_lib_to_list(const char *lib_path, struct drv_lib_list **head) { typedef int (*alg_ops)(struct wd_alg_driver *drv); - struct drv_lib_list *node, *head = NULL; - char lib_dir_path[PATH_MAX] = {0}; - char lib_path[PATH_MAX] = {0}; - struct dirent *lib_dir; - alg_ops dl_func = NULL; - DIR *wd_dir; + struct drv_lib_list *node; + alg_ops dl_func; + + node = calloc(1, sizeof(*node)); + if (!node) + return; + + node->dlhandle = dlopen(lib_path, RTLD_NODELETE | RTLD_NOW); + if (!node->dlhandle) { + WD_ERR("failed to open lib file: %s, skipped\n", lib_path); + free(node); + return; + } + + dl_func = dlsym(node->dlhandle, "wd_alg_driver_register"); + if (!dl_func) { + WD_ERR("dlsym failed for %s: %s\n", lib_path, dlerror()); + dlclose(node->dlhandle); + free(node); + return; + } + + if (!*head) { + *head = node; + return; + } + add_lib_to_list(*head, node); +} + +static struct drv_lib_list *load_libraries_from_config(const char *config_path, + const char *lib_dir_path) +{ + char *path_buf, *lib_path, *line; + struct drv_lib_list *head = NULL; + FILE *config_file; int ret; - if (!cust_lib_dir) { - ret = wd_get_lib_file_path(NULL, lib_dir_path, true); - if (ret) - return NULL; - } else { - if (realpath(cust_lib_dir, lib_path) == NULL) { - WD_ERR("invalid: %s: no such file or directory!\n", lib_path); - return NULL; - } - strncpy(lib_dir_path, cust_lib_dir, PATH_MAX - 1); - lib_dir_path[PATH_MAX - 1] = '\0'; + path_buf = calloc(WD_PATH_DIR_NUM, PATH_MAX); + if (!path_buf) { + WD_ERR("fail to alloc memery for path_buf.\n"); + return NULL; } + lib_path = path_buf; + line = path_buf + PATH_MAX; - wd_dir = opendir(lib_dir_path); - if (!wd_dir) { - WD_ERR("UADK driver lib dir: %s not exist!\n", lib_dir_path); + config_file = fopen(config_path, "r"); + if (!config_file) { + WD_ERR("Failed to open config file: %s\n", config_path); + free(path_buf); + return NULL; + } + + /* Read config file line by line */ + while (fgets(line, PATH_MAX, config_file)) { + if (!line_check_valid(line)) + continue; + + ret = snprintf(lib_path, PATH_MAX, "%s/%s", lib_dir_path, line); + if (ret < 0) + break; + + create_lib_to_list(lib_path, &head); + } + + free(path_buf); + fclose(config_file); + return head; +} + +static struct drv_lib_list *load_all_libraries(DIR *wd_dir, const char *lib_dir_path) +{ + struct drv_lib_list *head = NULL; + struct dirent *lib_dir; + char *lib_path; + int ret; + + lib_path = calloc(1, PATH_MAX); + if (!lib_path) { + WD_ERR("fail to alloc memery for lib_path.\n"); return NULL; } + rewinddir(wd_dir); /* Ensure we're at the start of the directory */ + while ((lib_dir = readdir(wd_dir)) != NULL) { if (!strncmp(lib_dir->d_name, ".", LINUX_CRTDIR_SIZE) || !strncmp(lib_dir->d_name, "..", LINUX_PRTDIR_SIZE)) @@ -2400,43 +2515,74 @@ void *wd_dlopen_drv(const char *cust_lib_dir) if (ret) continue; - node = calloc(1, sizeof(*node)); - if (!node) - goto free_list; - ret = snprintf(lib_path, PATH_MAX, "%s/%s", lib_dir_path, lib_dir->d_name); if (ret < 0) - goto free_node; + break; - node->dlhandle = dlopen(lib_path, RTLD_NODELETE | RTLD_NOW); - if (!node->dlhandle) { - free(node); - /* there are many other files need to skip */ - continue; - } + create_lib_to_list(lib_path, &head); + } - dl_func = dlsym(node->dlhandle, "wd_alg_driver_register"); - if (dl_func == NULL) { - dlclose(node->dlhandle); - free(node); - continue; + free(lib_path); + return head; +} + +void *wd_dlopen_drv(const char *cust_lib_dir) +{ + char *path_buf, *lib_dir_path, *config_path, *lib_path; + struct drv_lib_list *head = NULL; + int ret, len; + DIR *wd_dir; + + path_buf = calloc(WD_PATH_DIR_NUM + 1, PATH_MAX); + if (!path_buf) { + WD_ERR("fail to alloc memory for path buffers.\n"); + return NULL; + } + + lib_dir_path = path_buf; + config_path = path_buf + PATH_MAX; + lib_path = config_path + PATH_MAX; + + if (!cust_lib_dir) { + ret = wd_get_lib_file_path(NULL, lib_dir_path, true); + if (ret) + goto free_path; + } else { + if (!realpath(cust_lib_dir, lib_path)) { + WD_ERR("invalid: %s: no such file or directory!\n", lib_path); + goto free_path; } - if (!head) - head = node; - else - add_lib_to_list(head, node); + len = snprintf(lib_dir_path, PATH_MAX, "%s", cust_lib_dir); + if (len < 0 || len >= PATH_MAX) + goto free_path; + + lib_dir_path[PATH_MAX - 1] = '\0'; } - closedir(wd_dir); - return (void *)head; + wd_dir = opendir(lib_dir_path); + if (!wd_dir) { + WD_ERR("UADK driver lib dir: %s not exist!\n", lib_dir_path); + goto free_path; + } -free_node: - free(node); -free_list: + len = snprintf(config_path, PATH_MAX, "%s/%s", lib_dir_path, WD_DRV_CONF_FILE); + if (len < 0 || len >= PATH_MAX) + goto close_dir; + + ret = access(config_path, F_OK); + if (!ret) + /* Load specified libraries from config file */ + head = load_libraries_from_config(config_path, lib_dir_path); + else + /* Load all valid .so files */ + head = load_all_libraries(wd_dir, lib_dir_path); + +close_dir: closedir(wd_dir); - wd_dlclose_drv(head); - return NULL; +free_path: + free(path_buf); + return (void *)head; } struct wd_alg_driver *wd_alg_drv_bind(int task_type, const char *alg_name) -- 2.33.0

From: Chenghai Huang <huangchenghai2@huawei.com> The current UADK v2 supports the nosav feature, which allows business operations to be executed when the driver is loaded with uacce_mode=2. However, it requires using the reserved memory allocated through uacce. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- drv/hisi_comp.c | 260 +++++++++++++++++++++++++------------- include/drv/wd_comp_drv.h | 13 ++ include/wd_comp.h | 2 + wd_comp.c | 43 ++++++- 4 files changed, 228 insertions(+), 90 deletions(-) diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index 7d6f94d..9d2a9af 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -77,6 +77,7 @@ #define max_in_data_size(outl) ((__u32)(((__u64)(outl) << 3) / 9) & 0xfffffffc) #define HZ_MAX_SIZE (8 * 1024 * 1024) +#define HW_CTX_SIZE 0x10000 #define RSV_OFFSET 64 #define CTX_DW1_OFFSET 4 @@ -244,15 +245,6 @@ struct hisi_zip_ctx { struct wd_ctx_config_internal config; }; -struct comp_sgl { - void *in; - void *out; - void *out_seq; - struct wd_datalist *list_src; - struct wd_datalist *list_dst; - struct wd_datalist *seq_start; -}; - static void dump_zip_msg(struct wd_comp_msg *msg) { WD_ERR("dump zip message after a task error occurs.\n"); @@ -376,11 +368,11 @@ static int check_enable_store_buf(struct wd_comp_msg *msg, __u32 out_size, int h return 0; } -static int get_sgl_from_pool(handle_t h_qp, struct comp_sgl *c_sgl) +static int get_sgl_from_pool(handle_t h_qp, struct comp_sgl *c_sgl, struct wd_mm_ops *mm_ops) { handle_t h_sgl_pool; - h_sgl_pool = hisi_qm_get_sglpool(h_qp, NULL); + h_sgl_pool = hisi_qm_get_sglpool(h_qp, mm_ops); if (unlikely(!h_sgl_pool)) { WD_ERR("failed to get sglpool!\n"); return -WD_EINVAL; @@ -415,6 +407,23 @@ err_free_sgl_in: return -WD_ENOMEM; } +static void free_hw_sgl(handle_t h_qp, struct comp_sgl *c_sgl, struct wd_mm_ops *mm_ops) +{ + handle_t h_sgl_pool; + + h_sgl_pool = hisi_qm_get_sglpool(h_qp, mm_ops); + if (unlikely(!h_sgl_pool)) { + WD_ERR("failed to get sglpool!\n"); + return; + } + + hisi_qm_put_hw_sgl(h_sgl_pool, c_sgl->in); + hisi_qm_put_hw_sgl(h_sgl_pool, c_sgl->out); + + if (c_sgl->seq_start) + hisi_qm_put_hw_sgl(h_sgl_pool, c_sgl->out_seq); +} + static void fill_comp_buf_size(struct hisi_zip_sqe *sqe, __u32 in_size, __u32 out_size) { @@ -422,8 +431,124 @@ static void fill_comp_buf_size(struct hisi_zip_sqe *sqe, __u32 in_size, sqe->dest_avail_out = out_size; } -static void fill_buf_addr(struct hisi_zip_sqe *sqe, struct hisi_comp_sqe_addr *addr) +static int zip_mem_map(struct wd_mm_ops *mm_ops, struct hisi_zip_sqe *sqe, + struct hisi_comp_sqe_addr *addr) +{ + void *phy_dst, *phy_ctx, *mempool; + void *phy_lit = NULL; + void *phy_src = NULL; + + mempool = mm_ops->usr; + /* When the src len is 0, map is not required */ + if (sqe->input_data_length) { + phy_src = mm_ops->iova_map(mempool, addr->src_addr, sqe->input_data_length); + if (!phy_src) { + WD_ERR("get zip src dma address fail!\n"); + return -WD_ENOMEM; + } + + sqe->source_addr_l = lower_32_bits(phy_src); + sqe->source_addr_h = upper_32_bits(phy_src); + } + + if (addr->lit_addr) { + phy_lit = mm_ops->iova_map(mempool, addr->lit_addr, sqe->dw13); + if (!phy_lit) { + WD_ERR("get zip lits dma address fail!\n"); + goto unmap_src; + } + + sqe->literals_addr_l = lower_32_bits(phy_lit); + sqe->literals_addr_h = upper_32_bits(phy_lit); + } + + phy_dst = mm_ops->iova_map(mempool, addr->dst_addr, sqe->dest_avail_out); + if (!phy_dst) { + WD_ERR("get zip dst dma address fail!\n"); + goto unmap_lit; + } + + sqe->dest_addr_l = lower_32_bits(phy_dst); + sqe->dest_addr_h = upper_32_bits(phy_dst); + + /* The source addr needs to be filled with a valid address when length is 0 */ + if (!sqe->input_data_length) { + sqe->source_addr_l = sqe->dest_addr_l; + sqe->source_addr_h = sqe->dest_addr_h; + } + + if (addr->ctx_addr) { + phy_ctx = mm_ops->iova_map(mempool, addr->ctx_addr, HW_CTX_SIZE); + if (!phy_ctx) { + WD_ERR("get zip ctx dma address fail!\n"); + goto unmap_dst; + } + + sqe->stream_ctx_addr_l = lower_32_bits(phy_ctx); + sqe->stream_ctx_addr_h = upper_32_bits(phy_ctx); + } + + return 0; + +unmap_dst: + mm_ops->iova_unmap(mempool, addr->dst_addr, phy_dst, sqe->dest_avail_out); +unmap_lit: + if (addr->lit_addr) + mm_ops->iova_unmap(mempool, addr->lit_addr, phy_lit, sqe->dw13); +unmap_src: + if (sqe->input_data_length) + mm_ops->iova_unmap(mempool, addr->src_addr, phy_src, sqe->input_data_length); + return -WD_ENOMEM; +} + +static void zip_mem_unmap(struct wd_comp_msg *msg, struct hisi_zip_sqe *sqe) +{ + void *dma_addr, *src_addr, *seq_addr, *lit_addr; + struct wd_mm_ops *mm_ops = msg->mm_ops; + struct wd_comp_req *req = &msg->req; + void *mempool = mm_ops->usr; + + if (msg->data_fmt == WD_SGL_BUF) + src_addr = msg->c_sgl.in; + else + src_addr = req->src; + + if (sqe->input_data_length) { + dma_addr = VA_ADDR(sqe->source_addr_h, sqe->source_addr_l); + mm_ops->iova_unmap(mempool, src_addr, dma_addr, sqe->input_data_length); + } + + if (msg->alg_type == WD_LZ77_ZSTD || msg->alg_type == WD_LZ77_ONLY) { + if (msg->data_fmt == WD_SGL_BUF) { + seq_addr = msg->c_sgl.out_seq; + lit_addr = msg->c_sgl.out; + } else { + seq_addr = req->dst + sqe->dw13; + lit_addr = req->dst; + } + + dma_addr = VA_ADDR(sqe->literals_addr_h, sqe->literals_addr_l); + mm_ops->iova_unmap(mempool, lit_addr, dma_addr, sqe->dw13); + dma_addr = VA_ADDR(sqe->dest_addr_h, sqe->dest_addr_l); + mm_ops->iova_unmap(mempool, seq_addr, dma_addr, sqe->dest_avail_out); + } else { + dma_addr = VA_ADDR(sqe->dest_addr_h, sqe->dest_addr_l); + mm_ops->iova_unmap(mempool, req->dst, dma_addr, sqe->dest_avail_out); + } + + if (msg->stream_mode == WD_COMP_STATEFUL) { + dma_addr = VA_ADDR(sqe->stream_ctx_addr_h, sqe->stream_ctx_addr_l); + mm_ops->iova_unmap(mempool, msg->ctx_buf, dma_addr, HW_CTX_SIZE); + } +} + +static int fill_buf_addr(struct hisi_zip_sqe *sqe, struct hisi_comp_sqe_addr *addr, + struct wd_mm_ops *mm_ops) { + /* No-SVA mode and Memory is USER mode or PROXY mode */ + if (!mm_ops->sva_mode) + return zip_mem_map(mm_ops, sqe, addr); + sqe->source_addr_l = lower_32_bits(addr->src_addr); sqe->source_addr_h = upper_32_bits(addr->src_addr); sqe->dest_addr_l = lower_32_bits(addr->dst_addr); @@ -438,6 +563,8 @@ static void fill_buf_addr(struct hisi_zip_sqe *sqe, struct hisi_comp_sqe_addr *a sqe->stream_ctx_addr_l = lower_32_bits(addr->ctx_addr); sqe->stream_ctx_addr_h = upper_32_bits(addr->ctx_addr); } + + return 0; } static int fill_buf_deflate_generic(struct hisi_zip_sqe *sqe, @@ -497,15 +624,13 @@ static int fill_buf_deflate_generic(struct hisi_zip_sqe *sqe, fill_comp_buf_size(sqe, in_size, out_size); - if (msg->ctx_buf) + if (msg->stream_mode == WD_COMP_STATEFUL) ctx_buf = msg->ctx_buf + RSV_OFFSET; addr.src_addr = src; addr.dst_addr = dst; addr.ctx_addr = ctx_buf; - fill_buf_addr(sqe, &addr); - - return 0; + return fill_buf_addr(sqe, &addr, msg->mm_ops); } static int fill_buf_deflate(handle_t h_qp, struct hisi_zip_sqe *sqe, @@ -569,9 +694,7 @@ static int fill_buf_lz4(handle_t h_qp, struct hisi_zip_sqe *sqe, addr.src_addr = src; addr.dst_addr = dst; - fill_buf_addr(sqe, &addr); - - return 0; + return fill_buf_addr(sqe, &addr, msg->mm_ops); } static void fill_buf_type_sgl(struct hisi_zip_sqe *sqe) @@ -584,25 +707,28 @@ static void fill_buf_type_sgl(struct hisi_zip_sqe *sqe) } static int fill_buf_addr_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, - struct wd_datalist *list_src, struct wd_datalist *list_dst) + struct comp_sgl *c_sgl, struct wd_mm_ops *mm_ops) { struct hisi_comp_sqe_addr addr = {0}; - struct comp_sgl c_sgl; int ret; - c_sgl.list_src = list_src; - c_sgl.list_dst = list_dst; - c_sgl.seq_start = NULL; - - ret = get_sgl_from_pool(h_qp, &c_sgl); + ret = get_sgl_from_pool(h_qp, c_sgl, mm_ops); if (unlikely(ret)) return ret; - addr.src_addr = c_sgl.in; - addr.dst_addr = c_sgl.out; - fill_buf_addr(sqe, &addr); + addr.src_addr = c_sgl->in; + if (c_sgl->seq_start) { + addr.lit_addr = c_sgl->out; + addr.dst_addr = c_sgl->out_seq; + } else { + addr.dst_addr = c_sgl->out; + } + + ret = fill_buf_addr(sqe, &addr, mm_ops); + if (unlikely(ret)) + free_hw_sgl(h_qp, c_sgl, mm_ops); - return 0; + return ret; } static void fill_buf_sgl_skip(struct hisi_zip_sqe *sqe, __u32 src_skip, @@ -624,7 +750,6 @@ static int fill_buf_deflate_sgl_generic(handle_t h_qp, struct hisi_zip_sqe *sqe, int head_size) { struct wd_comp_req *req = &msg->req; - struct wd_datalist *list_src = req->list_src; struct wd_datalist *list_dst = req->list_dst; __u32 out_size = msg->avail_out; __u32 in_size = req->src_len; @@ -654,10 +779,6 @@ static int fill_buf_deflate_sgl_generic(handle_t h_qp, struct hisi_zip_sqe *sqe, fill_buf_type_sgl(sqe); - ret = fill_buf_addr_sgl(h_qp, sqe, list_src, list_dst); - if (unlikely(ret)) - return ret; - if (head != NULL && msg->req.op_type == WD_DIR_COMPRESS) { memcpy(req->list_dst->data, head, head_size); dst_skip = head_size; @@ -681,7 +802,10 @@ static int fill_buf_deflate_sgl_generic(handle_t h_qp, struct hisi_zip_sqe *sqe, fill_comp_buf_size(sqe, in_size, out_size); - return 0; + msg->c_sgl.list_src = req->list_src; + msg->c_sgl.list_dst = list_dst; + msg->c_sgl.seq_start = NULL; + return fill_buf_addr_sgl(h_qp, sqe, &msg->c_sgl, msg->mm_ops); } static int fill_buf_deflate_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, @@ -826,7 +950,7 @@ static int fill_buf_lz77_zstd(handle_t h_qp, struct hisi_zip_sqe *sqe, if (unlikely(seq_avail_out > HZ_MAX_SIZE)) seq_avail_out = HZ_MAX_SIZE; - if (msg->ctx_buf) { + if (msg->stream_mode == WD_COMP_STATEFUL) { ctx_buf = msg->ctx_buf + RSV_OFFSET; if (msg->alg_type == WD_LZ77_ZSTD) { if (data->blk_type != COMP_BLK) @@ -851,9 +975,7 @@ static int fill_buf_lz77_zstd(handle_t h_qp, struct hisi_zip_sqe *sqe, addr.dst_addr = req->dst + lits_size; addr.lit_addr = req->dst; addr.ctx_addr = ctx_buf; - fill_buf_addr(sqe, &addr); - - return 0; + return fill_buf_addr(sqe, &addr, msg->mm_ops); } static struct wd_datalist *get_seq_start_list(struct wd_comp_req *req) @@ -977,7 +1099,6 @@ static int fill_buf_lz77_zstd_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, __u32 in_size = msg->req.src_len; __u32 out_size = msg->avail_out; struct wd_datalist *seq_start; - struct comp_sgl c_sgl; __u32 lits_size; int ret; @@ -1003,27 +1124,15 @@ static int fill_buf_lz77_zstd_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, fill_buf_size_lz77_zstd(sqe, in_size, lits_size, out_size - lits_size); - c_sgl.list_src = req->list_src; - c_sgl.list_dst = req->list_dst; - c_sgl.seq_start = seq_start; - - ret = get_sgl_from_pool(h_qp, &c_sgl); - if (unlikely(ret)) - return ret; - - addr.src_addr = c_sgl.in; - addr.dst_addr = c_sgl.out_seq; - addr.lit_addr = c_sgl.out; - fill_buf_addr(sqe, &addr); - - return 0; + msg->c_sgl.list_src = req->list_src; + msg->c_sgl.list_dst = req->list_dst; + msg->c_sgl.seq_start = seq_start; + return fill_buf_addr_sgl(h_qp, sqe, &msg->c_sgl, msg->mm_ops); } static int fill_buf_lz4_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, struct wd_comp_msg *msg) { - struct wd_datalist *list_src = msg->req.list_src; - struct wd_datalist *list_dst = msg->req.list_dst; int ret; ret = check_lz4_msg(msg, WD_SGL_BUF); @@ -1034,7 +1143,10 @@ static int fill_buf_lz4_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, fill_comp_buf_size(sqe, msg->req.src_len, msg->avail_out); - return fill_buf_addr_sgl(h_qp, sqe, list_src, list_dst); + msg->c_sgl.list_src = msg->req.list_src; + msg->c_sgl.list_dst = msg->req.list_dst; + msg->c_sgl.seq_start = NULL; + return fill_buf_addr_sgl(h_qp, sqe, &msg->c_sgl, msg->mm_ops); } static void fill_sqe_type_v1(struct hisi_zip_sqe *sqe) @@ -1419,31 +1531,6 @@ static int fill_zip_comp_sqe(struct hisi_qp *qp, struct wd_comp_msg *msg, return 0; } -static void free_hw_sgl(handle_t h_qp, struct hisi_zip_sqe *sqe, - enum wd_comp_alg_type alg_type) -{ - void *hw_sgl_in, *hw_sgl_out; - handle_t h_sgl_pool; - - h_sgl_pool = hisi_qm_get_sglpool(h_qp, NULL); - if (unlikely(!h_sgl_pool)) { - WD_ERR("failed to get sglpool to free hw sgl!\n"); - return; - } - - hw_sgl_in = VA_ADDR(sqe->source_addr_h, sqe->source_addr_l); - hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_in); - - hw_sgl_out = VA_ADDR(sqe->dest_addr_h, sqe->dest_addr_l); - hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_out); - - if (alg_type == WD_LZ77_ZSTD || alg_type == WD_LZ77_ONLY) { - hw_sgl_out = VA_ADDR(sqe->literals_addr_h, - sqe->literals_addr_l); - hisi_qm_put_hw_sgl(h_sgl_pool, hw_sgl_out); - } -} - static int hisi_zip_comp_send(struct wd_alg_driver *drv, handle_t ctx, void *comp_msg) { struct hisi_qp *qp = wd_ctx_get_priv(ctx); @@ -1467,7 +1554,7 @@ static int hisi_zip_comp_send(struct wd_alg_driver *drv, handle_t ctx, void *com ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (unlikely(ret < 0)) { if (msg->req.data_fmt == WD_SGL_BUF) - free_hw_sgl(h_qp, &sqe, msg->alg_type); + free_hw_sgl(h_qp, &msg->c_sgl, msg->mm_ops); if (ret != -WD_EBUSY) WD_ERR("failed to send to hardware, ret = %d!\n", ret); @@ -1619,8 +1706,11 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, recv_msg->checksum = sqe->dw31; recv_msg->alg_type = alg_type; + if (recv_msg->mm_ops && !recv_msg->mm_ops->sva_mode) + zip_mem_unmap(recv_msg, sqe); + if (buf_type == WD_SGL_BUF) - free_hw_sgl((handle_t)qp, sqe, alg_type); + free_hw_sgl((handle_t)qp, &recv_msg->c_sgl, recv_msg->mm_ops); if (unlikely(recv_msg->req.status == WD_IN_EPARA)) dump_zip_msg(recv_msg); diff --git a/include/drv/wd_comp_drv.h b/include/drv/wd_comp_drv.h index 213cf2d..95f7fb7 100644 --- a/include/drv/wd_comp_drv.h +++ b/include/drv/wd_comp_drv.h @@ -24,9 +24,22 @@ enum wd_comp_state { WD_COMP_STATELESS, }; +struct comp_sgl { + void *in; + void *out; + void *out_seq; + struct wd_datalist *list_src; + struct wd_datalist *list_dst; + struct wd_datalist *seq_start; +}; + /* fixme wd_comp_msg */ struct wd_comp_msg { struct wd_comp_req req; + struct wd_mm_ops *mm_ops; + enum wd_mem_type mm_type; + /* Store sgl addr for nosva */ + struct comp_sgl c_sgl; /* Denoted HW ctx cache, for stream mode */ void *ctx_buf; /* Denoted by enum wd_comp_alg_type */ diff --git a/include/wd_comp.h b/include/wd_comp.h index 8e056d1..5d09536 100644 --- a/include/wd_comp.h +++ b/include/wd_comp.h @@ -151,6 +151,8 @@ struct wd_comp_sess_setup { enum wd_comp_winsz_type win_sz; /* Denoted by enum wd_comp_winsz_type */ enum wd_comp_op_type op_type; /* Denoted by enum wd_comp_op_type */ void *sched_param; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; /** diff --git a/wd_comp.c b/wd_comp.c index 5e62462..c67b7f1 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -15,7 +15,7 @@ #include "drv/wd_comp_drv.h" #include "wd_comp.h" -#define HW_CTX_SIZE (64 * 1024) +#define HW_CTX_SIZE 0x10000 #define STREAM_CHUNK (128 * 1024) #define WD_ZLIB_HEADER_SZ 2 #define WD_GZIP_HEADER_SZ 10 @@ -41,6 +41,8 @@ struct wd_comp_sess { __u32 checksum; __u8 *ctx_buf; void *sched_key; + struct wd_mm_ops mm_ops; + enum wd_mem_type mm_type; }; struct wd_comp_setting { @@ -437,6 +439,24 @@ static int wd_comp_check_sess_params(struct wd_comp_sess_setup *setup) return WD_SUCCESS; } +static int wd_alloc_ctx_buf(struct wd_mm_ops *mm_ops, struct wd_comp_sess *sess) +{ + + sess->ctx_buf = mm_ops->alloc(mm_ops->usr, HW_CTX_SIZE); + if (!sess->ctx_buf) + return -WD_ENOMEM; + + memset(sess->ctx_buf, 0, HW_CTX_SIZE); + + return WD_SUCCESS; +} + +static void wd_free_ctx_buf(struct wd_mm_ops *mm_ops, struct wd_comp_sess *sess) +{ + mm_ops->free(mm_ops->usr, sess->ctx_buf); + sess->ctx_buf = NULL; +} + handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) { struct wd_comp_sess *sess; @@ -453,8 +473,15 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) if (!sess) return (handle_t)0; - sess->ctx_buf = calloc(1, HW_CTX_SIZE); - if (!sess->ctx_buf) + /* Memory type set */ + ret = wd_mem_ops_init(wd_comp_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type); + if (ret) { + WD_ERR("failed to init memory ops!\n"); + goto sess_err; + } + + ret = wd_alloc_ctx_buf(&setup->mm_ops, sess); + if (ret) goto sess_err; sess->alg_type = setup->alg_type; @@ -462,6 +489,9 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) sess->win_sz = setup->win_sz; sess->stream_pos = WD_COMP_STREAM_NEW; + sess->mm_type = setup->mm_type; + memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops)); + /* Some simple scheduler don't need scheduling parameters */ sess->sched_key = (void *)wd_comp_setting.sched.sched_init( wd_comp_setting.sched.h_sched_ctx, setup->sched_param); @@ -473,7 +503,7 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) return (handle_t)sess; sched_err: - free(sess->ctx_buf); + wd_free_ctx_buf(&setup->mm_ops, sess); sess_err: free(sess); return (handle_t)0; @@ -487,7 +517,7 @@ void wd_comp_free_sess(handle_t h_sess) return; if (sess->ctx_buf) - free(sess->ctx_buf); + wd_free_ctx_buf(&sess->mm_ops, sess); if (sess->sched_key) free(sess->sched_key); @@ -522,6 +552,9 @@ static void fill_comp_msg(struct wd_comp_sess *sess, struct wd_comp_msg *msg, msg->win_sz = sess->win_sz; msg->avail_out = req->dst_len; + msg->mm_type = sess->mm_type; + msg->mm_ops = &sess->mm_ops; + msg->req.last = 1; } -- 2.33.0

From: Junchong Pan <panjunchong@h-partners.com> Add --sgl operations for sec and zip. Signed-off-by: Junchong Pan <panjunchong@h-partners.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- uadk_tool/benchmark/sec_uadk_benchmark.c | 205 +++++++++++++++++++---- uadk_tool/benchmark/uadk_benchmark.c | 13 ++ uadk_tool/benchmark/uadk_benchmark.h | 3 + uadk_tool/benchmark/zip_uadk_benchmark.c | 204 ++++++++++++++++++---- 4 files changed, 359 insertions(+), 66 deletions(-) diff --git a/uadk_tool/benchmark/sec_uadk_benchmark.c b/uadk_tool/benchmark/sec_uadk_benchmark.c index 141b161..4a64c94 100644 --- a/uadk_tool/benchmark/sec_uadk_benchmark.c +++ b/uadk_tool/benchmark/sec_uadk_benchmark.c @@ -72,6 +72,7 @@ static unsigned int g_algtype; static unsigned int g_optype; static unsigned int g_maclen; static unsigned int g_dev_id; +static unsigned int g_data_fmt; struct aead_alg_info { int index; @@ -970,6 +971,9 @@ static void save_aead_dst_data(u8 *addr, u32 size) static void read_aead_dst_data(u8 *addr, u32 len) { char file_name[SEC_SAVE_FILE_LEN] = {0}; + struct wd_datalist *current; + size_t bytes_to_read = 0; + size_t bytes_read = 0; char *alg_name; FILE *fp; int size; @@ -992,10 +996,29 @@ static void read_aead_dst_data(u8 *addr, u32 len) size = ftell(fp); rewind(fp); - size = fread(addr, 1, size, fp); - addr[size] = '\0'; - memcpy(g_save_mac, (char *)addr + len, SEC_MAX_MAC_LEN); + if (!g_data_fmt) { + size = fread(addr, 1, size, fp); + addr[size] = '\0'; + + memcpy(g_save_mac, (char *)addr + len, SEC_MAX_MAC_LEN); + } else { + current = (struct wd_datalist *)addr; + while (current && size > 0) { + bytes_to_read = current->len; + if (bytes_to_read > size) + bytes_to_read = size; + bytes_read = fread(current->data, 1, bytes_to_read, fp); + + if (bytes_read != bytes_to_read) { + SEC_TST_PRT("partial read: expected %zu, got %zu\n", bytes_to_read, bytes_read); + fclose(fp); + return; + } + size -= bytes_read; + current = current->next; + } + } fclose(fp); } @@ -1068,14 +1091,114 @@ static void free_ivkey_source(void) free(g_uadk_pool.iv); } +/* + * Calculate SGL unit size. + */ +static inline size_t cal_unit_sz(size_t sz) +{ + return (sz + SGL_ALIGNED_BYTES - 1) & ~(SGL_ALIGNED_BYTES - 1); +} + +/* + * Create SGL or common memory buffer. + */ +static void *create_buf(int sgl, size_t sz, size_t unit_sz) +{ + struct wd_datalist *head, *p, *q; + int i, tail_sz, sgl_num; + void *buf; + + buf = malloc(sz); + if (!buf) { + SEC_TST_PRT("Fail to allocate buffer %ld size!\n", sz); + return NULL; + } + + memset_buf(buf, sz); + + if (sgl == WD_FLAT_BUF) + return buf; + + if (g_alg != AEAD_TYPE) { + get_rand_data(buf, g_pktlen); + } else { + if (!g_optype) + get_aead_data(buf, g_pktlen + SEC_AEAD_LEN); + } + + tail_sz = sz % unit_sz; + sgl_num = sz / unit_sz; /* the number with unit_sz bytes */ + + /* the additional slot is for tail_sz */ + head = calloc(sgl_num + (tail_sz ? 1 : 0), sizeof(struct wd_datalist)); + if (!head) { + SEC_TST_PRT("Fail to allocate memory for SGL head!\n"); + goto out; + } + + q = NULL; + for (i = 0; i < sgl_num; i++) { + p = &head[i]; + p->data = buf + i * unit_sz; + p->len = unit_sz; + if (q) + q->next = p; + q = p; + } + + if (tail_sz) { + p = &head[i]; + p->data = buf + i * unit_sz; + p->len = tail_sz; + if (q) + q->next = p; + q = p; + } + + if (q) + q->next = NULL; + + return head; +out: + free(buf); + return NULL; +} + +static void free_buf(int sgl, void *buf) +{ + struct wd_datalist *head = buf; + struct wd_datalist *p = head; + void *data_buf = NULL; + + if (!buf) + return; + + if (sgl == WD_FLAT_BUF) { + free(buf); + return; + } + + if (head) + data_buf = head->data; + + /* free the whole data buffer of SGL */ + if (data_buf) + free(p->data); + + /* free SGL headers */ + free(buf); +} + static int init_uadk_bd_pool(void) { unsigned long step; + int unit_sz; int i, j; int ret; // make the block not align to 4K step = sizeof(char) * g_pktlen * 2; + unit_sz = cal_unit_sz(step); ret = init_ivkey_source(); if (ret) { @@ -1097,12 +1220,13 @@ static int init_uadk_bd_pool(void) goto malloc_error1; } for (j = 0; j < MAX_POOL_LENTH; j++) { - g_uadk_pool.pool[i].bds[j].src = malloc(step); - memset(g_uadk_pool.pool[i].bds[j].src, 0, step); + g_uadk_pool.pool[i].bds[j].src = create_buf(g_data_fmt, step, unit_sz); if (!g_uadk_pool.pool[i].bds[j].src) goto malloc_error2; - g_uadk_pool.pool[i].bds[j].dst = malloc(step); - memset(g_uadk_pool.pool[i].bds[j].dst, 0, step); + if (g_alg == DIGEST_TYPE) + g_uadk_pool.pool[i].bds[j].dst = malloc(step); + else + g_uadk_pool.pool[i].bds[j].dst = create_buf(g_data_fmt, step, unit_sz); if (!g_uadk_pool.pool[i].bds[j].dst) goto malloc_error3; g_uadk_pool.pool[i].bds[j].mac = malloc(SEC_MAX_MAC_LEN); @@ -1110,19 +1234,29 @@ static int init_uadk_bd_pool(void) if (!g_uadk_pool.pool[i].bds[j].mac) goto malloc_error4; - if (g_alg != AEAD_TYPE) { - get_rand_data(g_uadk_pool.pool[i].bds[j].src, g_pktlen); - if (g_prefetch) - get_rand_data(g_uadk_pool.pool[i].bds[j].dst, - g_pktlen); - } else { - if (!g_optype) - get_aead_data(g_uadk_pool.pool[i].bds[j].src, - g_pktlen + SEC_AEAD_LEN); - else { + if (g_data_fmt == WD_FLAT_BUF) { + if (g_alg != AEAD_TYPE) { + get_rand_data(g_uadk_pool.pool[i].bds[j].src, g_pktlen); + if (g_prefetch) + get_rand_data(g_uadk_pool.pool[i].bds[j].dst, + g_pktlen); + } else { + if (!g_optype) + get_aead_data(g_uadk_pool.pool[i].bds[j].src, + g_pktlen + SEC_AEAD_LEN); + else { + read_aead_dst_data(g_uadk_pool.pool[i].bds[j].src, + g_pktlen + SEC_AEAD_LEN); + memcpy(g_uadk_pool.pool[i].bds[j].mac, g_save_mac, SEC_MAX_MAC_LEN); + } + } + } else if (g_data_fmt == WD_SGL_BUF) { + if (g_optype && g_alg == AEAD_TYPE) { read_aead_dst_data(g_uadk_pool.pool[i].bds[j].src, - g_pktlen + SEC_AEAD_LEN); + g_pktlen + SEC_AEAD_LEN); memcpy(g_uadk_pool.pool[i].bds[j].mac, g_save_mac, SEC_MAX_MAC_LEN); + } else if (g_prefetch && g_alg == DIGEST_TYPE) { + get_rand_data(g_uadk_pool.pool[i].bds[j].dst, g_pktlen); } } } @@ -1132,20 +1266,23 @@ static int init_uadk_bd_pool(void) return 0; malloc_error4: - free(g_uadk_pool.pool[i].bds[j].dst); + if (g_alg == DIGEST_TYPE) + free(g_uadk_pool.pool[i].bds[j].dst); + else + free_buf(g_data_fmt, g_uadk_pool.pool[i].bds[j].dst); malloc_error3: - free(g_uadk_pool.pool[i].bds[j].src); + free_buf(g_data_fmt, g_uadk_pool.pool[i].bds[j].src); malloc_error2: for (j--; j >= 0; j--) { - free(g_uadk_pool.pool[i].bds[j].src); - free(g_uadk_pool.pool[i].bds[j].dst); + free_buf(g_data_fmt, g_uadk_pool.pool[i].bds[j].src); + free_buf(g_data_fmt, g_uadk_pool.pool[i].bds[j].dst); free(g_uadk_pool.pool[i].bds[j].mac); } malloc_error1: for (i--; i >= 0; i--) { for (j = 0; j < MAX_POOL_LENTH; j++) { - free(g_uadk_pool.pool[i].bds[j].src); - free(g_uadk_pool.pool[i].bds[j].dst); + free_buf(g_data_fmt, g_uadk_pool.pool[i].bds[j].src); + free_buf(g_data_fmt, g_uadk_pool.pool[i].bds[j].dst); free(g_uadk_pool.pool[i].bds[j].mac); } free(g_uadk_pool.pool[i].bds); @@ -1173,8 +1310,11 @@ static void free_uadk_bd_pool(void) for (i = 0; i < g_thread_num; i++) { if (g_uadk_pool.pool[i].bds) { for (j = 0; j < MAX_POOL_LENTH; j++) { - free(g_uadk_pool.pool[i].bds[j].src); - free(g_uadk_pool.pool[i].bds[j].dst); + free_buf(g_data_fmt, g_uadk_pool.pool[i].bds[j].src); + if (g_alg == DIGEST_TYPE) + free(g_uadk_pool.pool[i].bds[j].dst); + else + free_buf(g_data_fmt, g_uadk_pool.pool[i].bds[j].dst); free(g_uadk_pool.pool[i].bds[j].mac); } } @@ -1655,7 +1795,7 @@ static void *sec_uadk_cipher_async(void *arg) creq.in_bytes = g_pktlen; creq.out_bytes = g_pktlen; creq.out_buf_bytes = g_pktlen; - creq.data_fmt = 0; + creq.data_fmt = g_data_fmt; creq.state = 0; creq.cb = cipher_async_cb; @@ -1787,7 +1927,7 @@ static void *sec_uadk_aead_async(void *arg) else areq.out_bytes = g_pktlen + 32; // aadsize + authsize = 32; - areq.data_fmt = 0; + areq.data_fmt = g_data_fmt; areq.state = 0; areq.cb = aead_async_cb; @@ -1895,7 +2035,7 @@ static void *sec_uadk_digest_async(void *arg) dreq.in_bytes = g_pktlen; dreq.out_bytes = pdata->d_outbytes; dreq.out_buf_bytes = pdata->d_outbytes; - dreq.data_fmt = 0; + dreq.data_fmt = g_data_fmt; dreq.state = 0; dreq.has_next = 0; dreq.cb = digest_async_cb; @@ -1997,7 +2137,7 @@ static void *sec_uadk_cipher_sync(void *arg) creq.in_bytes = g_pktlen; creq.out_bytes = g_pktlen; creq.out_buf_bytes = g_pktlen; - creq.data_fmt = 0; + creq.data_fmt = g_data_fmt; creq.state = 0; while(1) { @@ -2101,7 +2241,7 @@ static void *sec_uadk_aead_sync(void *arg) else areq.out_bytes = g_pktlen + 32; // aadsize + authsize = 32; - areq.data_fmt = 0; + areq.data_fmt = g_data_fmt; areq.state = 0; while(1) { @@ -2176,7 +2316,7 @@ static void *sec_uadk_digest_sync(void *arg) dreq.in_bytes = g_pktlen; dreq.out_bytes = pdata->d_outbytes; dreq.out_buf_bytes = pdata->d_outbytes; - dreq.data_fmt = 0; + dreq.data_fmt = g_data_fmt; dreq.state = 0; dreq.has_next = 0; @@ -2355,6 +2495,7 @@ int sec_uadk_benchmark(struct acc_option *options) g_alg = options->subtype; g_optype = options->optype; g_algtype = options->algtype; + g_data_fmt = options->data_fmt; if (g_alg == AEAD_TYPE) { g_maclen = get_aead_mac_len(g_algtype); diff --git a/uadk_tool/benchmark/uadk_benchmark.c b/uadk_tool/benchmark/uadk_benchmark.c index 24737c5..f57c4f9 100644 --- a/uadk_tool/benchmark/uadk_benchmark.c +++ b/uadk_tool/benchmark/uadk_benchmark.c @@ -370,6 +370,15 @@ void segmentfault_handler(int sig) exit(1); } +void memset_buf(void *buf, size_t sz) +{ + char *ch = (char *)buf; + size_t i; + + for (i = 0; i < sz; i++) + ch[i] = 0; +} + /*-------------------------------------main code------------------------------------------------------*/ static void parse_alg_param(struct acc_option *option) { @@ -741,6 +750,7 @@ int acc_cmd_parse(int argc, char *argv[], struct acc_option *option) {"init2", no_argument, 0, 17}, {"device", required_argument, 0, 18}, {"memory", required_argument, 0, 19}, + {"sgl", no_argument, 0, 20}, {0, 0, 0, 0} }; @@ -815,6 +825,9 @@ int acc_cmd_parse(int argc, char *argv[], struct acc_option *option) case 19: option->mem_type = strtol(optarg, NULL, 0); break; + case 20: + option->data_fmt = WD_SGL_BUF; + break; default: ACC_TST_PRT("invalid: bad input parameter!\n"); print_benchmark_help(); diff --git a/uadk_tool/benchmark/uadk_benchmark.h b/uadk_tool/benchmark/uadk_benchmark.h index 81ace1b..83fd7fa 100644 --- a/uadk_tool/benchmark/uadk_benchmark.h +++ b/uadk_tool/benchmark/uadk_benchmark.h @@ -37,6 +37,7 @@ #define SEND_USLEEP 100 #define SEC_2_USEC 1000000 #define HASH_ZISE 16 +#define SGL_ALIGNED_BYTES 64 #define SCHED_SINGLE "sched_single" #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) @@ -82,6 +83,7 @@ struct acc_option { u32 sched_type; int task_type; int mem_type; + u32 data_fmt; }; enum uadk_mem_mode { @@ -230,6 +232,7 @@ extern u32 get_recv_time(void); extern void cal_avg_latency(u32 count); extern int get_alg_name(int alg, char *alg_name); extern void segmentfault_handler(int sig); +extern void memset_buf(void *buf, size_t sz); int uadk_parse_dev_id(char *dev_name); int acc_cmd_parse(int argc, char *argv[], struct acc_option *option); diff --git a/uadk_tool/benchmark/zip_uadk_benchmark.c b/uadk_tool/benchmark/zip_uadk_benchmark.c index fc81c2b..092f710 100644 --- a/uadk_tool/benchmark/zip_uadk_benchmark.c +++ b/uadk_tool/benchmark/zip_uadk_benchmark.c @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: Apache-2.0 */ #include <numa.h> +#include <math.h> #include "uadk_benchmark.h" #include "zip_uadk_benchmark.h" @@ -84,6 +85,7 @@ static unsigned int g_pktlen; static unsigned int g_prefetch; static unsigned int g_state; static unsigned int g_dev_id; +static unsigned int g_data_fmt; #ifndef ZLIB_FSE static ZSTD_CCtx* zstd_soft_fse_init(unsigned int level) @@ -175,8 +177,15 @@ fd_error: static int load_file_data(const char *alg, u32 pkg_len, u32 optype) { struct zip_file_head *fhead = NULL; + struct wd_datalist *src_curr = NULL; + struct wd_datalist *dst_curr = NULL; char file_path[PATH_SIZE]; + size_t total_read = 0; + size_t total_len = 0; + size_t remaining = 0; ssize_t size = 0xff; + size_t copied = 0; + size_t len = 0; int i, j, fd; int ret; @@ -216,30 +225,63 @@ static int load_file_data(const char *alg, u32 pkg_len, u32 optype) // read data for one buffer one buffer from file line for (j = 0; j < MAX_POOL_LENTH_COMP; j++) { - memset(g_zip_pool.pool[0].bds[j].src, 0x0, - g_zip_pool.pool[0].bds[j].src_len); - if (size != 0) { // zero size buffer no need to read; - size = read(fd, g_zip_pool.pool[0].bds[j].src, - fhead->blk_sz[j]); - if (size < 0) { - ZIP_TST_PRT("Decompress read data error size: %lu!\n", size); - ret = -EINVAL; - goto read_err; - } else if (size == 0) { - ZIP_TST_PRT("Read file to the end!"); + if (g_data_fmt == 0) { + if (fhead->blk_sz[j] != 0) { + size = read(fd, g_zip_pool.pool[0].bds[j].src, fhead->blk_sz[j]); + if (size < 0) { + ZIP_TST_PRT("Decompress read data error size: %ld!\n", size); + ret = -EINVAL; + goto read_err; + } + g_zip_pool.pool[0].bds[j].src_len = size; + } else { + g_zip_pool.pool[0].bds[j].src_len = 0; } + } else { + src_curr = (struct wd_datalist *)g_zip_pool.pool[0].bds[j].src; + remaining = fhead->blk_sz[j]; + total_read = 0; + while (src_curr && remaining > 0) { + len = fmin(remaining, src_curr->len); + size = read(fd, src_curr->data, len); + if (size < 0) { + ZIP_TST_PRT("Decompress read data error at block %d!\n", j); + ret = -EINVAL; + goto read_err; + } + total_read += size; + remaining -= size; + src_curr = src_curr->next; + } + g_zip_pool.pool[0].bds[j].src_len = total_read; } - g_zip_pool.pool[0].bds[j].src_len = size; } for (i = 1; i < g_thread_num; i++) { for (j = 0; j < MAX_POOL_LENTH_COMP; j++) { - if (g_zip_pool.pool[0].bds[j].src_len) + if (g_zip_pool.pool[0].bds[j].src_len == 0) + continue; + + if (g_data_fmt == 0) { memcpy(g_zip_pool.pool[i].bds[j].src, g_zip_pool.pool[0].bds[j].src, g_zip_pool.pool[0].bds[j].src_len); - g_zip_pool.pool[i].bds[j].src_len = - g_zip_pool.pool[0].bds[j].src_len; + g_zip_pool.pool[i].bds[j].src_len = g_zip_pool.pool[0].bds[j].src_len; + } else { + src_curr = (struct wd_datalist *)g_zip_pool.pool[0].bds[j].src; + dst_curr = (struct wd_datalist *)g_zip_pool.pool[i].bds[j].src; + total_len = g_zip_pool.pool[0].bds[j].src_len; + copied = 0; + while (src_curr && dst_curr && copied < total_len) { + len = fmin(src_curr->len, dst_curr->len); + len = fmin(len, total_len - copied); + memcpy(dst_curr->data, src_curr->data, len); + copied += len; + src_curr = src_curr->next; + dst_curr = dst_curr->next; + } + g_zip_pool.pool[i].bds[j].src_len = total_len; + } } } @@ -575,8 +617,98 @@ static void uninit_ctx_config(void) wd_sched_rr_release(g_sched); } +/* + * Calculate SGL unit size. + */ +static inline size_t cal_unit_sz(size_t sz) +{ + return (sz + SGL_ALIGNED_BYTES - 1) & ~(SGL_ALIGNED_BYTES - 1); +} + +/* + * Create SGL or common memory buffer. + */ +static void *create_buf(int sgl, size_t sz, size_t unit_sz) +{ + struct wd_datalist *head, *p, *q; + int i, tail_sz, sgl_num; + void *buf; + + buf = malloc(sz); + if (!buf) { + ZIP_TST_PRT("Fail to allocate buffer %ld size!\n", sz); + return NULL; + } + + memset_buf(buf, sz); + + if (sgl == WD_FLAT_BUF) + return buf; + + if (sz == g_pktlen) { + get_rand_data(buf, sz * COMPRESSION_RATIO_FACTOR); + } else { + if (g_prefetch) + get_rand_data(buf, sz); + } + tail_sz = sz % unit_sz; + sgl_num = sz / unit_sz; /* the number with unit_sz bytes */ + + /* the additional slot is for tail_sz */ + head = calloc(sgl_num + 1, sizeof(struct wd_datalist)); + if (!head) { + ZIP_TST_PRT("Fail to allocate memory for SGL head!\n"); + goto out; + } + + q = NULL; + for (i = 0; i < sgl_num; i++) { + p = &head[i]; + p->data = buf + i * unit_sz; + p->len = unit_sz; + if (q) + q->next = p; + q = p; + } + + if (tail_sz) { + p = &head[i]; + p->data = buf + i * unit_sz; + p->len = tail_sz; + if (q) + q->next = p; + q = p; + } + + if (q) + q->next = NULL; + + return head; +out: + free(buf); + return NULL; +} + +static void free_buf(int sgl, void *buf) +{ + struct wd_datalist *p; + + if (!buf) + return; + if (sgl == WD_FLAT_BUF) { + free(buf); + return; + } + p = (struct wd_datalist *)buf; + /* free the whole data buffer of SGL */ + free(p->data); + /* free SGL headers */ + free(buf); +} + static int init_uadk_bd_pool(u32 optype) { + int unit_sz; u32 outsize; u32 insize; int i, j; @@ -605,37 +737,40 @@ static int init_uadk_bd_pool(u32 optype) goto malloc_error1; } for (j = 0; j < MAX_POOL_LENTH_COMP; j++) { - g_zip_pool.pool[i].bds[j].src = calloc(1, insize); + unit_sz = cal_unit_sz(insize); + g_zip_pool.pool[i].bds[j].src = create_buf(g_data_fmt, insize, unit_sz); if (!g_zip_pool.pool[i].bds[j].src) goto malloc_error2; g_zip_pool.pool[i].bds[j].src_len = insize; - g_zip_pool.pool[i].bds[j].dst = malloc(outsize); + unit_sz = cal_unit_sz(outsize); + g_zip_pool.pool[i].bds[j].dst = create_buf(g_data_fmt, outsize, unit_sz); if (!g_zip_pool.pool[i].bds[j].dst) goto malloc_error3; g_zip_pool.pool[i].bds[j].dst_len = outsize; - get_rand_data(g_zip_pool.pool[i].bds[j].src, - insize * COMPRESSION_RATIO_FACTOR); - if (g_prefetch) - get_rand_data(g_zip_pool.pool[i].bds[j].dst, outsize); + if (g_data_fmt == WD_FLAT_BUF) { + get_rand_data(g_zip_pool.pool[i].bds[j].src, insize * COMPRESSION_RATIO_FACTOR); + if (g_prefetch) + get_rand_data(g_zip_pool.pool[i].bds[j].dst, outsize); + } } } return 0; malloc_error3: - free(g_zip_pool.pool[i].bds[j].src); + free_buf(g_data_fmt, g_zip_pool.pool[i].bds[j].src); malloc_error2: for (j--; j >= 0; j--) { - free(g_zip_pool.pool[i].bds[j].src); - free(g_zip_pool.pool[i].bds[j].dst); + free_buf(g_data_fmt, g_zip_pool.pool[i].bds[j].src); + free_buf(g_data_fmt, g_zip_pool.pool[i].bds[j].dst); } malloc_error1: for (i--; i >= 0; i--) { for (j = 0; j < MAX_POOL_LENTH_COMP; j++) { - free(g_zip_pool.pool[i].bds[j].src); - free(g_zip_pool.pool[i].bds[j].dst); + free_buf(g_data_fmt, g_zip_pool.pool[i].bds[j].src); + free_buf(g_data_fmt, g_zip_pool.pool[i].bds[j].dst); } free(g_zip_pool.pool[i].bds); g_zip_pool.pool[i].bds = NULL; @@ -654,8 +789,8 @@ static void free_uadk_bd_pool(void) for (i = 0; i < g_thread_num; i++) { if (g_zip_pool.pool[i].bds) { for (j = 0; j < MAX_POOL_LENTH_COMP; j++) { - free(g_zip_pool.pool[i].bds[j].src); - free(g_zip_pool.pool[i].bds[j].dst); + free_buf(g_data_fmt, g_zip_pool.pool[i].bds[j].src); + free_buf(g_data_fmt, g_zip_pool.pool[i].bds[j].dst); } } free(g_zip_pool.pool[i].bds); @@ -938,7 +1073,7 @@ static void *zip_uadk_blk_lz77_sync_run(void *arg) out_len = uadk_pool->bds[0].dst_len; creq.cb = NULL; - creq.data_fmt = 0; + creq.data_fmt = g_data_fmt; creq.status = 0; ftuple = malloc(sizeof(COMP_TUPLE_TAG) * MAX_POOL_LENTH_COMP); @@ -1048,7 +1183,7 @@ static void *zip_uadk_stm_lz77_sync_run(void *arg) out_len = uadk_pool->bds[0].dst_len; creq.cb = NULL; - creq.data_fmt = 0; + creq.data_fmt = g_data_fmt; creq.status = 0; ftuple = malloc(sizeof(COMP_TUPLE_TAG) * MAX_POOL_LENTH_COMP); @@ -1153,7 +1288,7 @@ static void *zip_uadk_blk_lz77_async_run(void *arg) out_len = uadk_pool->bds[0].dst_len; creq.cb = zip_lz77_async_cb; - creq.data_fmt = 0; + creq.data_fmt = g_data_fmt; creq.status = 0; while(1) { @@ -1242,7 +1377,7 @@ static void *zip_uadk_blk_sync_run(void *arg) out_len = uadk_pool->bds[0].dst_len; creq.cb = NULL; - creq.data_fmt = 0; + creq.data_fmt = g_data_fmt; creq.priv = 0; creq.status = 0; @@ -1317,7 +1452,7 @@ static void *zip_uadk_stm_sync_run(void *arg) out_len = uadk_pool->bds[0].dst_len; creq.cb = NULL; - creq.data_fmt = 0; + creq.data_fmt = g_data_fmt; creq.priv = 0; creq.status = 0; @@ -1397,7 +1532,7 @@ static void *zip_uadk_blk_async_run(void *arg) out_len = uadk_pool->bds[0].dst_len; creq.cb = zip_async_cb; - creq.data_fmt = 0; + creq.data_fmt = g_data_fmt; creq.priv = 0; creq.status = 0; @@ -1634,6 +1769,7 @@ int zip_uadk_benchmark(struct acc_option *options) g_pktlen = options->pktlen; g_ctxnum = options->ctxnums; g_prefetch = options->prefetch; + g_data_fmt = options->data_fmt; if (options->optype >= WD_DIR_MAX * 2) { ZIP_TST_PRT("ZIP optype error: %u\n", options->optype); -- 2.33.0

From: Wenkai Lin <linwenkai6@hisilicon.com> Extract a common function fill_hashagg_data_info. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- drv/hisi_dae.c | 56 +++++++++++++++++++------------------------------- 1 file changed, 21 insertions(+), 35 deletions(-) diff --git a/drv/hisi_dae.c b/drv/hisi_dae.c index 4f4d13c..49387aa 100644 --- a/drv/hisi_dae.c +++ b/drv/hisi_dae.c @@ -26,7 +26,7 @@ #define DAE_VCHAR_OFFSET_SIZE 2 #define DAE_COL_BIT_NUM 4 #define DAE_AGG_START_COL 16 -#define DAE_HASHAGG_MAX_ROW_NUN 50000 +#define DAE_HASHAGG_MAX_ROW_NUM 50000 /* align size */ #define DAE_CHAR_ALIGN_SIZE 4 @@ -294,23 +294,8 @@ static void fill_hashagg_merge_key_data(struct dae_sqe *sqe, struct dae_ext_sqe } } -static void fill_hashagg_normal_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe, - struct hashagg_col_data *cols_data, __u32 agg_cols_num) -{ - struct hw_agg_data *agg_data = cols_data->input_data; - __u32 i; - - for (i = 0; i < agg_cols_num; i++) { - sqe->agg_data_type[i] = agg_data[i].hw_type; - sqe->agg_data_type[i] |= agg_data[i].sum_outtype << DAE_COL_BIT_NUM; - ext_sqe->agg_data_info[i] = agg_data[i].data_info; - } - - sqe->agg_col_bitmap = GENMASK(agg_cols_num + DAE_AGG_START_COL - 1, DAE_AGG_START_COL); -} - -static void fill_hashagg_rehash_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe, - struct hw_agg_data *agg_data, __u32 agg_cols_num) +static void fill_hashagg_data_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe, + struct hw_agg_data *agg_data, __u32 agg_cols_num) { __u32 i; @@ -344,14 +329,14 @@ static void fill_hashagg_input_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext hw_agg_addr = &addr_list->input_addr[DAE_AGG_START_COL]; usr_agg_addr = msg->req.agg_cols; agg_col_num = msg->agg_cols_num; - fill_hashagg_normal_info(sqe, ext_sqe, cols_data, agg_col_num); + fill_hashagg_data_info(sqe, ext_sqe, agg_data, agg_col_num); break; case WD_AGG_REHASH_INPUT: agg_data = cols_data->output_data; hw_agg_addr = &addr_list->input_addr[DAE_AGG_START_COL]; usr_agg_addr = msg->req.agg_cols; agg_col_num = cols_data->output_num; - fill_hashagg_rehash_info(sqe, ext_sqe, agg_data, agg_col_num); + fill_hashagg_data_info(sqe, ext_sqe, agg_data, agg_col_num); break; case WD_AGG_STREAM_OUTPUT: case WD_AGG_REHASH_OUTPUT: @@ -359,7 +344,7 @@ static void fill_hashagg_input_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext hw_agg_addr = &addr_list->output_addr[DAE_AGG_START_COL]; usr_agg_addr = msg->req.out_agg_cols; agg_col_num = cols_data->output_num; - fill_hashagg_normal_info(sqe, ext_sqe, cols_data, cols_data->input_num); + fill_hashagg_data_info(sqe, ext_sqe, cols_data->input_data, cols_data->input_num); break; } @@ -385,7 +370,7 @@ static void fill_hashagg_merge_input_data(struct dae_sqe *sqe, struct dae_ext_sq struct hashagg_ctx *agg_ctx = msg->priv; struct hashagg_col_data *cols_data = &agg_ctx->cols_data; - fill_hashagg_rehash_info(sqe, ext_sqe, cols_data->output_data, msg->agg_cols_num); + fill_hashagg_data_info(sqe, ext_sqe, cols_data->output_data, msg->agg_cols_num); } static void fill_hashagg_ext_addr(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe, @@ -422,26 +407,15 @@ static void fill_hashagg_info(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe, static int check_hashagg_param(struct wd_agg_msg *msg) { - struct hashagg_col_data *cols_data; - struct hashagg_ctx *agg_ctx; - if (!msg) { WD_ERR("invalid: input hashagg msg is NULL!\n"); return -WD_EINVAL; } - agg_ctx = msg->priv; - cols_data = &agg_ctx->cols_data; - if (cols_data->output_num > DAE_MAX_OUTPUT_COLS) { - WD_ERR("invalid: input hashagg output num %u is more than %d!\n", - cols_data->output_num, DAE_MAX_OUTPUT_COLS); - return -WD_EINVAL; - } - if ((msg->pos == WD_AGG_STREAM_INPUT || msg->pos == WD_AGG_REHASH_INPUT) && - msg->row_count > DAE_HASHAGG_MAX_ROW_NUN) { + msg->row_count > DAE_HASHAGG_MAX_ROW_NUM) { WD_ERR("invalid: input hashagg row count %u is more than %d!\n", - msg->row_count, DAE_HASHAGG_MAX_ROW_NUN); + msg->row_count, DAE_HASHAGG_MAX_ROW_NUM); return -WD_EINVAL; } @@ -1005,6 +979,7 @@ static int transfer_input_col_info(struct wd_agg_col_info *agg_cols, struct hw_agg_data *user_output_data, __u32 cols_num, __u32 *output_num) { + __u32 tmp = *output_num; __u32 i, j, k = 0; int ret; @@ -1013,7 +988,15 @@ static int transfer_input_col_info(struct wd_agg_col_info *agg_cols, WD_ERR("invalid: col alg num(%u) more than 2!\n", agg_cols[i].col_alg_num); return -WD_EINVAL; } + tmp += agg_cols[i].col_alg_num; + } + if (tmp > DAE_MAX_OUTPUT_COLS) { + WD_ERR("invalid: output col num is more than %d!\n", DAE_MAX_OUTPUT_COLS); + return -WD_EINVAL; + } + + for (i = 0; i < cols_num; i++) { for (j = 0; j < agg_cols[i].col_alg_num; j++) { ret = hashagg_check_input_data(&agg_cols[i], &user_input_data[i], &user_output_data[k], j); @@ -1137,6 +1120,9 @@ static int transfer_data_to_hw_type(struct hashagg_col_data *cols_data, struct wd_agg_col_info *agg_cols = setup->agg_cols_info; int ret; + if (setup->is_count_all) + cols_data->output_num++; + ret = transfer_input_col_info(agg_cols, user_input_data, user_output_data, setup->agg_cols_num, &cols_data->output_num); if (ret) -- 2.33.0