[PATCH 1/4] uadk: fix scheduler ctx allocation and sched_params initialization
The scheduler ctx allocation under SCHED_POLICY_DEV and session initialization in several modules contain bugs that cause async process hangs, device starvation, and nondeterministic scheduling behavior. The following four issues are addressed: Under SCHED_POLICY_DEV, allocating sync ctxs then async ctxs in two passes let concurrent contention split them across devices. The DEV scheduler pins the region to ctxs[0]'s device and forbids cross-region fallback, so the async domain ended empty: async picks returned idx 65535, sends failed with -22, and the async process hung. Add a DEV-only path that opens sync/async ctxs interleaved on ONE device, with a preferred_dev_path field to pin later ctxs of the group. Then simplify it to a linear two-pass per device: open all sync then all async; any shortfall rolls back the whole group and tries the next device. wd_hw_alloc_ctx picked the first device whose wd_request_ctx() succeeded. When PF and VFs share the same NUMA node, concurrent businesses all bind to the same leading device, starving the others. Select the device with the most available contexts (wd_get_avail_ctx) instead, with fallback to remaining devices if the chosen one fails to open. wd_agg_alloc_sess, wd_join_gather_alloc_sess and wd_udma_alloc_sess declared a local struct wd_sched_params without initialization, setting only alg_name and ctxs before passing it to wd_sched_set_param, which unconditionally reads pkt_size, data_mode and prio_mode. The uninitialized stack values caused nondeterministic scheduling behavior. Initialize the struct with = {0} at declaration. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Co-developed-by: Wenkai Lin <linwenkai6@hisilicon.com> Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> Co-developed-by: Weili Qian <qianweili@huawei.com> Signed-off-by: Weili Qian <qianweili@huawei.com> --- drv/wd_drv.c | 67 ++++++++++-- include/wd_alg.h | 3 + wd_agg.c | 2 +- wd_join_gather.c | 2 +- wd_udma.c | 2 +- wd_util.c | 279 ++++++++++++++++++++++++++++++++++++++--------- 6 files changed, 291 insertions(+), 64 deletions(-) diff --git a/drv/wd_drv.c b/drv/wd_drv.c index 381cee1..08014dd 100644 --- a/drv/wd_drv.c +++ b/drv/wd_drv.c @@ -181,12 +181,12 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) { struct wd_drv_ctx_params *ctx_params = (struct wd_drv_ctx_params *)params; struct uacce_dev_list *dev_list, *used_list = NULL; + struct uacce_dev_list *curr, *best = NULL; char alg_type[CRYPTO_MAX_ALG_NAME]; - struct uacce_dev_list *curr; + int target_numa, best_avail = 0; struct wd_ctx_h *ctx_h; - int target_numa; + int ret, avail; handle_t hctx; - int ret; if (!params || !ctx) { WD_ERR("invalid: parameters are NULL!\n"); @@ -220,14 +220,61 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) goto out; } - curr = used_list; - while (curr) { - if (curr->dev) { - hctx = wd_request_ctx(curr->dev); - if (hctx) - goto success; + /* + * Preferred-device binding (SCHED_POLICY_DEV): open exactly this device + * and fail strictly if unavailable, so the sync and async ctxs of one + * (driver, op_type, numa) stay on the same device. No NUMA fallback. + */ + if (ctx_params->preferred_dev_path) { + for (curr = used_list; curr; curr = curr->next) { + if (curr->dev && !strcmp(curr->dev->char_dev_path, + ctx_params->preferred_dev_path)) + break; } - curr = curr->next; + + if (!curr) { + ret = -WD_ENODEV; + goto out; + } + + hctx = wd_request_ctx(curr->dev); + if (!hctx) { + ret = -WD_EBUSY; + goto out; + } + + goto success; + } + + /* Pick the device with the most available contexts to balance load. */ + for (curr = used_list; curr; curr = curr->next) { + if (!curr->dev) + continue; + avail = wd_get_avail_ctx(curr->dev); + if (avail > best_avail) { + best_avail = avail; + best = curr; + } + } + + if (!best) { + WD_ERR("failed to request ctx on NUMA node %d for %s\n", + target_numa, alg_name); + ret = -WD_EBUSY; + goto out; + } + + hctx = wd_request_ctx(best->dev); + if (hctx) + goto success; + + /* Fall back to remaining devices if the best one fails to open */ + for (curr = used_list; curr; curr = curr->next) { + if (curr == best || !curr->dev) + continue; + hctx = wd_request_ctx(curr->dev); + if (hctx) + goto success; } WD_ERR("failed to request ctx on NUMA node %d for %s\n", diff --git a/include/wd_alg.h b/include/wd_alg.h index b256a2f..3cc4e43 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -96,6 +96,8 @@ enum alg_drv_type { * @ctx_mode: CTX_MODE_SYNC or CTX_MODE_ASYNC * @op_type: Operation type * @bmp: NUMA node bitmask (optional, NULL if not needed) + * @preferred_dev_path: Preferred device char_dev_path; NULL/empty = auto-select, + * non-empty = open only this device and fail strictly if unavailable */ struct wd_drv_ctx_params { __u8 ctx_mode; @@ -103,6 +105,7 @@ struct wd_drv_ctx_params { int numa_id; bool epoll_en; struct bitmask *bmp; + const char *preferred_dev_path; }; /** diff --git a/wd_agg.c b/wd_agg.c index 95cb869..2c7825d 100644 --- a/wd_agg.c +++ b/wd_agg.c @@ -446,7 +446,7 @@ static int wd_agg_uninit_sess_priv(struct wd_agg_sess *sess) handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) { - struct wd_sched_params params; + struct wd_sched_params params = {0}; struct wd_agg_sess *sess; __u32 out_agg_cols_num = 0; int ret; diff --git a/wd_join_gather.c b/wd_join_gather.c index e43dbf9..27472d3 100644 --- a/wd_join_gather.c +++ b/wd_join_gather.c @@ -533,7 +533,7 @@ static int wd_join_gather_init_sess(struct wd_join_gather_sess *sess, handle_t wd_join_gather_alloc_sess(struct wd_join_gather_sess_setup *setup) { - struct wd_sched_params params; + struct wd_sched_params params = {0}; struct wd_join_gather_sess *sess; int ret; diff --git a/wd_udma.c b/wd_udma.c index 582c8d1..59841bb 100644 --- a/wd_udma.c +++ b/wd_udma.c @@ -78,7 +78,7 @@ void wd_udma_free_sess(handle_t sess) handle_t wd_udma_alloc_sess(struct wd_udma_sess_setup *setup) { - struct wd_sched_params params; + struct wd_sched_params params = {0}; struct wd_udma_sess *sess; int ret; diff --git a/wd_util.c b/wd_util.c index 1b4c74e..0d482f9 100644 --- a/wd_util.c +++ b/wd_util.c @@ -2613,17 +2613,17 @@ static int wd_alg_sched_instance(struct wd_sched *sched, return WD_SUCCESS; } -static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, - __u32 allocated_count) +static void wd_free_ctxs_batch_range(struct wd_init_attrs *attrs, + __u32 start_idx, __u32 end_idx) { struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_alg_driver *drv; __u32 i; - if (!internal_config || !internal_config->ctxs || !allocated_count) + if (!internal_config || !internal_config->ctxs || start_idx >= end_idx) return; - for (i = 0; i < allocated_count; i++) { + for (i = start_idx; i < end_idx; i++) { if (!internal_config->ctxs[i].ctx) continue; @@ -2635,21 +2635,76 @@ static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, } } +static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, + __u32 allocated_count) +{ + wd_free_ctxs_batch_range(attrs, 0, allocated_count); +} + +static void wd_collect_numa_nodes(struct bitmask *bmp, int *numa_nodes, + __u8 *numa_count) +{ + int max_node, n; + + max_node = numa_max_node() + 1; + if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) + max_node = UADK_MAX_NUMA_NODES; + + *numa_count = 0; + + for (n = 0; n < max_node; n++) { + if (numa_bitmask_isbitset(bmp, n)) + numa_nodes[(*numa_count)++] = n; + } + + if (!*numa_count) { + numa_nodes[0] = 0; + *numa_count = 1; + } +} + +static int wd_alloc_one_ctx(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + struct wd_drv_ctx_params *dparams, + __u32 *ctx_idx) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + handle_t ctx; + int ret; + + ret = drv->alloc_ctx(attrs->alg, dparams, &ctx); + if (ret < 0) + return ret; + + if (!ctx) + return -WD_EINVAL; + + internal_config->ctxs[*ctx_idx].ctx = ctx; + internal_config->ctxs[*ctx_idx].op_type = dparams->op_type; + internal_config->ctxs[*ctx_idx].ctx_mode = dparams->ctx_mode; + internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; + internal_config->ctxs[*ctx_idx].drv = drv; + (*ctx_idx)++; + return WD_SUCCESS; +} + static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, struct wd_alg_driver *drv, __u8 ctx_mode, __u8 op_type, __u32 *ctx_idx) { - struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_ctx_params *ctx_params = attrs->ctx_params; - struct wd_drv_ctx_params dparams; + struct wd_drv_ctx_params dparams = {0}; int numa_nodes[UADK_MAX_NUMA_NODES]; - __u32 mode_ctx_num, numa_count = 0; - __u32 numa_idx, j; - int max_node, n; - handle_t ctx; + __u32 mode_ctx_num, numa_idx, j; + __u8 numa_count; int ret; + dparams.ctx_mode = ctx_mode; + dparams.op_type = op_type; + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + if (ctx_mode == CTX_MODE_SYNC) mode_ctx_num = ctx_params->ctx_set_num[op_type].sync_ctx_num; else @@ -2661,46 +2716,26 @@ static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, attrs->sched_type == SCHED_POLICY_SINGLE) && mode_ctx_num > 1) mode_ctx_num = 1; - if (drv->calc_type == UADK_ALG_HW) { - max_node = numa_max_node() + 1; - if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) - max_node = UADK_MAX_NUMA_NODES; - for (n = 0; n < max_node; n++) { - if (numa_bitmask_isbitset(ctx_params->bmp, n)) - numa_nodes[numa_count++] = n; - } - } else { - numa_nodes[0] = 0; - numa_count = 1; - } - if (!numa_count) { + if (drv->calc_type != UADK_ALG_HW) { numa_nodes[0] = 0; numa_count = 1; + } else { + wd_collect_numa_nodes(ctx_params->bmp, numa_nodes, &numa_count); } for (numa_idx = 0; numa_idx < numa_count; numa_idx++) { + dparams.numa_id = numa_nodes[numa_idx]; + for (j = 0; j < mode_ctx_num; j++) { - memset(&dparams, 0, sizeof(dparams)); - dparams.ctx_mode = ctx_mode; - dparams.op_type = op_type; - dparams.numa_id = numa_nodes[numa_idx]; - dparams.bmp = ctx_params->bmp; - dparams.epoll_en = false; - ret = drv->alloc_ctx(attrs->alg, &dparams, &ctx); - if (!ctx || ret < 0) { - if (ret == -WD_ENODEV) - break; + ret = wd_alloc_one_ctx(attrs, drv, &dparams, ctx_idx); + if (ret == -WD_ENODEV) + break; + + if (ret) { WD_ERR("failed to alloc ctx %u from driver %s on numa %d!\n", *ctx_idx, drv->drv_name, numa_nodes[numa_idx]); return ret; } - - internal_config->ctxs[*ctx_idx].ctx = ctx; - internal_config->ctxs[*ctx_idx].op_type = dparams.op_type; - internal_config->ctxs[*ctx_idx].ctx_mode = dparams.ctx_mode; - internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; - internal_config->ctxs[*ctx_idx].drv = drv; - (*ctx_idx)++; } } @@ -2749,6 +2784,142 @@ err_ctxs: return ret; } +/* + * SCHED_POLICY_DEV: open all sync ctxs then all async ctxs of + * (drv, op_type, numa) on ONE device pinned by dparams->preferred_dev_path. + * If the device cannot hold the whole group, roll back every ctx opened on + * it and return -WD_ENODEV so the caller tries the next device. + */ +static int wd_alloc_dev_group(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + struct wd_drv_ctx_params *dparams, + __u32 *ctx_idx) +{ + struct wd_ctx_nums *nums = &attrs->ctx_params->ctx_set_num[dparams->op_type]; + __u32 want[2] = {nums->sync_ctx_num, nums->async_ctx_num}; + __u8 modes[2] = {CTX_MODE_SYNC, CTX_MODE_ASYNC}; + __u32 group_start = *ctx_idx; + __u32 i, j; + int ret; + + for (i = 0; i < 2; i++) { + dparams->ctx_mode = modes[i]; + for (j = 0; j < want[i]; j++) { + ret = wd_alloc_one_ctx(attrs, drv, dparams, ctx_idx); + if (ret) { + wd_free_ctxs_batch_range(attrs, group_start, *ctx_idx); + *ctx_idx = group_start; + if (ret != -WD_EBUSY && ret != -WD_ENODEV) + return ret; + return -WD_ENODEV; + } + } + } + return WD_SUCCESS; +} + +/* + * SCHED_POLICY_DEV: per (drv, op_type, numa) open all sync+async ctxs on one + * device. Enumerates devices on each numa node and pins each group to one + * device via preferred_dev_path, trying the next device if the current one + * cannot hold the whole group. All devices exhausted => error out. + */ +static int wd_alloc_ctxs_batch_dev(struct wd_init_attrs *attrs, __u32 *start_idx) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + struct wd_ctx_params *ctx_params = attrs->ctx_params; + __u32 ctx_idx, drv_idx, start, group_start; + struct uacce_dev_list *dev_list = NULL; + int numa_nodes[UADK_MAX_NUMA_NODES]; + char alg_type[CRYPTO_MAX_ALG_NAME]; + struct wd_drv_ctx_params dparams; + struct wd_alg_driver *drv; + __u8 op_type, numa_count; + struct wd_ctx_nums *nums; + struct uacce_dev_list *p; + int n, ret; + + memset(&dparams, 0, sizeof(dparams)); + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + + start = *start_idx; + ctx_idx = start; + for (drv_idx = 0; drv_idx < internal_config->drv_count; drv_idx++) { + drv = internal_config->drv_array[drv_idx]; + if (!drv || !drv->alloc_ctx) { + WD_ERR("failed to check driver %s alloc_ctx!\n", + drv ? drv->drv_name : "unknown"); + ret = -WD_EINVAL; + goto err_ctxs; + } + if (drv->calc_type != UADK_ALG_HW) + continue; + + ret = wd_get_alg_type(attrs->alg, alg_type); + if (ret) { + ret = -WD_EINVAL; + goto err_ctxs; + } + if (!strcmp(alg_type, "ecc")) + (void)strcpy(alg_type, "sm2"); + if (!strcmp(alg_type, "comp")) + (void)strcpy(alg_type, "zlib"); + + dev_list = wd_get_accel_list(alg_type); + if (!dev_list) + continue; + + wd_collect_numa_nodes(ctx_params->bmp, numa_nodes, &numa_count); + + for (op_type = 0; op_type < ctx_params->op_type_num; op_type++) { + nums = &ctx_params->ctx_set_num[op_type]; + if (!nums->sync_ctx_num && !nums->async_ctx_num) + continue; + + dparams.op_type = op_type; + group_start = ctx_idx; + for (n = 0; n < numa_count; n++) { + dparams.numa_id = numa_nodes[n]; + for (p = dev_list; p; p = p->next) { + if (!p->dev || p->dev->numa_id != numa_nodes[n]) + continue; + + /* Skip devices that cannot hold the whole group */ + if (wd_get_avail_ctx(p->dev) < (int)(nums->sync_ctx_num + + nums->async_ctx_num)) + continue; + + dparams.preferred_dev_path = p->dev->char_dev_path; + ret = wd_alloc_dev_group(attrs, drv, + &dparams, + &ctx_idx); + if (!ret) + break; + if (ret != -WD_ENODEV) + goto err_ctxs; + } + } + + if (ctx_idx == group_start) { + ret = -WD_ENODEV; + goto err_ctxs; + } + } + wd_free_list_accels(dev_list); + dev_list = NULL; + } + + *start_idx = ctx_idx; + return ctx_idx == start ? -WD_ENODEV : WD_SUCCESS; + +err_ctxs: + if (dev_list) + wd_free_list_accels(dev_list); + wd_free_ctxs_batch(attrs, ctx_idx); + return ret; +} + /** * wd_alg_ctx_uninit() - Release ctxs, scheduler, ctx_config. * @@ -2880,18 +3051,24 @@ int wd_alg_ctx_init(struct wd_init_attrs *attrs) return -WD_EINVAL; } - /* - * Ensure that contexts (ctx) with the same attributes are allocated first, - * thereby maintaining queue continuity within the contexts. - */ - ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); - if (ret) - return -WD_EINVAL; + if (attrs->sched_type == SCHED_POLICY_DEV) { + ret = wd_alloc_ctxs_batch_dev(attrs, &ctx_idx); + /* DEV tolerates partial: only a truly empty result fails. */ + if (ret == -WD_ENODEV) + return -WD_EINVAL; - /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ - ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); - if (ret) - return -WD_EINVAL; + if (ret) + return ret; + } else { + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; + + /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; + } /* Backfill actual allocated count */ internal_config = attrs->ctx_config_internal; -- 2.28.0.windows.1
From: Wenkai Lin <linwenkai6@hisilicon.com> The SEC module has duplicated BD3 fill/parse paths across cipher, digest, and aead, a stream mode scene misconfiguration for common digests, and silently swallowed computation errors in the CE driver. The following three issues are addressed: Unify the BD3 fill/parse paths for cipher, digest, and aead on V3+ hardware. Previously each algorithm duplicated udata handling, scene selection, and DIF detection in the qm_fill/qm_parse entry points. Route everything through fill_xxx_bd3/parse_xxx_bd3, add DIF and udata address helpers, select scene (STORAGE/IPSEC/STREAM) by is_storage, and guard unmap in error paths. BD1/BD2 paths are retained for V2 hardware. The common digest (such as SHA/SM3) that is not GMAC supports streaming hash processing, but SCENE_STREAM was only described for CTR/GCM streaming encryption in the FS, and the hash function for streaming mode in FS014/FS017 was ignored. Use SCENE_STREAM for common digests; GMAC uses SCENE_IPSEC independently. In sm3_ce_drv_send and isa_ce_cipher_send, the return value of the software digest/cipher computation is silently overwritten by the subsequent wd_get_sqe_from_queue call. If the computation fails, the error is swallowed and the function may return success. Add an early return on error before calling wd_get_sqe_from_queue. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- drv/isa_ce_sm3.c | 2 + drv/isa_ce_sm4.c | 2 + v1/drv/hisi_sec_udrv.c | 465 ++++++++++++++++++++++++++--------------- 3 files changed, 298 insertions(+), 171 deletions(-) diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c index b41f6c5..117b95e 100644 --- a/drv/isa_ce_sm3.c +++ b/drv/isa_ce_sm3.c @@ -374,6 +374,8 @@ static int sm3_ce_drv_send(handle_t ctx, void *digest_msg) WD_ERR("invalid digest mode!\n"); ret = -WD_EINVAL; } + if (ret) + return ret; ret = wd_get_sqe_from_queue(sfctx, msg->tag); if (ret) diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c index 863b99e..7547c00 100644 --- a/drv/isa_ce_sm4.c +++ b/drv/isa_ce_sm4.c @@ -399,6 +399,8 @@ static int isa_ce_cipher_send(handle_t ctx, void *wd_msg) ret = sm4_xts_encrypt(msg, &rkey); else ret = sm4_xts_decrypt(msg, &rkey); + if (ret) + return ret; break; default: WD_ERR("The current block cipher mode is not supported!\n"); diff --git a/v1/drv/hisi_sec_udrv.c b/v1/drv/hisi_sec_udrv.c index cf4bcd9..6ad6c57 100644 --- a/v1/drv/hisi_sec_udrv.c +++ b/v1/drv/hisi_sec_udrv.c @@ -823,11 +823,6 @@ static int fill_cipher_bd3_alg(struct wcrypto_cipher_msg *msg, struct hisi_sec_bd3_sqe *sqe) { int ret = WD_SUCCESS; - - ret = cipher_param_check(msg); - if (unlikely(ret)) - return ret; - __u8 c_key_len = 0; switch (msg->alg) { @@ -971,19 +966,72 @@ static int fill_cipher_bd3_mode(struct wcrypto_cipher_msg *msg, return ret; } +static void fill_cipher_bd3_dif(struct hisi_sec_bd3_sqe *sqe, + struct wd_sec_udata *udata) +{ + sqe->skip_data.gran_num = udata->gran_num; + sqe->skip_data.src_skip_data_len = udata->src_offset; + sqe->skip_data.dst_skip_data_len = udata->dst_offset; + + sqe->storage_scene.gen_ver_val = udata->dif.ver; + sqe->storage_scene.gen_app_val = udata->dif.app; + sqe->storage_scene.gen_page_pad_ctrl = udata->dif.ctrl.gen.page_layout_gen_type; + sqe->storage_scene.gen_grd_ctrl = udata->dif.ctrl.gen.grd_gen_type; + sqe->storage_scene.gen_ver_ctrl = udata->dif.ctrl.gen.ver_gen_type; + sqe->storage_scene.gen_app_ctrl = udata->dif.ctrl.gen.app_gen_type; + sqe->storage_scene.gen_ref_ctrl = udata->dif.ctrl.gen.ref_gen_type; + sqe->storage_scene.page_pad_type = udata->dif.ctrl.gen.page_layout_pad_type; + sqe->storage_scene.block_size = udata->block_size; + sqe->storage_scene.private_info = udata->dif.priv_info; + sqe->storage_scene.chk_grd_ctrl = udata->dif.ctrl.verify.grd_verify_type; + sqe->storage_scene.chk_ref_ctrl = udata->dif.ctrl.verify.ref_verify_type; + sqe->storage_scene.lba_l = udata->dif.lba & QM_L32BITS_MASK; + sqe->storage_scene.lba_h = udata->dif.lba >> QM_HADDR_SHIFT; +} + +static void fill_cipher_bd3_udata_addr(struct wcrypto_cipher_msg *msg, + struct hisi_sec_bd3_sqe *sqe, __u8 is_storage) +{ + uintptr_t phy; + + phy = (uintptr_t)msg->in; + sqe->data_src_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->data_src_addr_h = HI_U32(phy); + phy = (uintptr_t)msg->out; + sqe->data_dst_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->data_dst_addr_h = HI_U32(phy); + phy = (uintptr_t)msg->key; + sqe->c_key_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->c_key_addr_h = HI_U32(phy); + if (msg->iv_bytes) { + phy = (uintptr_t)msg->iv; + if (is_storage) { + sqe->skip_data.c_iv_a_key_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->skip_data.c_iv_a_key_h = HI_U32(phy); + } else { + sqe->ipsec_scene.c_ivin_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->ipsec_scene.c_ivin_addr_h = HI_U32(phy); + } + } +} + static int fill_cipher_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, struct wcrypto_cipher_msg *msg, struct wcrypto_cipher_tag *tag) { + struct wd_sec_udata *udata = tag ? tag->priv : NULL; + __u8 is_storage = udata && udata->gran_num != 0; int ret; + if (!is_storage) { + ret = cipher_param_check(msg); + if (unlikely(ret)) + return ret; + } + sqe->type = BD_TYPE3; - sqe->scene = SCENE_IPSEC; + sqe->scene = is_storage ? SCENE_STORAGE : SCENE_IPSEC; sqe->de = DATA_DST_ADDR_ENABLE; - if (msg->in_bytes > MAX_CIPHER_LENGTH) { - WD_ERR("input data is too large.\n"); - return -WD_EINVAL; - } sqe->c_len = msg->in_bytes; fill_bd3_addr_type(msg->data_fmt, sqe); @@ -1000,10 +1048,19 @@ static int fill_cipher_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, return ret; } - ret = fill_cipher_bd3_area(q, msg, sqe); - if (ret != WD_SUCCESS) { - WD_ERR("fail to fill_cipher_bd3_addr.\n"); - return ret; + if (is_storage && msg->mode == WCRYPTO_CIPHER_XTS) + sqe->ci_gen = CI_GEN_BY_LBA; + + if (udata) { + fill_cipher_bd3_udata_addr(msg, sqe, is_storage); + if (is_storage) + fill_cipher_bd3_dif(sqe, udata); + } else { + ret = fill_cipher_bd3_area(q, msg, sqe); + if (ret != WD_SUCCESS) { + WD_ERR("fail to fill_cipher_bd3_addr.\n"); + return ret; + } } if (tag) @@ -1129,10 +1186,8 @@ int qm_fill_cipher_bd3_sqe(void *message, struct qm_queue_info *info, __u16 i) { struct wcrypto_cipher_msg *msg = message; struct wcrypto_cipher_tag *tag = (void *)(uintptr_t)msg->usr_data; - struct wd_sec_udata *udata = tag->priv; struct wd_queue *q = info->q; struct hisi_sec_bd3_sqe *sqe3; - struct hisi_sec_sqe *sqe; uintptr_t temp; int ret; @@ -1144,35 +1199,17 @@ int qm_fill_cipher_bd3_sqe(void *message, struct qm_queue_info *info, __u16 i) } temp = (uintptr_t)info->sq_base + i * info->sqe_size; + sqe3 = (struct hisi_sec_bd3_sqe *)temp; + memset(sqe3, 0, sizeof(struct hisi_sec_bd3_sqe)); - /* - * For user self-defined data with DIF scence, will fill BD1. - * For user self-defined data without DIF scence, will fill BD2. - * For non user self-defined data scence, will fill BD3. - */ - if (udata) { - sqe = (struct hisi_sec_sqe *)temp; - memset(sqe, 0, sizeof(struct hisi_sec_sqe)); - if (udata->gran_num != 0) - ret = fill_cipher_bd1(q, sqe, msg, tag); - else - ret = fill_cipher_bd2(q, sqe, msg, tag); - } else { - sqe3 = (struct hisi_sec_bd3_sqe *)temp; - memset(sqe3, 0, sizeof(struct hisi_sec_bd3_sqe)); - ret = fill_cipher_bd3(q, sqe3, msg, tag); - } - + ret = fill_cipher_bd3(q, sqe3, msg, tag); if (ret != WD_SUCCESS) return ret; info->req_cache[i] = msg; #ifdef DEBUG_LOG - if (udata) - sec_dump_bd((unsigned char *)sqe, SQE_BYTES_NUMS); - else - sec_dump_bd((unsigned char *)sqe3, SQE_BYTES_NUMS); + sec_dump_bd((unsigned char *)sqe3, SQE_BYTES_NUMS); #endif return ret; @@ -1707,24 +1744,45 @@ static int digest_param_check_v3(struct wcrypto_digest_msg *msg) return WD_SUCCESS; } -static int fill_digest_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, - struct wcrypto_digest_msg *msg, struct wcrypto_digest_tag *tag) +static void fill_digest_bd3_udata_addr(struct wcrypto_digest_msg *msg, + struct hisi_sec_bd3_sqe *sqe) { uintptr_t phy; - int ret; - ret = digest_param_check_v3(msg); - if (unlikely(ret)) - return ret; + phy = (uintptr_t)msg->in; + sqe->data_src_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->data_src_addr_h = HI_U32(phy); + phy = (uintptr_t)msg->out; + sqe->mac_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->mac_addr_h = HI_U32(phy); - sqe->type = BD_TYPE3; - if (msg->alg == WCRYPTO_AES_GMAC) - sqe->scene = SCENE_IPSEC; - else - sqe->scene = SCENE_STREAM; + if (msg->mode == WCRYPTO_DIGEST_HMAC) { + sqe->a_key_len = msg->key_bytes / SEC_SQE_LEN_RATE; + phy = (uintptr_t)msg->key; + sqe->auth_key_iv.a_key_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->auth_key_iv.a_key_addr_h = HI_U32(phy); + } +} + +static void fill_digest_bd3_dif(struct hisi_sec_bd3_sqe *sqe, + struct wd_sec_udata *udata) +{ + sqe->skip_data.gran_num = udata->gran_num; + sqe->skip_data.src_skip_data_len = udata->src_offset; + sqe->storage_scene.block_size = udata->block_size; + sqe->storage_scene.private_info = udata->dif.priv_info; + sqe->storage_scene.chk_grd_ctrl = udata->dif.ctrl.verify.grd_verify_type; + sqe->storage_scene.chk_ref_ctrl = udata->dif.ctrl.verify.ref_verify_type; + sqe->storage_scene.lba_l = udata->dif.lba & QM_L32BITS_MASK; + sqe->storage_scene.lba_h = udata->dif.lba >> QM_HADDR_SHIFT; +} + +static int fill_digest_bd3_map(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, + struct wcrypto_digest_msg *msg) +{ + uintptr_t phy; + int ret; - sqe->auth = AUTH_MAC_CALCULATE; - sqe->a_len = msg->in_bytes; phy = (uintptr_t)drv_iova_map(q, msg->in, msg->in_bytes); if (unlikely(!phy)) { WD_ERR("Get message in dma address fail!\n"); @@ -1740,33 +1798,84 @@ static int fill_digest_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, WD_ERR("Get digest bd3 message out dma address fail!\n"); goto map_out_error; } + + ret = set_hmac_mode_v3(msg, sqe, q); + if (unlikely(ret)) + goto unmap_out; + + return WD_SUCCESS; + +unmap_out: + unmap_addr(q, msg->out, msg->out_bytes, sqe->mac_addr_l, + sqe->mac_addr_h, msg->data_fmt); +map_out_error: + phy = DMA_ADDR(sqe->data_src_addr_h, sqe->data_src_addr_l); + drv_iova_unmap(q, msg->in, (void *)(uintptr_t)phy, msg->in_bytes); + return ret; +} + +static int fill_digest_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, + struct wcrypto_digest_msg *msg, struct wcrypto_digest_tag *tag) +{ + struct wd_sec_udata *udata = tag ? tag->priv : NULL; + __u8 is_storage = udata && udata->gran_num != 0; + uintptr_t phy; + int ret; + + ret = digest_param_check_v3(msg); + if (unlikely(ret)) + return ret; + + sqe->type = BD_TYPE3; + + /* + * GMAC is an IPSEC-specific auth algo (RFC4543); other digests use + * stream scene for long-data hash processing. + */ + if (is_storage) + sqe->scene = SCENE_STORAGE; + else if (msg->alg == WCRYPTO_AES_GMAC) + sqe->scene = SCENE_IPSEC; + else + sqe->scene = SCENE_STREAM; + + sqe->auth = AUTH_MAC_CALCULATE; + sqe->a_len = msg->in_bytes; + + if (udata) { + fill_digest_bd3_udata_addr(msg, sqe); + if (is_storage) + fill_digest_bd3_dif(sqe, udata); + } else { + ret = fill_digest_bd3_map(q, sqe, msg); + if (unlikely(ret)) + return ret; + } + sqe->mac_len = msg->out_bytes / WORD_BYTES; ret = fill_digest_bd3_alg(msg, sqe); if (ret != WD_SUCCESS) { WD_ERR("fill_digest_bd3_alg fail!\n"); - goto map_alg_error; + goto unmap; } ret = qm_fill_digest_long_bd3(msg, sqe); if (ret) - goto map_alg_error; - - ret = set_hmac_mode_v3(msg, sqe, q); - if (ret) - goto map_alg_error; + goto unmap; if (tag) sqe->tag_l = tag->wcrypto_tag.ctx_id; return ret; -map_alg_error: - unmap_addr(q, msg->out, msg->out_bytes, sqe->mac_addr_l, - sqe->mac_addr_h, msg->data_fmt); -map_out_error: - phy = DMA_ADDR(sqe->data_src_addr_h, sqe->data_src_addr_l); - drv_iova_unmap(q, msg->in, (void *)(uintptr_t)phy, msg->in_bytes); +unmap: + if (!udata) { + unmap_addr(q, msg->out, msg->out_bytes, sqe->mac_addr_l, + sqe->mac_addr_h, msg->data_fmt); + phy = DMA_ADDR(sqe->data_src_addr_h, sqe->data_src_addr_l); + drv_iova_unmap(q, msg->in, (void *)(uintptr_t)phy, msg->in_bytes); + } return ret; } @@ -1776,23 +1885,15 @@ int qm_fill_digest_bd3_sqe(void *message, struct qm_queue_info *info, __u16 i) struct wcrypto_digest_tag *tag = (void *)(uintptr_t)msg->usr_data; struct wd_queue *q = info->q; struct hisi_sec_bd3_sqe *sqe; - struct hisi_sec_sqe *sqe2; uintptr_t temp; int ret; temp = (uintptr_t)info->sq_base + i * info->sqe_size; + sqe = (struct hisi_sec_bd3_sqe *)temp; + memset(sqe, 0, sizeof(struct hisi_sec_bd3_sqe)); + fill_bd3_addr_type(msg->data_fmt, sqe); - if (tag->priv) { - sqe2 = (struct hisi_sec_sqe *)temp; - memset(sqe2, 0, sizeof(struct hisi_sec_sqe)); - fill_bd_addr_type(msg->data_fmt, sqe2); - ret = fill_digest_bd_udata(q, sqe2, msg, tag); - } else { - sqe = (struct hisi_sec_bd3_sqe *)temp; - memset(sqe, 0, sizeof(struct hisi_sec_bd3_sqe)); - fill_bd3_addr_type(msg->data_fmt, sqe); - ret = fill_digest_bd3(q, sqe, msg, tag); - } + ret = fill_digest_bd3(q, sqe, msg, tag); if (ret != WD_SUCCESS) return ret; @@ -1890,7 +1991,7 @@ static void parse_cipher_bd2(struct wd_queue *q, struct hisi_sec_sqe *sqe, /* In user self-define data case, may not need addr map, just return */ tag = (void *)(uintptr_t)cipher_msg->usr_data; - if (tag->priv) + if (tag && tag->priv) return; dma_addr = DMA_ADDR(sqe->type2.data_src_addr_h, @@ -1919,16 +2020,24 @@ static void parse_cipher_bd2(struct wd_queue *q, struct hisi_sec_sqe *sqe, static void parse_cipher_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, struct wcrypto_cipher_msg *cipher_msg) { + struct wcrypto_cipher_tag *tag; __u64 dma_addr; if (sqe->done != SEC_HW_TASK_DONE || sqe->error_type) { WD_ERR("Fail to parse SEC BD3 %s, done=0x%x, etype=0x%x\n", "cipher", (__u32)sqe->done, (__u32)sqe->error_type); cipher_msg->result = WD_IN_EPARA; + } else if (sqe->dc == DIF_VERIFY_FAIL) { + cipher_msg->result = WD_VERIFY_ERR; } else { cipher_msg->result = WD_SUCCESS; } + /* In user self-define data case, may not need addr map, just return */ + tag = (void *)(uintptr_t)cipher_msg->usr_data; + if (tag && tag->priv) + return; + dma_addr = DMA_ADDR(sqe->data_src_addr_h, sqe->data_src_addr_l); drv_iova_unmap(q, cipher_msg->in, (void *)(uintptr_t)dma_addr, cipher_msg->in_bytes); @@ -1990,36 +2099,15 @@ int qm_parse_cipher_bd3_sqe(void *msg, const struct qm_queue_info *info, struct wcrypto_cipher_msg *cipher_msg = info->req_cache[i]; struct hisi_sec_bd3_sqe *sqe3 = msg; struct wd_queue *q = info->q; - struct hisi_sec_sqe *sqe; if (unlikely(!cipher_msg)) { WD_ERR("info->req_cache is null at index:%hu\n", i); return 0; } + if (unlikely(usr && sqe3->tag_l != usr)) + return 0; - switch (sqe3->type) { - case BD_TYPE3: - if (unlikely(usr && sqe3->tag_l != usr)) - return 0; - parse_cipher_bd3(q, sqe3, cipher_msg); - break; - case BD_TYPE2: - sqe = (struct hisi_sec_sqe *)sqe3; - if (usr && sqe->type2.tag != usr) - return 0; - parse_cipher_bd2(q, sqe, cipher_msg); - break; - case BD_TYPE1: - sqe = (struct hisi_sec_sqe *)sqe3; - if (usr && sqe->type1.tag != usr) - return 0; - parse_cipher_bd1(q, sqe, cipher_msg); - break; - default: - WD_ERR("SEC BD Type error\n"); - cipher_msg->result = WD_IN_EPARA; - break; - } + parse_cipher_bd3(q, sqe3, cipher_msg); #ifdef DEBUG_LOG sec_dump_bd((unsigned char *)msg, SQE_BYTES_NUMS); @@ -2068,7 +2156,7 @@ static void parse_digest_bd2(struct wd_queue *q, struct hisi_sec_sqe *sqe, digest_msg->result = WD_SUCCESS; tag = (void *)(uintptr_t)digest_msg->usr_data; - if (tag->priv) + if (tag && tag->priv) return; dma_addr = DMA_ADDR(sqe->type2.data_src_addr_h, @@ -2674,16 +2762,80 @@ out: return ret; } +static int fill_aead_bd3_udata_addr(struct wcrypto_aead_msg *msg, + struct hisi_sec_bd3_sqe *sqe, struct wd_aead_udata *udata) +{ + uintptr_t phy; + + sqe->auth_src_offset = udata->src_offset; + sqe->cipher_src_offset = udata->src_offset + msg->assoc_bytes; + + phy = (uintptr_t)msg->in; + sqe->data_src_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->data_src_addr_h = HI_U32(phy); + phy = (uintptr_t)msg->out; + sqe->data_dst_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->data_dst_addr_h = HI_U32(phy); + phy = (uintptr_t)msg->iv; + sqe->ipsec_scene.c_ivin_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->ipsec_scene.c_ivin_addr_h = HI_U32(phy); + + phy = (uintptr_t)udata->ckey; + sqe->c_key_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->c_key_addr_h = HI_U32(phy); + phy = (uintptr_t)udata->mac; + sqe->mac_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->mac_addr_h = HI_U32(phy); + if (msg->cmode == WCRYPTO_CIPHER_CCM || msg->cmode == WCRYPTO_CIPHER_GCM) { + if (unlikely(!udata->aiv)) { + WD_ERR("invalid aead udata: aiv is NULL!\n"); + return -WD_EINVAL; + } + phy = (uintptr_t)udata->aiv; + sqe->auth_key_iv.a_ivin_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->auth_key_iv.a_ivin_addr_h = HI_U32(phy); + } + + return WD_SUCCESS; +} + +static int init_msg_with_udata(struct wcrypto_aead_msg *req, struct wd_aead_udata *udata) +{ + if (!udata->ckey || !udata->mac) { + WD_ERR("invalid udata para!\n"); + return -WD_EINVAL; + } + + if (req->cmode == WCRYPTO_CIPHER_CCM || req->cmode == WCRYPTO_CIPHER_GCM) { + req->ckey_bytes = udata->ckey_bytes; + req->auth_bytes = udata->mac_bytes; + } else { + WD_ERR("invalid cmode para!\n"); + return -WD_EINVAL; + } + + return WD_SUCCESS; +} + static int fill_aead_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, struct wcrypto_aead_msg *msg, struct wcrypto_aead_tag *tag) { + struct wd_aead_udata *udata = tag ? tag->priv : NULL; int ret; + if (udata) { + ret = init_msg_with_udata(msg, udata); + if (ret != WD_SUCCESS) + return ret; + } + sqe->type = BD_TYPE3; sqe->scene = SCENE_IPSEC; sqe->de = DATA_DST_ADDR_ENABLE; sqe->c_len = msg->in_bytes; - sqe->cipher_src_offset = msg->assoc_bytes; + sqe->cipher_src_offset = udata ? udata->src_offset + msg->assoc_bytes + : msg->assoc_bytes; + sqe->auth_src_offset = udata ? udata->src_offset : 0; sqe->a_len = msg->in_bytes + msg->assoc_bytes; ret = fill_aead_bd3_alg(msg, sqe); @@ -2698,15 +2850,21 @@ static int fill_aead_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, return ret; } - ret = fill_aead_bd3_addr(q, msg, sqe); - if (ret != WD_SUCCESS) { - WD_ERR("fail to fill_aead_bd3_addr!\n"); - return ret; - } + if (udata) { + ret = fill_aead_bd3_udata_addr(msg, sqe, udata); + if (unlikely(ret)) + return ret; + } else { + ret = fill_aead_bd3_addr(q, msg, sqe); + if (ret != WD_SUCCESS) { + WD_ERR("fail to fill_aead_bd3_addr!\n"); + return ret; + } - ret = fill_aead_stream_bd3(q, msg, sqe); - if (unlikely(ret)) - return ret; + ret = fill_aead_stream_bd3(q, msg, sqe); + if (unlikely(ret)) + return ret; + } if (tag) sqe->tag_l = tag->wcrypto_tag.ctx_id; @@ -2777,7 +2935,6 @@ int qm_fill_aead_bd3_sqe(void *message, struct qm_queue_info *info, __u16 i) struct wcrypto_aead_tag *tag = (void *)(uintptr_t)msg->usr_data; struct wd_queue *q = info->q; struct hisi_sec_bd3_sqe *sqe; - struct hisi_sec_sqe *sqe2; uintptr_t temp; int ret; @@ -2789,17 +2946,11 @@ int qm_fill_aead_bd3_sqe(void *message, struct qm_queue_info *info, __u16 i) } temp = (uintptr_t)info->sq_base + i * info->sqe_size; - if (tag->priv) { - sqe2 = (struct hisi_sec_sqe *)temp; - memset(sqe2, 0, sizeof(struct hisi_sec_sqe)); - fill_bd_addr_type(msg->data_fmt, sqe2); - ret = fill_aead_bd_udata(q, sqe2, msg, tag); - } else { - sqe = (struct hisi_sec_bd3_sqe *)temp; - memset(sqe, 0, sizeof(struct hisi_sec_bd3_sqe)); - fill_bd3_addr_type(msg->data_fmt, sqe); - ret = fill_aead_bd3(q, sqe, msg, tag); - } + sqe = (struct hisi_sec_bd3_sqe *)temp; + memset(sqe, 0, sizeof(struct hisi_sec_bd3_sqe)); + fill_bd3_addr_type(msg->data_fmt, sqe); + + ret = fill_aead_bd3(q, sqe, msg, tag); if (ret != WD_SUCCESS) return ret; @@ -2815,6 +2966,7 @@ int qm_fill_aead_bd3_sqe(void *message, struct qm_queue_info *info, __u16 i) static void parse_aead_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe3, struct wcrypto_aead_msg *msg) { + struct wcrypto_aead_tag *tag; __u8 mac[AEAD_IV_MAX_BYTES] = { 0 }; __u64 dma_addr; int ret; @@ -2828,6 +2980,11 @@ static void parse_aead_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe3, msg->result = WD_SUCCESS; } + /* In user self-define data case, may not need addr map, just return */ + tag = (void *)(uintptr_t)msg->usr_data; + if (tag && tag->priv) + return; + /* * We obtain a memory from IV SGL as a temporary address space for MAC， * After the encryption is completed, copy the data from this temporary @@ -2883,26 +3040,16 @@ int qm_parse_aead_bd3_sqe(void *msg, const struct qm_queue_info *info, { struct wcrypto_aead_msg *aead_msg = info->req_cache[i]; struct hisi_sec_bd3_sqe *sqe = msg; - struct hisi_sec_sqe *sqe2 = msg; struct wd_queue *q = info->q; if (unlikely(!aead_msg)) { WD_ERR("info->req_cache is null at index:%hu\n", i); return 0; } + if (unlikely(usr && sqe->tag_l != usr)) + return 0; - if (sqe->type == BD_TYPE3) { - if (usr && sqe->tag_l != usr) - return 0; - parse_aead_bd3(q, sqe, aead_msg); - } else if (sqe->type == BD_TYPE2) { - if (usr && sqe2->type2.tag != usr) - return 0; - parse_aead_bd2(q, sqe2, aead_msg); - } else { - WD_ERR("SEC BD Type error\n"); - aead_msg->result = WD_IN_EPARA; - } + parse_aead_bd3(q, sqe, aead_msg); #ifdef DEBUG_LOG sec_dump_bd((unsigned char *)msg, SQE_BYTES_NUMS); @@ -2914,16 +3061,24 @@ int qm_parse_aead_bd3_sqe(void *msg, const struct qm_queue_info *info, static void parse_digest_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, struct wcrypto_digest_msg *digest_msg) { + struct wcrypto_digest_tag *tag; __u64 dma_addr; if (sqe->done != SEC_HW_TASK_DONE || sqe->error_type) { WD_ERR("SEC BD3 %s fail!done=0x%x, etype=0x%x\n", "digest", (__u32)sqe->done, (__u32)sqe->error_type); digest_msg->result = WD_IN_EPARA; + } else if (sqe->dc == DIF_VERIFY_FAIL) { + digest_msg->result = WD_VERIFY_ERR; } else { digest_msg->result = WD_SUCCESS; } + /* In user self-define data case, may not need addr map, just return */ + tag = (void *)(uintptr_t)digest_msg->usr_data; + if (tag && tag->priv) + return; + dma_addr = DMA_ADDR(sqe->data_src_addr_h, sqe->data_src_addr_l); drv_iova_unmap(q, digest_msg->in, (void *)(uintptr_t)dma_addr, digest_msg->in_bytes); @@ -2946,30 +3101,16 @@ int qm_parse_digest_bd3_sqe(void *msg, const struct qm_queue_info *info, { struct wcrypto_digest_msg *digest_msg = info->req_cache[i]; struct hisi_sec_bd3_sqe *sqe = msg; - struct hisi_sec_sqe *sqe2 = msg; struct wd_queue *q = info->q; if (unlikely(!digest_msg)) { WD_ERR("info->req_cache is null at index:%hu\n", i); return 0; } + if (unlikely(usr && sqe->tag_l != usr)) + return 0; - if (sqe->type == BD_TYPE3) { - if (usr && sqe->tag_l != usr) - return 0; - parse_digest_bd3(q, sqe, digest_msg); - } else if (sqe->type == BD_TYPE2) { - if (usr && sqe2->type2.tag != usr) - return 0; - parse_digest_bd2(q, sqe2, digest_msg); - } else if (sqe->type == BD_TYPE1) { - if (usr && sqe2->type1.tag != usr) - return 0; - parse_digest_bd1(q, sqe2, digest_msg); - } else { - WD_ERR("SEC Digest BD Type error\n"); - digest_msg->result = WD_IN_EPARA; - } + parse_digest_bd3(q, sqe, digest_msg); #ifdef DEBUG_LOG sec_dump_bd((unsigned char *)msg, SQE_BYTES_NUMS); @@ -3332,24 +3473,6 @@ static int fill_aead_bd2(struct wd_queue *q, struct hisi_sec_sqe *sqe, return fill_aead_bd2_addr(q, msg, sqe); } -static int init_msg_with_udata(struct wcrypto_aead_msg *req, struct wd_aead_udata *udata) -{ - if (!udata->ckey || !udata->mac) { - WD_ERR("invalid udata para!\n"); - return -WD_EINVAL; - } - - if (req->cmode == WCRYPTO_CIPHER_CCM || req->cmode == WCRYPTO_CIPHER_GCM) { - req->ckey_bytes = udata->ckey_bytes; - req->auth_bytes = udata->mac_bytes; - } else { - WD_ERR("invalid cmode para!\n"); - return -WD_EINVAL; - } - - return WD_SUCCESS; -} - static int fill_aead_bd2_udata(struct wd_queue *q, struct hisi_sec_sqe *sqe, struct wcrypto_aead_msg *msg, struct wcrypto_aead_tag *tag) { @@ -3433,7 +3556,7 @@ static void parse_aead_bd2(struct wd_queue *q, struct hisi_sec_sqe *sqe, } tag = (void *)(uintptr_t)msg->usr_data; - if (tag->priv) + if (tag && tag->priv) return; /* -- 2.28.0.windows.1
From: Weili Qian <qianweili@huawei.com> Two error-path bugs cause undefined behavior on mmap memory and allow invalid column sizes to reach hardware DMA. In wd_rsa.c create_sess_key, prikey is allocated via sess->mm_ops.alloc which may be backed by mmap (WD_MMAP_MEM), but the pubkey alloc failure path freed it with free(), causing undefined behavior on mmap memory. Use sess->mm_ops.free to match the allocator and the normal release path in del_sess_key. wd_alg_init_fallback in wd_util.c discarded fb_driver->init() return value and always returned 0, and wd_agg_rehash_sync_inner in wd_agg.c ignored wd_agg_set_col_size return value, allowing invalid column sizes to hardware DMA. Propagate the init return value and check wd_agg_set_col_size so callers can detect failures. Signed-off-by: Weili Qian <qianweili@huawei.com> --- wd_agg.c | 4 +++- wd_rsa.c | 2 +- wd_util.c | 4 +--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/wd_agg.c b/wd_agg.c index 2c7825d..ed4ba74 100644 --- a/wd_agg.c +++ b/wd_agg.c @@ -1500,7 +1500,9 @@ static int wd_agg_rehash_sync_inner(struct wd_agg_sess *sess, struct wd_agg_req } out_req->real_out_row_count = msg.out_row_count; - wd_agg_set_col_size(sess, in_req, out_req->real_out_row_count); + ret = wd_agg_set_col_size(sess, in_req, out_req->real_out_row_count); + if (unlikely(ret)) + return ret; in_req->in_row_count = out_req->real_out_row_count; fill_request_msg_input(&in_msg, in_req, sess, true); diff --git a/wd_rsa.c b/wd_rsa.c index c1f2d9a..59c230d 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -930,7 +930,7 @@ static int create_sess_key(struct wd_rsa_sess_setup *setup, (int)GEN_PARAMS_SZ(sess->key_size); sess->pubkey = sess->mm_ops.alloc(sess->mm_ops.usr, len); if (!sess->pubkey) { - free(sess->prikey); + sess->mm_ops.free(sess->mm_ops.usr, sess->prikey); WD_ERR("failed to alloc sess pubkey!\n"); return -WD_ENOMEM; } diff --git a/wd_util.c b/wd_util.c index 0d482f9..b5eaa7d 100644 --- a/wd_util.c +++ b/wd_util.c @@ -1566,9 +1566,7 @@ static int wd_alg_init_fallback(struct wd_alg_driver *fb_driver) return -WD_EINVAL; } - fb_driver->init(NULL, NULL); - - return 0; + return fb_driver->init(NULL, NULL); } static void wd_alg_uninit_fallback(struct wd_alg_driver *fb_driver) -- 2.28.0.windows.1
From: Weili Qian <qianweili@huawei.com> Multiple memory safety bugs across the driver and algorithm layers cause heap overflow, out-of-bounds reads, use-after-free, resource leaks, and uninitialized spinlocks that can lead to crashes and data corruption. The following five issues are addressed: wd_parse_log_level in wd.c allocated st_size bytes but fscanf "%[^\n ]" appends a '\0', causing off-by-one heap overflow when a token is exactly st_size long. _find_next_bit in wd_mempool.c used "start > bits" in the loop guard while the initial guard used "start >= bits", reading map out of bounds when bits is a multiple of BITS_PER_LONG. Allocate st_size + 1 and use ">=" for the loop guard. pthread_spin_init returns a positive errno on failure, but wd_blockpool_create/wd_mempool_create checked "if (ret < 0)" which was always false, and wd_soft_alloc_ctx did not check the return value at all, leaving spinlocks uninitialized. Use "if (ret)" to detect failure, check both spin_init calls in wd_soft_alloc_ctx, clean up on failure, and return -ret so callers can detect the error. hisi_qm_create_sglpool missed NULL check on hisi_qm_align_sgl and set sgl_num to i instead of i + 1 on failure, leaking the already-allocated sgl[i]. hisi_qm_get_hw_sgl, hisi_sec_cipher_send/_v3, cipher/digest/ aead recv and fill_zip_comp_sqe also leaked SGL/DMA resources or passed NULL to hardware on error paths. Add NULL checks, fix sgl_num, and recycle SGL entries on error paths. wd_*_poll_ctx in wd_ecc/rsa/dh/udma.c put recv_msg.tag (0 from zero-initialized recv_msg) back into the message pool on drv->recv failure, marking slot 0 as free even when in use and causing double-use or use-after-free. sm2_enc_parse in hisi_hpre.c leaked the second SQE and its DMA mappings when parse_second_sqe partially succeeded. Remove the wd_put_msg_to_pool call on recv failure and free the second SQE before jumping to free_first. wd_pool_init in wd_bmm.c allocated blk_array with malloc, leaving elements uninitialized and passing garbage blk/blk_dma to wd_iova_unmap on the error cleanup path. wd_comp_init_nolock in wd_comp.c called wd_alg_init_driver redundantly since the caller already invokes it, causing double initialization. Use calloc to zero-initialize blk_array and remove the redundant wd_alg_init_driver call. Signed-off-by: Weili Qian <qianweili@huawei.com> --- drv/hisi_comp.c | 7 ++++++- drv/hisi_hpre.c | 2 ++ drv/hisi_qm_udrv.c | 15 +++++++++++++-- drv/hisi_sec.c | 34 ++++++++++++++++++++-------------- drv/wd_drv.c | 16 ++++++++++++++-- wd.c | 3 ++- wd_bmm.c | 2 +- wd_comp.c | 6 ------ wd_dh.c | 2 -- wd_ecc.c | 2 -- wd_mempool.c | 6 +++--- wd_rsa.c | 2 -- wd_udma.c | 2 -- 13 files changed, 61 insertions(+), 38 deletions(-) diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index b798811..c1b83c3 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -1555,8 +1555,13 @@ static int fill_zip_comp_sqe(struct hisi_qp *qp, struct wd_comp_msg *msg, ops[alg_type].fill_alg(sqe); ret = ops[alg_type].fill_comp_level(sqe, msg->comp_lv); - if (unlikely(ret)) + if (unlikely(ret)) { + if (msg->mm_ops && !msg->mm_ops->sva_mode) + zip_mem_unmap(msg, sqe); + if (msg->req.data_fmt == WD_SGL_BUF) + free_hw_sgl((handle_t)qp, &msg->c_sgl, msg->mm_ops); return ret; + } ops[alg_type].fill_tag(sqe, msg->tag); diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c index 70abe92..6d61fa4 100644 --- a/drv/hisi_hpre.c +++ b/drv/hisi_hpre.c @@ -2713,6 +2713,8 @@ static int sm2_enc_parse(handle_t h_qp, struct wd_ecc_msg *msg, ret = parse_second_sqe(h_qp, msg, &second); if (unlikely(ret)) { WD_ERR("failed to parse second BD, ret = %d!\n", ret); + if (second) + free_req(second); goto free_first; } diff --git a/drv/hisi_qm_udrv.c b/drv/hisi_qm_udrv.c index 6da9887..8617b10 100644 --- a/drv/hisi_qm_udrv.c +++ b/drv/hisi_qm_udrv.c @@ -803,6 +803,10 @@ handle_t hisi_qm_create_sglpool(__u32 sgl_num, __u32 sge_num, struct wd_mm_ops * sgl_pool->sgl_align[i] = hisi_qm_align_sgl(sgl_pool->sgl[i], sge_num, sgl_pool->mm_ops); + if (!sgl_pool->sgl_align[i]) { + sgl_pool->sgl_num = i + 1; + goto err_out; + } } sgl_pool->sgl_num = sgl_num; @@ -999,11 +1003,18 @@ void *hisi_qm_get_hw_sgl(handle_t sgl_pool, struct wd_datalist *sgl) ret = WD_ERR_PTR(-WD_EBUSY); goto err_out; } - if (mm_ops) + if (mm_ops) { cur->next_dma = (uintptr_t)mm_ops->iova_map(mm_ops->usr, next, sizeof(*next)); - else + if (!cur->next_dma) { + WD_ERR("invalid: iova_map for sgl chain failed!\n"); + hisi_qm_sgl_push(pool, next); + ret = WD_ERR_PTR(-WD_ENOMEM); + goto err_out; + } + } else { cur->next_dma = (uintptr_t)next; + } cur->next = next; cur = next; head->entry_sum_in_chain += pool->sge_num; diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 10bb34e..df086d7 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -1439,22 +1439,25 @@ static int hisi_sec_cipher_send(handle_t ctx, void *wd_msg) ret = fill_cipher_bd2_addr(msg, &sqe); if (ret < 0) { WD_ERR("cipher map memory is err(%d)!\n", ret); - return ret; + goto put_sgl; } ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (ret < 0) { if (ret != -WD_EBUSY) WD_ERR("cipher send sqe is err(%d)!\n", ret); - - if (msg->data_fmt == WD_SGL_BUF) - hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, - msg->out, msg->mm_ops); - destroy_cipher_bd2_addr(msg, &sqe); - return ret; + goto destroy_addr; } return 0; + +destroy_addr: + destroy_cipher_bd2_addr(msg, &sqe); +put_sgl: + if (msg->data_fmt == WD_SGL_BUF) + hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, + msg->out, msg->mm_ops); + return ret; } int hisi_sec_cipher_recv(handle_t ctx, void *wd_msg) @@ -1735,22 +1738,25 @@ static int hisi_sec_cipher_send_v3(handle_t ctx, void *wd_msg) ret = fill_cipher_bd3_addr(msg, &sqe); if (ret < 0) { WD_ERR("cipher map memory is err(%d)!\n", ret); - return ret; + goto put_sgl; } ret = hisi_qm_send(h_qp, &sqe, 1, &count); if (ret < 0) { if (ret != -WD_EBUSY) WD_ERR("cipher send sqe is err(%d)!\n", ret); - - if (msg->data_fmt == WD_SGL_BUF) - hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, - msg->out, msg->mm_ops); - destroy_cipher_bd3_addr(msg, &sqe); - return ret; + goto destroy_addr; } return 0; + +destroy_addr: + destroy_cipher_bd3_addr(msg, &sqe); +put_sgl: + if (msg->data_fmt == WD_SGL_BUF) + hisi_sec_put_sgl(h_qp, msg->alg_type, msg->in, + msg->out, msg->mm_ops); + return ret; } static void parse_cipher_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, diff --git a/drv/wd_drv.c b/drv/wd_drv.c index 08014dd..4bac551 100644 --- a/drv/wd_drv.c +++ b/drv/wd_drv.c @@ -11,6 +11,7 @@ int wd_soft_alloc_ctx(char *alg_name, void *params, handle_t *ctx) { struct wd_soft_ctx *sfctx; + int ret; if (!params || !ctx) { WD_ERR("invalid: params, or ctx is NULL!\n"); @@ -27,8 +28,19 @@ int wd_soft_alloc_ctx(char *alg_name, void *params, handle_t *ctx) /* Initialize as software context */ sfctx->fd = -1; sfctx->ctx_type = UADK_ALG_SOFT; - pthread_spin_init(&sfctx->slock, PTHREAD_PROCESS_PRIVATE); - pthread_spin_init(&sfctx->rlock, PTHREAD_PROCESS_PRIVATE); + ret = pthread_spin_init(&sfctx->slock, PTHREAD_PROCESS_PRIVATE); + if (ret) { + WD_ERR("failed to init slock!\n"); + free(sfctx); + return -ret; + } + ret = pthread_spin_init(&sfctx->rlock, PTHREAD_PROCESS_PRIVATE); + if (ret) { + WD_ERR("failed to init rlock!\n"); + pthread_spin_destroy(&sfctx->slock); + free(sfctx); + return -ret; + } /* Return context handle */ *ctx = (handle_t)sfctx; diff --git a/wd.c b/wd.c index f973c70..bd0a3aa 100644 --- a/wd.c +++ b/wd.c @@ -114,11 +114,12 @@ static void wd_parse_log_level(void) goto close_file; } - file_contents = malloc(file_info.st_size); + file_contents = malloc(file_info.st_size + 1); if (!file_contents) { WD_ERR("failed to get file contents memory.\n"); goto close_file; } + file_contents[file_info.st_size] = '\0'; while (fscanf(in_file, " %[^\n ] ", file_contents) != EOF) { if (!strcmp("local5.debug", file_contents)) diff --git a/wd_bmm.c b/wd_bmm.c index 811afa2..e393e59 100644 --- a/wd_bmm.c +++ b/wd_bmm.c @@ -707,7 +707,7 @@ static int wd_pool_init(struct wd_blkpool *p) } /* Allocate block array */ - p->blk_array = (struct wd_blk_hd *)malloc(act_num * sizeof(struct wd_blk_hd)); + p->blk_array = (struct wd_blk_hd *)calloc(act_num, sizeof(struct wd_blk_hd)); if (!p->blk_array) { WD_ERR("Failed to allocate block array.\n"); return -WD_ENOMEM; diff --git a/wd_comp.c b/wd_comp.c index 35af141..523a189 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -156,14 +156,8 @@ static int wd_comp_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_comp_setting.config); - if (ret) - goto out_clear_pool; - return 0; -out_clear_pool: - wd_uninit_async_request_pool(&wd_comp_setting.pool); out_clear_sched: wd_clear_sched(&wd_comp_setting.sched); out_clear_ctx_config: diff --git a/wd_dh.c b/wd_dh.c index 8026fd6..f9958b2 100644 --- a/wd_dh.c +++ b/wd_dh.c @@ -520,8 +520,6 @@ int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count) } else if (unlikely(ret)) { WD_ERR("failed to async recv, ret = %d!\n", ret); *count = rcv_cnt; - wd_put_msg_to_pool(&wd_dh_setting.pool, idx, - rcv_msg.tag); return ret; } rcv_cnt++; diff --git a/wd_ecc.c b/wd_ecc.c index 4b141c4..679a3fb 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -2519,8 +2519,6 @@ int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count) } else if (ret < 0) { WD_ERR("failed to async recv, ret = %d!\n", ret); *count = rcv_cnt; - wd_put_msg_to_pool(&wd_ecc_setting.pool, idx, - recv_msg.tag); return ret; } rcv_cnt++; diff --git a/wd_mempool.c b/wd_mempool.c index 3ce9ef0..4081130 100644 --- a/wd_mempool.c +++ b/wd_mempool.c @@ -239,7 +239,7 @@ static unsigned long _find_next_bit(unsigned long *map, unsigned long bits, while (!tmp) { start += BITS_PER_LONG; - if (start > bits) + if (start >= bits) return bits; tmp = map[start / BITS_PER_LONG]; @@ -561,7 +561,7 @@ handle_t wd_blockpool_create(handle_t mempool, size_t block_size, bp->free_block_num = block_num; bp->mp = mp; ret = pthread_spin_init(&bp->lock, PTHREAD_PROCESS_PRIVATE); - if (ret < 0) + if (ret) goto err_free_bp; ret = alloc_mem_from_mempool(mp, bp); @@ -910,7 +910,7 @@ handle_t wd_mempool_create(size_t size, int node) mp->size = tmp; mp->blk_size = WD_MEMPOOL_BLOCK_SIZE; ret = pthread_spin_init(&mp->lock, PTHREAD_PROCESS_PRIVATE); - if (ret < 0) + if (ret) goto free_pool; ret = alloc_mem_from_hugepage(mp); diff --git a/wd_rsa.c b/wd_rsa.c index 59c230d..539285d 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -583,8 +583,6 @@ int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count) return ret; } else if (ret < 0) { WD_ERR("failed to async recv, ret = %d!\n", ret); - wd_put_msg_to_pool(&wd_rsa_setting.pool, idx, - recv_msg.tag); return ret; } rcv_cnt++; diff --git a/wd_udma.c b/wd_udma.c index 59841bb..b92e75b 100644 --- a/wd_udma.c +++ b/wd_udma.c @@ -337,8 +337,6 @@ static int wd_udma_poll_ctx(__u32 idx, __u32 expt, __u32 *count) } else if (unlikely(ret)) { WD_ERR("failed to async recv, ret = %d!\n", ret); *count = rcv_cnt; - wd_put_msg_to_pool(&wd_udma_setting.pool, idx, - rcv_msg.tag); return ret; } rcv_cnt++; -- 2.28.0.windows.1
participants (1)
-
Chenghai Huang