The scheduler ctx allocation under SCHED_POLICY_DEV and session initialization in several modules contain bugs that cause async process hangs, device starvation, and nondeterministic scheduling behavior. The following four issues are addressed: Under SCHED_POLICY_DEV, allocating sync ctxs then async ctxs in two passes let concurrent contention split them across devices. The DEV scheduler pins the region to ctxs[0]'s device and forbids cross-region fallback, so the async domain ended empty: async picks returned idx 65535, sends failed with -22, and the async process hung. Add a DEV-only path that opens sync/async ctxs interleaved on ONE device, with a preferred_dev_path field to pin later ctxs of the group. Then simplify it to a linear two-pass per device: open all sync then all async; any shortfall rolls back the whole group and tries the next device. wd_hw_alloc_ctx picked the first device whose wd_request_ctx() succeeded. When PF and VFs share the same NUMA node, concurrent businesses all bind to the same leading device, starving the others. Select the device with the most available contexts (wd_get_avail_ctx) instead, with fallback to remaining devices if the chosen one fails to open. wd_agg_alloc_sess, wd_join_gather_alloc_sess and wd_udma_alloc_sess declared a local struct wd_sched_params without initialization, setting only alg_name and ctxs before passing it to wd_sched_set_param, which unconditionally reads pkt_size, data_mode and prio_mode. The uninitialized stack values caused nondeterministic scheduling behavior. Initialize the struct with = {0} at declaration. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Co-developed-by: Wenkai Lin <linwenkai6@hisilicon.com> Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> Co-developed-by: Weili Qian <qianweili@huawei.com> Signed-off-by: Weili Qian <qianweili@huawei.com> --- drv/wd_drv.c | 67 ++++++++++-- include/wd_alg.h | 3 + wd_agg.c | 2 +- wd_join_gather.c | 2 +- wd_udma.c | 2 +- wd_util.c | 279 ++++++++++++++++++++++++++++++++++++++--------- 6 files changed, 291 insertions(+), 64 deletions(-) diff --git a/drv/wd_drv.c b/drv/wd_drv.c index 381cee1..08014dd 100644 --- a/drv/wd_drv.c +++ b/drv/wd_drv.c @@ -181,12 +181,12 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) { struct wd_drv_ctx_params *ctx_params = (struct wd_drv_ctx_params *)params; struct uacce_dev_list *dev_list, *used_list = NULL; + struct uacce_dev_list *curr, *best = NULL; char alg_type[CRYPTO_MAX_ALG_NAME]; - struct uacce_dev_list *curr; + int target_numa, best_avail = 0; struct wd_ctx_h *ctx_h; - int target_numa; + int ret, avail; handle_t hctx; - int ret; if (!params || !ctx) { WD_ERR("invalid: parameters are NULL!\n"); @@ -220,14 +220,61 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) goto out; } - curr = used_list; - while (curr) { - if (curr->dev) { - hctx = wd_request_ctx(curr->dev); - if (hctx) - goto success; + /* + * Preferred-device binding (SCHED_POLICY_DEV): open exactly this device + * and fail strictly if unavailable, so the sync and async ctxs of one + * (driver, op_type, numa) stay on the same device. No NUMA fallback. + */ + if (ctx_params->preferred_dev_path) { + for (curr = used_list; curr; curr = curr->next) { + if (curr->dev && !strcmp(curr->dev->char_dev_path, + ctx_params->preferred_dev_path)) + break; } - curr = curr->next; + + if (!curr) { + ret = -WD_ENODEV; + goto out; + } + + hctx = wd_request_ctx(curr->dev); + if (!hctx) { + ret = -WD_EBUSY; + goto out; + } + + goto success; + } + + /* Pick the device with the most available contexts to balance load. */ + for (curr = used_list; curr; curr = curr->next) { + if (!curr->dev) + continue; + avail = wd_get_avail_ctx(curr->dev); + if (avail > best_avail) { + best_avail = avail; + best = curr; + } + } + + if (!best) { + WD_ERR("failed to request ctx on NUMA node %d for %s\n", + target_numa, alg_name); + ret = -WD_EBUSY; + goto out; + } + + hctx = wd_request_ctx(best->dev); + if (hctx) + goto success; + + /* Fall back to remaining devices if the best one fails to open */ + for (curr = used_list; curr; curr = curr->next) { + if (curr == best || !curr->dev) + continue; + hctx = wd_request_ctx(curr->dev); + if (hctx) + goto success; } WD_ERR("failed to request ctx on NUMA node %d for %s\n", diff --git a/include/wd_alg.h b/include/wd_alg.h index b256a2f..3cc4e43 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -96,6 +96,8 @@ enum alg_drv_type { * @ctx_mode: CTX_MODE_SYNC or CTX_MODE_ASYNC * @op_type: Operation type * @bmp: NUMA node bitmask (optional, NULL if not needed) + * @preferred_dev_path: Preferred device char_dev_path; NULL/empty = auto-select, + * non-empty = open only this device and fail strictly if unavailable */ struct wd_drv_ctx_params { __u8 ctx_mode; @@ -103,6 +105,7 @@ struct wd_drv_ctx_params { int numa_id; bool epoll_en; struct bitmask *bmp; + const char *preferred_dev_path; }; /** diff --git a/wd_agg.c b/wd_agg.c index 95cb869..2c7825d 100644 --- a/wd_agg.c +++ b/wd_agg.c @@ -446,7 +446,7 @@ static int wd_agg_uninit_sess_priv(struct wd_agg_sess *sess) handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) { - struct wd_sched_params params; + struct wd_sched_params params = {0}; struct wd_agg_sess *sess; __u32 out_agg_cols_num = 0; int ret; diff --git a/wd_join_gather.c b/wd_join_gather.c index e43dbf9..27472d3 100644 --- a/wd_join_gather.c +++ b/wd_join_gather.c @@ -533,7 +533,7 @@ static int wd_join_gather_init_sess(struct wd_join_gather_sess *sess, handle_t wd_join_gather_alloc_sess(struct wd_join_gather_sess_setup *setup) { - struct wd_sched_params params; + struct wd_sched_params params = {0}; struct wd_join_gather_sess *sess; int ret; diff --git a/wd_udma.c b/wd_udma.c index 582c8d1..59841bb 100644 --- a/wd_udma.c +++ b/wd_udma.c @@ -78,7 +78,7 @@ void wd_udma_free_sess(handle_t sess) handle_t wd_udma_alloc_sess(struct wd_udma_sess_setup *setup) { - struct wd_sched_params params; + struct wd_sched_params params = {0}; struct wd_udma_sess *sess; int ret; diff --git a/wd_util.c b/wd_util.c index 1b4c74e..0d482f9 100644 --- a/wd_util.c +++ b/wd_util.c @@ -2613,17 +2613,17 @@ static int wd_alg_sched_instance(struct wd_sched *sched, return WD_SUCCESS; } -static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, - __u32 allocated_count) +static void wd_free_ctxs_batch_range(struct wd_init_attrs *attrs, + __u32 start_idx, __u32 end_idx) { struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_alg_driver *drv; __u32 i; - if (!internal_config || !internal_config->ctxs || !allocated_count) + if (!internal_config || !internal_config->ctxs || start_idx >= end_idx) return; - for (i = 0; i < allocated_count; i++) { + for (i = start_idx; i < end_idx; i++) { if (!internal_config->ctxs[i].ctx) continue; @@ -2635,21 +2635,76 @@ static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, } } +static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, + __u32 allocated_count) +{ + wd_free_ctxs_batch_range(attrs, 0, allocated_count); +} + +static void wd_collect_numa_nodes(struct bitmask *bmp, int *numa_nodes, + __u8 *numa_count) +{ + int max_node, n; + + max_node = numa_max_node() + 1; + if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) + max_node = UADK_MAX_NUMA_NODES; + + *numa_count = 0; + + for (n = 0; n < max_node; n++) { + if (numa_bitmask_isbitset(bmp, n)) + numa_nodes[(*numa_count)++] = n; + } + + if (!*numa_count) { + numa_nodes[0] = 0; + *numa_count = 1; + } +} + +static int wd_alloc_one_ctx(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + struct wd_drv_ctx_params *dparams, + __u32 *ctx_idx) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + handle_t ctx; + int ret; + + ret = drv->alloc_ctx(attrs->alg, dparams, &ctx); + if (ret < 0) + return ret; + + if (!ctx) + return -WD_EINVAL; + + internal_config->ctxs[*ctx_idx].ctx = ctx; + internal_config->ctxs[*ctx_idx].op_type = dparams->op_type; + internal_config->ctxs[*ctx_idx].ctx_mode = dparams->ctx_mode; + internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; + internal_config->ctxs[*ctx_idx].drv = drv; + (*ctx_idx)++; + return WD_SUCCESS; +} + static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, struct wd_alg_driver *drv, __u8 ctx_mode, __u8 op_type, __u32 *ctx_idx) { - struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_ctx_params *ctx_params = attrs->ctx_params; - struct wd_drv_ctx_params dparams; + struct wd_drv_ctx_params dparams = {0}; int numa_nodes[UADK_MAX_NUMA_NODES]; - __u32 mode_ctx_num, numa_count = 0; - __u32 numa_idx, j; - int max_node, n; - handle_t ctx; + __u32 mode_ctx_num, numa_idx, j; + __u8 numa_count; int ret; + dparams.ctx_mode = ctx_mode; + dparams.op_type = op_type; + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + if (ctx_mode == CTX_MODE_SYNC) mode_ctx_num = ctx_params->ctx_set_num[op_type].sync_ctx_num; else @@ -2661,46 +2716,26 @@ static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, attrs->sched_type == SCHED_POLICY_SINGLE) && mode_ctx_num > 1) mode_ctx_num = 1; - if (drv->calc_type == UADK_ALG_HW) { - max_node = numa_max_node() + 1; - if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) - max_node = UADK_MAX_NUMA_NODES; - for (n = 0; n < max_node; n++) { - if (numa_bitmask_isbitset(ctx_params->bmp, n)) - numa_nodes[numa_count++] = n; - } - } else { - numa_nodes[0] = 0; - numa_count = 1; - } - if (!numa_count) { + if (drv->calc_type != UADK_ALG_HW) { numa_nodes[0] = 0; numa_count = 1; + } else { + wd_collect_numa_nodes(ctx_params->bmp, numa_nodes, &numa_count); } for (numa_idx = 0; numa_idx < numa_count; numa_idx++) { + dparams.numa_id = numa_nodes[numa_idx]; + for (j = 0; j < mode_ctx_num; j++) { - memset(&dparams, 0, sizeof(dparams)); - dparams.ctx_mode = ctx_mode; - dparams.op_type = op_type; - dparams.numa_id = numa_nodes[numa_idx]; - dparams.bmp = ctx_params->bmp; - dparams.epoll_en = false; - ret = drv->alloc_ctx(attrs->alg, &dparams, &ctx); - if (!ctx || ret < 0) { - if (ret == -WD_ENODEV) - break; + ret = wd_alloc_one_ctx(attrs, drv, &dparams, ctx_idx); + if (ret == -WD_ENODEV) + break; + + if (ret) { WD_ERR("failed to alloc ctx %u from driver %s on numa %d!\n", *ctx_idx, drv->drv_name, numa_nodes[numa_idx]); return ret; } - - internal_config->ctxs[*ctx_idx].ctx = ctx; - internal_config->ctxs[*ctx_idx].op_type = dparams.op_type; - internal_config->ctxs[*ctx_idx].ctx_mode = dparams.ctx_mode; - internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; - internal_config->ctxs[*ctx_idx].drv = drv; - (*ctx_idx)++; } } @@ -2749,6 +2784,142 @@ err_ctxs: return ret; } +/* + * SCHED_POLICY_DEV: open all sync ctxs then all async ctxs of + * (drv, op_type, numa) on ONE device pinned by dparams->preferred_dev_path. + * If the device cannot hold the whole group, roll back every ctx opened on + * it and return -WD_ENODEV so the caller tries the next device. + */ +static int wd_alloc_dev_group(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + struct wd_drv_ctx_params *dparams, + __u32 *ctx_idx) +{ + struct wd_ctx_nums *nums = &attrs->ctx_params->ctx_set_num[dparams->op_type]; + __u32 want[2] = {nums->sync_ctx_num, nums->async_ctx_num}; + __u8 modes[2] = {CTX_MODE_SYNC, CTX_MODE_ASYNC}; + __u32 group_start = *ctx_idx; + __u32 i, j; + int ret; + + for (i = 0; i < 2; i++) { + dparams->ctx_mode = modes[i]; + for (j = 0; j < want[i]; j++) { + ret = wd_alloc_one_ctx(attrs, drv, dparams, ctx_idx); + if (ret) { + wd_free_ctxs_batch_range(attrs, group_start, *ctx_idx); + *ctx_idx = group_start; + if (ret != -WD_EBUSY && ret != -WD_ENODEV) + return ret; + return -WD_ENODEV; + } + } + } + return WD_SUCCESS; +} + +/* + * SCHED_POLICY_DEV: per (drv, op_type, numa) open all sync+async ctxs on one + * device. Enumerates devices on each numa node and pins each group to one + * device via preferred_dev_path, trying the next device if the current one + * cannot hold the whole group. All devices exhausted => error out. + */ +static int wd_alloc_ctxs_batch_dev(struct wd_init_attrs *attrs, __u32 *start_idx) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + struct wd_ctx_params *ctx_params = attrs->ctx_params; + __u32 ctx_idx, drv_idx, start, group_start; + struct uacce_dev_list *dev_list = NULL; + int numa_nodes[UADK_MAX_NUMA_NODES]; + char alg_type[CRYPTO_MAX_ALG_NAME]; + struct wd_drv_ctx_params dparams; + struct wd_alg_driver *drv; + __u8 op_type, numa_count; + struct wd_ctx_nums *nums; + struct uacce_dev_list *p; + int n, ret; + + memset(&dparams, 0, sizeof(dparams)); + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + + start = *start_idx; + ctx_idx = start; + for (drv_idx = 0; drv_idx < internal_config->drv_count; drv_idx++) { + drv = internal_config->drv_array[drv_idx]; + if (!drv || !drv->alloc_ctx) { + WD_ERR("failed to check driver %s alloc_ctx!\n", + drv ? drv->drv_name : "unknown"); + ret = -WD_EINVAL; + goto err_ctxs; + } + if (drv->calc_type != UADK_ALG_HW) + continue; + + ret = wd_get_alg_type(attrs->alg, alg_type); + if (ret) { + ret = -WD_EINVAL; + goto err_ctxs; + } + if (!strcmp(alg_type, "ecc")) + (void)strcpy(alg_type, "sm2"); + if (!strcmp(alg_type, "comp")) + (void)strcpy(alg_type, "zlib"); + + dev_list = wd_get_accel_list(alg_type); + if (!dev_list) + continue; + + wd_collect_numa_nodes(ctx_params->bmp, numa_nodes, &numa_count); + + for (op_type = 0; op_type < ctx_params->op_type_num; op_type++) { + nums = &ctx_params->ctx_set_num[op_type]; + if (!nums->sync_ctx_num && !nums->async_ctx_num) + continue; + + dparams.op_type = op_type; + group_start = ctx_idx; + for (n = 0; n < numa_count; n++) { + dparams.numa_id = numa_nodes[n]; + for (p = dev_list; p; p = p->next) { + if (!p->dev || p->dev->numa_id != numa_nodes[n]) + continue; + + /* Skip devices that cannot hold the whole group */ + if (wd_get_avail_ctx(p->dev) < (int)(nums->sync_ctx_num + + nums->async_ctx_num)) + continue; + + dparams.preferred_dev_path = p->dev->char_dev_path; + ret = wd_alloc_dev_group(attrs, drv, + &dparams, + &ctx_idx); + if (!ret) + break; + if (ret != -WD_ENODEV) + goto err_ctxs; + } + } + + if (ctx_idx == group_start) { + ret = -WD_ENODEV; + goto err_ctxs; + } + } + wd_free_list_accels(dev_list); + dev_list = NULL; + } + + *start_idx = ctx_idx; + return ctx_idx == start ? -WD_ENODEV : WD_SUCCESS; + +err_ctxs: + if (dev_list) + wd_free_list_accels(dev_list); + wd_free_ctxs_batch(attrs, ctx_idx); + return ret; +} + /** * wd_alg_ctx_uninit() - Release ctxs, scheduler, ctx_config. * @@ -2880,18 +3051,24 @@ int wd_alg_ctx_init(struct wd_init_attrs *attrs) return -WD_EINVAL; } - /* - * Ensure that contexts (ctx) with the same attributes are allocated first, - * thereby maintaining queue continuity within the contexts. - */ - ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); - if (ret) - return -WD_EINVAL; + if (attrs->sched_type == SCHED_POLICY_DEV) { + ret = wd_alloc_ctxs_batch_dev(attrs, &ctx_idx); + /* DEV tolerates partial: only a truly empty result fails. */ + if (ret == -WD_ENODEV) + return -WD_EINVAL; - /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ - ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); - if (ret) - return -WD_EINVAL; + if (ret) + return ret; + } else { + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; + + /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; + } /* Backfill actual allocated count */ internal_config = attrs->ctx_config_internal; -- 2.28.0.windows.1