Under SCHED_POLICY_DEV, allocating sync then async ctxs in two passes let concurrent contention split them across devices, leaving the async domain empty (async picks returned 65535, sends failed with -22). Add a DEV-only path that opens sync/async ctxs interleaved on ONE device: the first ctx auto-selects via NUMA scan, its dev_path pins every later ctx through preferred_dev_path, and wd_hw_alloc_ctx opens that device without NUMA fallback. Other policies unchanged. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> --- drv/wd_drv.c | 26 +++++ include/wd_alg.h | 3 + wd_util.c | 283 ++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 261 insertions(+), 51 deletions(-) diff --git a/drv/wd_drv.c b/drv/wd_drv.c index 381cee1..c839ed4 100644 --- a/drv/wd_drv.c +++ b/drv/wd_drv.c @@ -220,6 +220,32 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) goto out; } + /* + * Preferred-device binding (SCHED_POLICY_DEV): open exactly this device + * and fail strictly if unavailable, so the sync and async ctxs of one + * (driver, op_type, numa) stay on the same device. No NUMA fallback. + */ + if (ctx_params->preferred_dev_path) { + for (curr = used_list; curr; curr = curr->next) { + if (curr->dev && !strcmp(curr->dev->char_dev_path, + ctx_params->preferred_dev_path)) + break; + } + + if (!curr) { + ret = -WD_ENODEV; + goto out; + } + + hctx = wd_request_ctx(curr->dev); + if (!hctx) { + ret = -WD_EBUSY; + goto out; + } + + goto success; + } + curr = used_list; while (curr) { if (curr->dev) { diff --git a/include/wd_alg.h b/include/wd_alg.h index b256a2f..3cc4e43 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -96,6 +96,8 @@ enum alg_drv_type { * @ctx_mode: CTX_MODE_SYNC or CTX_MODE_ASYNC * @op_type: Operation type * @bmp: NUMA node bitmask (optional, NULL if not needed) + * @preferred_dev_path: Preferred device char_dev_path; NULL/empty = auto-select, + * non-empty = open only this device and fail strictly if unavailable */ struct wd_drv_ctx_params { __u8 ctx_mode; @@ -103,6 +105,7 @@ struct wd_drv_ctx_params { int numa_id; bool epoll_en; struct bitmask *bmp; + const char *preferred_dev_path; }; /** diff --git a/wd_util.c b/wd_util.c index 1b4c74e..c0173a3 100644 --- a/wd_util.c +++ b/wd_util.c @@ -2613,17 +2613,17 @@ static int wd_alg_sched_instance(struct wd_sched *sched, return WD_SUCCESS; } -static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, - __u32 allocated_count) +static void wd_free_ctxs_batch_range(struct wd_init_attrs *attrs, + __u32 start_idx, __u32 end_idx) { struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_alg_driver *drv; __u32 i; - if (!internal_config || !internal_config->ctxs || !allocated_count) + if (!internal_config || !internal_config->ctxs || start_idx >= end_idx) return; - for (i = 0; i < allocated_count; i++) { + for (i = start_idx; i < end_idx; i++) { if (!internal_config->ctxs[i].ctx) continue; @@ -2635,21 +2635,76 @@ static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, } } +static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, + __u32 allocated_count) +{ + wd_free_ctxs_batch_range(attrs, 0, allocated_count); +} + +static void wd_collect_numa_nodes(struct bitmask *bmp, int *numa_nodes, + __u8 *numa_count) +{ + int max_node, n; + + max_node = numa_max_node() + 1; + if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) + max_node = UADK_MAX_NUMA_NODES; + + *numa_count = 0; + + for (n = 0; n < max_node; n++) { + if (numa_bitmask_isbitset(bmp, n)) + numa_nodes[(*numa_count)++] = n; + } + + if (!*numa_count) { + numa_nodes[0] = 0; + *numa_count = 1; + } +} + +static int wd_alloc_one_ctx(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + struct wd_drv_ctx_params *dparams, + __u32 *ctx_idx) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + handle_t ctx; + int ret; + + ret = drv->alloc_ctx(attrs->alg, dparams, &ctx); + if (ret < 0) + return ret; + + if (!ctx) + return -WD_EINVAL; + + internal_config->ctxs[*ctx_idx].ctx = ctx; + internal_config->ctxs[*ctx_idx].op_type = dparams->op_type; + internal_config->ctxs[*ctx_idx].ctx_mode = dparams->ctx_mode; + internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; + internal_config->ctxs[*ctx_idx].drv = drv; + (*ctx_idx)++; + return WD_SUCCESS; +} + static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, struct wd_alg_driver *drv, __u8 ctx_mode, __u8 op_type, __u32 *ctx_idx) { - struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_ctx_params *ctx_params = attrs->ctx_params; - struct wd_drv_ctx_params dparams; + struct wd_drv_ctx_params dparams = {0}; int numa_nodes[UADK_MAX_NUMA_NODES]; - __u32 mode_ctx_num, numa_count = 0; - __u32 numa_idx, j; - int max_node, n; - handle_t ctx; + __u32 mode_ctx_num, numa_idx, j; + __u8 numa_count; int ret; + dparams.ctx_mode = ctx_mode; + dparams.op_type = op_type; + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + if (ctx_mode == CTX_MODE_SYNC) mode_ctx_num = ctx_params->ctx_set_num[op_type].sync_ctx_num; else @@ -2661,46 +2716,26 @@ static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, attrs->sched_type == SCHED_POLICY_SINGLE) && mode_ctx_num > 1) mode_ctx_num = 1; - if (drv->calc_type == UADK_ALG_HW) { - max_node = numa_max_node() + 1; - if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) - max_node = UADK_MAX_NUMA_NODES; - for (n = 0; n < max_node; n++) { - if (numa_bitmask_isbitset(ctx_params->bmp, n)) - numa_nodes[numa_count++] = n; - } - } else { - numa_nodes[0] = 0; - numa_count = 1; - } - if (!numa_count) { + if (drv->calc_type != UADK_ALG_HW) { numa_nodes[0] = 0; numa_count = 1; + } else { + wd_collect_numa_nodes(ctx_params->bmp, numa_nodes, &numa_count); } for (numa_idx = 0; numa_idx < numa_count; numa_idx++) { + dparams.numa_id = numa_nodes[numa_idx]; + for (j = 0; j < mode_ctx_num; j++) { - memset(&dparams, 0, sizeof(dparams)); - dparams.ctx_mode = ctx_mode; - dparams.op_type = op_type; - dparams.numa_id = numa_nodes[numa_idx]; - dparams.bmp = ctx_params->bmp; - dparams.epoll_en = false; - ret = drv->alloc_ctx(attrs->alg, &dparams, &ctx); - if (!ctx || ret < 0) { - if (ret == -WD_ENODEV) - break; + ret = wd_alloc_one_ctx(attrs, drv, &dparams, ctx_idx); + if (ret == -WD_ENODEV) + break; + + if (ret) { WD_ERR("failed to alloc ctx %u from driver %s on numa %d!\n", *ctx_idx, drv->drv_name, numa_nodes[numa_idx]); return ret; } - - internal_config->ctxs[*ctx_idx].ctx = ctx; - internal_config->ctxs[*ctx_idx].op_type = dparams.op_type; - internal_config->ctxs[*ctx_idx].ctx_mode = dparams.ctx_mode; - internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; - internal_config->ctxs[*ctx_idx].drv = drv; - (*ctx_idx)++; } } @@ -2749,6 +2784,146 @@ err_ctxs: return ret; } +/* + * SCHED_POLICY_DEV: open sync/async ctxs of (drv, op_type, numa) interleaved on + * ONE device. The caller pins the device via dparams->preferred_dev_path. + * Stop when both counts are met or the device is full. If both modes are + * requested but only one could be opened, roll back and return -WD_ENODEV so + * the caller can try the next device; partial within one mode is kept. + */ +static int wd_alloc_dev_group(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + struct wd_drv_ctx_params *dparams, + __u32 *ctx_idx) +{ + struct wd_ctx_nums *nums = &attrs->ctx_params->ctx_set_num[dparams->op_type]; + __u32 want[2] = {nums->sync_ctx_num, nums->async_ctx_num}; + __u8 modes[2] = {CTX_MODE_SYNC, CTX_MODE_ASYNC}; + __u32 group_start = *ctx_idx; + __u32 done[2] = {0, 0}; + __u32 i; + int ret; + + while (done[0] < want[0] || done[1] < want[1]) { + for (i = 0; i < 2; i++) { + if (done[i] >= want[i]) + continue; + + dparams->ctx_mode = modes[i]; + ret = wd_alloc_one_ctx(attrs, drv, dparams, ctx_idx); + if (ret) { + if (ret != -WD_EBUSY && ret != -WD_ENODEV) + return ret; + /* + * Device full. If both modes were requested + * but only one got ctxs, the empty domain + * would cause idx 65535 at runtime. Roll back + * this group so the caller tries another dev. + */ + if (want[0] && want[1] && + (!done[0] || !done[1])) { + wd_free_ctxs_batch_range(attrs, group_start, *ctx_idx); + *ctx_idx = group_start; + return -WD_ENODEV; + } + return WD_SUCCESS; + } + done[i]++; + } + } + return WD_SUCCESS; +} + +/* + * SCHED_POLICY_DEV: per (drv, op_type, numa) open sync/async on one device + * (interleaved, partial allowed). Enumerates devices on each numa node and + * pins each group to one device via preferred_dev_path, trying the next + * device if the current one cannot hold both modes. + */ +static int wd_alloc_ctxs_batch_dev(struct wd_init_attrs *attrs, __u32 *start_idx) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + struct wd_ctx_params *ctx_params = attrs->ctx_params; + struct uacce_dev_list *dev_list, *p; + int numa_nodes[UADK_MAX_NUMA_NODES]; + char alg_type[CRYPTO_MAX_ALG_NAME]; + struct wd_drv_ctx_params dparams; + __u32 ctx_idx, drv_idx, start; + struct wd_alg_driver *drv; + __u8 op_type, numa_count; + struct wd_ctx_nums *nums; + int n, ret; + + memset(&dparams, 0, sizeof(dparams)); + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + + start = *start_idx; + ctx_idx = start; + for (drv_idx = 0; drv_idx < internal_config->drv_count; drv_idx++) { + drv = internal_config->drv_array[drv_idx]; + if (!drv || !drv->alloc_ctx) { + WD_ERR("failed to check driver %s alloc_ctx!\n", + drv ? drv->drv_name : "unknown"); + ret = -WD_EINVAL; + goto err_ctxs; + } + if (drv->calc_type != UADK_ALG_HW) + continue; + + ret = wd_get_alg_type(attrs->alg, alg_type); + if (ret) { + ret = -WD_EINVAL; + goto err_ctxs; + } + if (!strcmp(alg_type, "ecc")) + (void)strcpy(alg_type, "sm2"); + if (!strcmp(alg_type, "comp")) + (void)strcpy(alg_type, "zlib"); + + dev_list = wd_get_accel_list(alg_type); + if (!dev_list) + continue; + + wd_collect_numa_nodes(ctx_params->bmp, numa_nodes, &numa_count); + + for (op_type = 0; op_type < ctx_params->op_type_num; op_type++) { + nums = &ctx_params->ctx_set_num[op_type]; + if (!nums->sync_ctx_num && !nums->async_ctx_num) + continue; + + dparams.op_type = op_type; + for (n = 0; n < numa_count; n++) { + dparams.numa_id = numa_nodes[n]; + for (p = dev_list; p; p = p->next) { + if (!p->dev || p->dev->numa_id != numa_nodes[n]) + continue; + + dparams.preferred_dev_path = p->dev->char_dev_path; + ret = wd_alloc_dev_group(attrs, drv, + &dparams, + &ctx_idx); + if (!ret) + break; + if (ret != -WD_ENODEV) + goto err_ctxs; + } + } + } + wd_free_list_accels(dev_list); + dev_list = NULL; + } + + *start_idx = ctx_idx; + return ctx_idx == start ? -WD_ENODEV : WD_SUCCESS; + +err_ctxs: + if (dev_list) + wd_free_list_accels(dev_list); + wd_free_ctxs_batch(attrs, ctx_idx); + return ret; +} + /** * wd_alg_ctx_uninit() - Release ctxs, scheduler, ctx_config. * @@ -2880,18 +3055,24 @@ int wd_alg_ctx_init(struct wd_init_attrs *attrs) return -WD_EINVAL; } - /* - * Ensure that contexts (ctx) with the same attributes are allocated first, - * thereby maintaining queue continuity within the contexts. - */ - ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); - if (ret) - return -WD_EINVAL; + if (attrs->sched_type == SCHED_POLICY_DEV) { + ret = wd_alloc_ctxs_batch_dev(attrs, &ctx_idx); + /* DEV tolerates partial: only a truly empty result fails. */ + if (ret == -WD_ENODEV) + return -WD_EINVAL; - /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ - ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); - if (ret) - return -WD_EINVAL; + if (ret) + return ret; + } else { + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; + + /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; + } /* Backfill actual allocated count */ internal_config = attrs->ctx_config_internal; -- 2.50.1.windows.1