- Acc - mailweb.openeuler.org

[PATCH] uadk: bind sync/async ctx to same device under SCHED_POLICY_DEV
by Chenghai Huang 16 Oct '26

16 Oct '26

Under SCHED_POLICY_DEV, allocating sync then async ctxs in two passes let concurrent contention split them across devices, leaving the async domain empty (async picks returned 65535, sends failed with -22). Add a DEV-only path that opens sync/async ctxs interleaved on ONE device: the first ctx auto-selects via NUMA scan, its dev_path pins every later ctx through preferred_dev_path, and wd_hw_alloc_ctx opens that device without NUMA fallback. Other policies unchanged. Signed-off-by: Chenghai Huang <huangchenghai2(a)huawei.com> --- drv/wd_drv.c | 26 +++++ include/wd_alg.h | 3 + wd_util.c | 283 ++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 261 insertions(+), 51 deletions(-) diff --git a/drv/wd_drv.c b/drv/wd_drv.c index 381cee1..c839ed4 100644 --- a/drv/wd_drv.c +++ b/drv/wd_drv.c @@ -220,6 +220,32 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) goto out; } + /* + * Preferred-device binding (SCHED_POLICY_DEV): open exactly this device + * and fail strictly if unavailable, so the sync and async ctxs of one + * (driver, op_type, numa) stay on the same device. No NUMA fallback. + */ + if (ctx_params->preferred_dev_path) { + for (curr = used_list; curr; curr = curr->next) { + if (curr->dev && !strcmp(curr->dev->char_dev_path, + ctx_params->preferred_dev_path)) + break; + } + + if (!curr) { + ret = -WD_ENODEV; + goto out; + } + + hctx = wd_request_ctx(curr->dev); + if (!hctx) { + ret = -WD_EBUSY; + goto out; + } + + goto success; + } + curr = used_list; while (curr) { if (curr->dev) { diff --git a/include/wd_alg.h b/include/wd_alg.h index b256a2f..3cc4e43 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -96,6 +96,8 @@ enum alg_drv_type { * @ctx_mode: CTX_MODE_SYNC or CTX_MODE_ASYNC * @op_type: Operation type * @bmp: NUMA node bitmask (optional, NULL if not needed) + * @preferred_dev_path: Preferred device char_dev_path; NULL/empty = auto-select, + * non-empty = open only this device and fail strictly if unavailable */ struct wd_drv_ctx_params { __u8 ctx_mode; @@ -103,6 +105,7 @@ struct wd_drv_ctx_params { int numa_id; bool epoll_en; struct bitmask *bmp; + const char *preferred_dev_path; }; /** diff --git a/wd_util.c b/wd_util.c index 1b4c74e..c0173a3 100644 --- a/wd_util.c +++ b/wd_util.c @@ -2613,17 +2613,17 @@ static int wd_alg_sched_instance(struct wd_sched *sched, return WD_SUCCESS; } -static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, - __u32 allocated_count) +static void wd_free_ctxs_batch_range(struct wd_init_attrs *attrs, + __u32 start_idx, __u32 end_idx) { struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_alg_driver *drv; __u32 i; - if (!internal_config || !internal_config->ctxs || !allocated_count) + if (!internal_config || !internal_config->ctxs || start_idx >= end_idx) return; - for (i = 0; i < allocated_count; i++) { + for (i = start_idx; i < end_idx; i++) { if (!internal_config->ctxs[i].ctx) continue; @@ -2635,21 +2635,76 @@ static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, } } +static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, + __u32 allocated_count) +{ + wd_free_ctxs_batch_range(attrs, 0, allocated_count); +} + +static void wd_collect_numa_nodes(struct bitmask *bmp, int *numa_nodes, + __u8 *numa_count) +{ + int max_node, n; + + max_node = numa_max_node() + 1; + if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) + max_node = UADK_MAX_NUMA_NODES; + + *numa_count = 0; + + for (n = 0; n < max_node; n++) { + if (numa_bitmask_isbitset(bmp, n)) + numa_nodes[(*numa_count)++] = n; + } + + if (!*numa_count) { + numa_nodes[0] = 0; + *numa_count = 1; + } +} + +static int wd_alloc_one_ctx(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + struct wd_drv_ctx_params *dparams, + __u32 *ctx_idx) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + handle_t ctx; + int ret; + + ret = drv->alloc_ctx(attrs->alg, dparams, &ctx); + if (ret < 0) + return ret; + + if (!ctx) + return -WD_EINVAL; + + internal_config->ctxs[*ctx_idx].ctx = ctx; + internal_config->ctxs[*ctx_idx].op_type = dparams->op_type; + internal_config->ctxs[*ctx_idx].ctx_mode = dparams->ctx_mode; + internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; + internal_config->ctxs[*ctx_idx].drv = drv; + (*ctx_idx)++; + return WD_SUCCESS; +} + static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, struct wd_alg_driver *drv, __u8 ctx_mode, __u8 op_type, __u32 *ctx_idx) { - struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_ctx_params *ctx_params = attrs->ctx_params; - struct wd_drv_ctx_params dparams; + struct wd_drv_ctx_params dparams = {0}; int numa_nodes[UADK_MAX_NUMA_NODES]; - __u32 mode_ctx_num, numa_count = 0; - __u32 numa_idx, j; - int max_node, n; - handle_t ctx; + __u32 mode_ctx_num, numa_idx, j; + __u8 numa_count; int ret; + dparams.ctx_mode = ctx_mode; + dparams.op_type = op_type; + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + if (ctx_mode == CTX_MODE_SYNC) mode_ctx_num = ctx_params->ctx_set_num[op_type].sync_ctx_num; else @@ -2661,46 +2716,26 @@ static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, attrs->sched_type == SCHED_POLICY_SINGLE) && mode_ctx_num > 1) mode_ctx_num = 1; - if (drv->calc_type == UADK_ALG_HW) { - max_node = numa_max_node() + 1; - if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) - max_node = UADK_MAX_NUMA_NODES; - for (n = 0; n < max_node; n++) { - if (numa_bitmask_isbitset(ctx_params->bmp, n)) - numa_nodes[numa_count++] = n; - } - } else { - numa_nodes[0] = 0; - numa_count = 1; - } - if (!numa_count) { + if (drv->calc_type != UADK_ALG_HW) { numa_nodes[0] = 0; numa_count = 1; + } else { + wd_collect_numa_nodes(ctx_params->bmp, numa_nodes, &numa_count); } for (numa_idx = 0; numa_idx < numa_count; numa_idx++) { + dparams.numa_id = numa_nodes[numa_idx]; + for (j = 0; j < mode_ctx_num; j++) { - memset(&dparams, 0, sizeof(dparams)); - dparams.ctx_mode = ctx_mode; - dparams.op_type = op_type; - dparams.numa_id = numa_nodes[numa_idx]; - dparams.bmp = ctx_params->bmp; - dparams.epoll_en = false; - ret = drv->alloc_ctx(attrs->alg, &dparams, &ctx); - if (!ctx || ret < 0) { - if (ret == -WD_ENODEV) - break; + ret = wd_alloc_one_ctx(attrs, drv, &dparams, ctx_idx); + if (ret == -WD_ENODEV) + break; + + if (ret) { WD_ERR("failed to alloc ctx %u from driver %s on numa %d!\n", *ctx_idx, drv->drv_name, numa_nodes[numa_idx]); return ret; } - - internal_config->ctxs[*ctx_idx].ctx = ctx; - internal_config->ctxs[*ctx_idx].op_type = dparams.op_type; - internal_config->ctxs[*ctx_idx].ctx_mode = dparams.ctx_mode; - internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; - internal_config->ctxs[*ctx_idx].drv = drv; - (*ctx_idx)++; } } @@ -2749,6 +2784,146 @@ err_ctxs: return ret; } +/* + * SCHED_POLICY_DEV: open sync/async ctxs of (drv, op_type, numa) interleaved on + * ONE device. The caller pins the device via dparams->preferred_dev_path. + * Stop when both counts are met or the device is full. If both modes are + * requested but only one could be opened, roll back and return -WD_ENODEV so + * the caller can try the next device; partial within one mode is kept. + */ +static int wd_alloc_dev_group(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + struct wd_drv_ctx_params *dparams, + __u32 *ctx_idx) +{ + struct wd_ctx_nums *nums = &attrs->ctx_params->ctx_set_num[dparams->op_type]; + __u32 want[2] = {nums->sync_ctx_num, nums->async_ctx_num}; + __u8 modes[2] = {CTX_MODE_SYNC, CTX_MODE_ASYNC}; + __u32 group_start = *ctx_idx; + __u32 done[2] = {0, 0}; + __u32 i; + int ret; + + while (done[0] < want[0] || done[1] < want[1]) { + for (i = 0; i < 2; i++) { + if (done[i] >= want[i]) + continue; + + dparams->ctx_mode = modes[i]; + ret = wd_alloc_one_ctx(attrs, drv, dparams, ctx_idx); + if (ret) { + if (ret != -WD_EBUSY && ret != -WD_ENODEV) + return ret; + /* + * Device full. If both modes were requested + * but only one got ctxs, the empty domain + * would cause idx 65535 at runtime. Roll back + * this group so the caller tries another dev. + */ + if (want[0] && want[1] && + (!done[0] || !done[1])) { + wd_free_ctxs_batch_range(attrs, group_start, *ctx_idx); + *ctx_idx = group_start; + return -WD_ENODEV; + } + return WD_SUCCESS; + } + done[i]++; + } + } + return WD_SUCCESS; +} + +/* + * SCHED_POLICY_DEV: per (drv, op_type, numa) open sync/async on one device + * (interleaved, partial allowed). Enumerates devices on each numa node and + * pins each group to one device via preferred_dev_path, trying the next + * device if the current one cannot hold both modes. + */ +static int wd_alloc_ctxs_batch_dev(struct wd_init_attrs *attrs, __u32 *start_idx) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + struct wd_ctx_params *ctx_params = attrs->ctx_params; + struct uacce_dev_list *dev_list, *p; + int numa_nodes[UADK_MAX_NUMA_NODES]; + char alg_type[CRYPTO_MAX_ALG_NAME]; + struct wd_drv_ctx_params dparams; + __u32 ctx_idx, drv_idx, start; + struct wd_alg_driver *drv; + __u8 op_type, numa_count; + struct wd_ctx_nums *nums; + int n, ret; + + memset(&dparams, 0, sizeof(dparams)); + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + + start = *start_idx; + ctx_idx = start; + for (drv_idx = 0; drv_idx < internal_config->drv_count; drv_idx++) { + drv = internal_config->drv_array[drv_idx]; + if (!drv || !drv->alloc_ctx) { + WD_ERR("failed to check driver %s alloc_ctx!\n", + drv ? drv->drv_name : "unknown"); + ret = -WD_EINVAL; + goto err_ctxs; + } + if (drv->calc_type != UADK_ALG_HW) + continue; + + ret = wd_get_alg_type(attrs->alg, alg_type); + if (ret) { + ret = -WD_EINVAL; + goto err_ctxs; + } + if (!strcmp(alg_type, "ecc")) + (void)strcpy(alg_type, "sm2"); + if (!strcmp(alg_type, "comp")) + (void)strcpy(alg_type, "zlib"); + + dev_list = wd_get_accel_list(alg_type); + if (!dev_list) + continue; + + wd_collect_numa_nodes(ctx_params->bmp, numa_nodes, &numa_count); + + for (op_type = 0; op_type < ctx_params->op_type_num; op_type++) { + nums = &ctx_params->ctx_set_num[op_type]; + if (!nums->sync_ctx_num && !nums->async_ctx_num) + continue; + + dparams.op_type = op_type; + for (n = 0; n < numa_count; n++) { + dparams.numa_id = numa_nodes[n]; + for (p = dev_list; p; p = p->next) { + if (!p->dev || p->dev->numa_id != numa_nodes[n]) + continue; + + dparams.preferred_dev_path = p->dev->char_dev_path; + ret = wd_alloc_dev_group(attrs, drv, + &dparams, + &ctx_idx); + if (!ret) + break; + if (ret != -WD_ENODEV) + goto err_ctxs; + } + } + } + wd_free_list_accels(dev_list); + dev_list = NULL; + } + + *start_idx = ctx_idx; + return ctx_idx == start ? -WD_ENODEV : WD_SUCCESS; + +err_ctxs: + if (dev_list) + wd_free_list_accels(dev_list); + wd_free_ctxs_batch(attrs, ctx_idx); + return ret; +} + /** * wd_alg_ctx_uninit() - Release ctxs, scheduler, ctx_config. * @@ -2880,18 +3055,24 @@ int wd_alg_ctx_init(struct wd_init_attrs *attrs) return -WD_EINVAL; } - /* - * Ensure that contexts (ctx) with the same attributes are allocated first, - * thereby maintaining queue continuity within the contexts. - */ - ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); - if (ret) - return -WD_EINVAL; + if (attrs->sched_type == SCHED_POLICY_DEV) { + ret = wd_alloc_ctxs_batch_dev(attrs, &ctx_idx); + /* DEV tolerates partial: only a truly empty result fails. */ + if (ret == -WD_ENODEV) + return -WD_EINVAL; - /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ - ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); - if (ret) - return -WD_EINVAL; + if (ret) + return ret; + } else { + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; + + /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; + } /* Backfill actual allocated count */ internal_config = attrs->ctx_config_internal; -- 2.50.1.windows.1

1 13

[PATCH] bind sync/async ctx to same device under SCHED_POLICY_DEV
by Chenghai Huang 14 Oct '26

14 Oct '26

Under SCHED_POLICY_DEV, allocating sync ctxs then async ctxs in two passes let concurrent contention split them across devices. The DEV scheduler pins the region to ctxs[0]'s device and forbids cross-region fallback, so the async domain ended empty: async picks returned idx 65535, sends failed with -22, and the async process hung. Add a DEV-only path that, per (drv, op_type, numa), opens sync/async ctxs interleaved on ONE device: the first ctx auto-selects via the existing NUMA scan, its dev_path pins every later ctx of the group through a new preferred_dev_path field in wd_drv_ctx_params, and wd_hw_alloc_ctx opens exactly that device without NUMA fallback. Stop when both counts are met or the device is full; the already-allocated ctxs are kept so the workload runs on whatever was obtained. No cross-device split. Other policies unchanged. Signed-off-by: Chenghai Huang <huangchenghai2(a)huawei.com> --- drv/wd_drv.c | 26 +++++ include/wd_alg.h | 3 + wd_util.c | 283 ++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 261 insertions(+), 51 deletions(-) diff --git a/drv/wd_drv.c b/drv/wd_drv.c index 381cee1..c839ed4 100644 --- a/drv/wd_drv.c +++ b/drv/wd_drv.c @@ -220,6 +220,32 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) goto out; } + /* + * Preferred-device binding (SCHED_POLICY_DEV): open exactly this device + * and fail strictly if unavailable, so the sync and async ctxs of one + * (driver, op_type, numa) stay on the same device. No NUMA fallback. + */ + if (ctx_params->preferred_dev_path) { + for (curr = used_list; curr; curr = curr->next) { + if (curr->dev && !strcmp(curr->dev->char_dev_path, + ctx_params->preferred_dev_path)) + break; + } + + if (!curr) { + ret = -WD_ENODEV; + goto out; + } + + hctx = wd_request_ctx(curr->dev); + if (!hctx) { + ret = -WD_EBUSY; + goto out; + } + + goto success; + } + curr = used_list; while (curr) { if (curr->dev) { diff --git a/include/wd_alg.h b/include/wd_alg.h index b256a2f..3cc4e43 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -96,6 +96,8 @@ enum alg_drv_type { * @ctx_mode: CTX_MODE_SYNC or CTX_MODE_ASYNC * @op_type: Operation type * @bmp: NUMA node bitmask (optional, NULL if not needed) + * @preferred_dev_path: Preferred device char_dev_path; NULL/empty = auto-select, + * non-empty = open only this device and fail strictly if unavailable */ struct wd_drv_ctx_params { __u8 ctx_mode; @@ -103,6 +105,7 @@ struct wd_drv_ctx_params { int numa_id; bool epoll_en; struct bitmask *bmp; + const char *preferred_dev_path; }; /** diff --git a/wd_util.c b/wd_util.c index 1b4c74e..c0173a3 100644 --- a/wd_util.c +++ b/wd_util.c @@ -2613,17 +2613,17 @@ static int wd_alg_sched_instance(struct wd_sched *sched, return WD_SUCCESS; } -static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, - __u32 allocated_count) +static void wd_free_ctxs_batch_range(struct wd_init_attrs *attrs, + __u32 start_idx, __u32 end_idx) { struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_alg_driver *drv; __u32 i; - if (!internal_config || !internal_config->ctxs || !allocated_count) + if (!internal_config || !internal_config->ctxs || start_idx >= end_idx) return; - for (i = 0; i < allocated_count; i++) { + for (i = start_idx; i < end_idx; i++) { if (!internal_config->ctxs[i].ctx) continue; @@ -2635,21 +2635,76 @@ static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, } } +static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, + __u32 allocated_count) +{ + wd_free_ctxs_batch_range(attrs, 0, allocated_count); +} + +static void wd_collect_numa_nodes(struct bitmask *bmp, int *numa_nodes, + __u8 *numa_count) +{ + int max_node, n; + + max_node = numa_max_node() + 1; + if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) + max_node = UADK_MAX_NUMA_NODES; + + *numa_count = 0; + + for (n = 0; n < max_node; n++) { + if (numa_bitmask_isbitset(bmp, n)) + numa_nodes[(*numa_count)++] = n; + } + + if (!*numa_count) { + numa_nodes[0] = 0; + *numa_count = 1; + } +} + +static int wd_alloc_one_ctx(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + struct wd_drv_ctx_params *dparams, + __u32 *ctx_idx) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + handle_t ctx; + int ret; + + ret = drv->alloc_ctx(attrs->alg, dparams, &ctx); + if (ret < 0) + return ret; + + if (!ctx) + return -WD_EINVAL; + + internal_config->ctxs[*ctx_idx].ctx = ctx; + internal_config->ctxs[*ctx_idx].op_type = dparams->op_type; + internal_config->ctxs[*ctx_idx].ctx_mode = dparams->ctx_mode; + internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; + internal_config->ctxs[*ctx_idx].drv = drv; + (*ctx_idx)++; + return WD_SUCCESS; +} + static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, struct wd_alg_driver *drv, __u8 ctx_mode, __u8 op_type, __u32 *ctx_idx) { - struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_ctx_params *ctx_params = attrs->ctx_params; - struct wd_drv_ctx_params dparams; + struct wd_drv_ctx_params dparams = {0}; int numa_nodes[UADK_MAX_NUMA_NODES]; - __u32 mode_ctx_num, numa_count = 0; - __u32 numa_idx, j; - int max_node, n; - handle_t ctx; + __u32 mode_ctx_num, numa_idx, j; + __u8 numa_count; int ret; + dparams.ctx_mode = ctx_mode; + dparams.op_type = op_type; + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + if (ctx_mode == CTX_MODE_SYNC) mode_ctx_num = ctx_params->ctx_set_num[op_type].sync_ctx_num; else @@ -2661,46 +2716,26 @@ static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, attrs->sched_type == SCHED_POLICY_SINGLE) && mode_ctx_num > 1) mode_ctx_num = 1; - if (drv->calc_type == UADK_ALG_HW) { - max_node = numa_max_node() + 1; - if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) - max_node = UADK_MAX_NUMA_NODES; - for (n = 0; n < max_node; n++) { - if (numa_bitmask_isbitset(ctx_params->bmp, n)) - numa_nodes[numa_count++] = n; - } - } else { - numa_nodes[0] = 0; - numa_count = 1; - } - if (!numa_count) { + if (drv->calc_type != UADK_ALG_HW) { numa_nodes[0] = 0; numa_count = 1; + } else { + wd_collect_numa_nodes(ctx_params->bmp, numa_nodes, &numa_count); } for (numa_idx = 0; numa_idx < numa_count; numa_idx++) { + dparams.numa_id = numa_nodes[numa_idx]; + for (j = 0; j < mode_ctx_num; j++) { - memset(&dparams, 0, sizeof(dparams)); - dparams.ctx_mode = ctx_mode; - dparams.op_type = op_type; - dparams.numa_id = numa_nodes[numa_idx]; - dparams.bmp = ctx_params->bmp; - dparams.epoll_en = false; - ret = drv->alloc_ctx(attrs->alg, &dparams, &ctx); - if (!ctx || ret < 0) { - if (ret == -WD_ENODEV) - break; + ret = wd_alloc_one_ctx(attrs, drv, &dparams, ctx_idx); + if (ret == -WD_ENODEV) + break; + + if (ret) { WD_ERR("failed to alloc ctx %u from driver %s on numa %d!\n", *ctx_idx, drv->drv_name, numa_nodes[numa_idx]); return ret; } - - internal_config->ctxs[*ctx_idx].ctx = ctx; - internal_config->ctxs[*ctx_idx].op_type = dparams.op_type; - internal_config->ctxs[*ctx_idx].ctx_mode = dparams.ctx_mode; - internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; - internal_config->ctxs[*ctx_idx].drv = drv; - (*ctx_idx)++; } } @@ -2749,6 +2784,146 @@ err_ctxs: return ret; } +/* + * SCHED_POLICY_DEV: open sync/async ctxs of (drv, op_type, numa) interleaved on + * ONE device. The caller pins the device via dparams->preferred_dev_path. + * Stop when both counts are met or the device is full. If both modes are + * requested but only one could be opened, roll back and return -WD_ENODEV so + * the caller can try the next device; partial within one mode is kept. + */ +static int wd_alloc_dev_group(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + struct wd_drv_ctx_params *dparams, + __u32 *ctx_idx) +{ + struct wd_ctx_nums *nums = &attrs->ctx_params->ctx_set_num[dparams->op_type]; + __u32 want[2] = {nums->sync_ctx_num, nums->async_ctx_num}; + __u8 modes[2] = {CTX_MODE_SYNC, CTX_MODE_ASYNC}; + __u32 group_start = *ctx_idx; + __u32 done[2] = {0, 0}; + __u32 i; + int ret; + + while (done[0] < want[0] || done[1] < want[1]) { + for (i = 0; i < 2; i++) { + if (done[i] >= want[i]) + continue; + + dparams->ctx_mode = modes[i]; + ret = wd_alloc_one_ctx(attrs, drv, dparams, ctx_idx); + if (ret) { + if (ret != -WD_EBUSY && ret != -WD_ENODEV) + return ret; + /* + * Device full. If both modes were requested + * but only one got ctxs, the empty domain + * would cause idx 65535 at runtime. Roll back + * this group so the caller tries another dev. + */ + if (want[0] && want[1] && + (!done[0] || !done[1])) { + wd_free_ctxs_batch_range(attrs, group_start, *ctx_idx); + *ctx_idx = group_start; + return -WD_ENODEV; + } + return WD_SUCCESS; + } + done[i]++; + } + } + return WD_SUCCESS; +} + +/* + * SCHED_POLICY_DEV: per (drv, op_type, numa) open sync/async on one device + * (interleaved, partial allowed). Enumerates devices on each numa node and + * pins each group to one device via preferred_dev_path, trying the next + * device if the current one cannot hold both modes. + */ +static int wd_alloc_ctxs_batch_dev(struct wd_init_attrs *attrs, __u32 *start_idx) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + struct wd_ctx_params *ctx_params = attrs->ctx_params; + struct uacce_dev_list *dev_list, *p; + int numa_nodes[UADK_MAX_NUMA_NODES]; + char alg_type[CRYPTO_MAX_ALG_NAME]; + struct wd_drv_ctx_params dparams; + __u32 ctx_idx, drv_idx, start; + struct wd_alg_driver *drv; + __u8 op_type, numa_count; + struct wd_ctx_nums *nums; + int n, ret; + + memset(&dparams, 0, sizeof(dparams)); + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + + start = *start_idx; + ctx_idx = start; + for (drv_idx = 0; drv_idx < internal_config->drv_count; drv_idx++) { + drv = internal_config->drv_array[drv_idx]; + if (!drv || !drv->alloc_ctx) { + WD_ERR("failed to check driver %s alloc_ctx!\n", + drv ? drv->drv_name : "unknown"); + ret = -WD_EINVAL; + goto err_ctxs; + } + if (drv->calc_type != UADK_ALG_HW) + continue; + + ret = wd_get_alg_type(attrs->alg, alg_type); + if (ret) { + ret = -WD_EINVAL; + goto err_ctxs; + } + if (!strcmp(alg_type, "ecc")) + (void)strcpy(alg_type, "sm2"); + if (!strcmp(alg_type, "comp")) + (void)strcpy(alg_type, "zlib"); + + dev_list = wd_get_accel_list(alg_type); + if (!dev_list) + continue; + + wd_collect_numa_nodes(ctx_params->bmp, numa_nodes, &numa_count); + + for (op_type = 0; op_type < ctx_params->op_type_num; op_type++) { + nums = &ctx_params->ctx_set_num[op_type]; + if (!nums->sync_ctx_num && !nums->async_ctx_num) + continue; + + dparams.op_type = op_type; + for (n = 0; n < numa_count; n++) { + dparams.numa_id = numa_nodes[n]; + for (p = dev_list; p; p = p->next) { + if (!p->dev || p->dev->numa_id != numa_nodes[n]) + continue; + + dparams.preferred_dev_path = p->dev->char_dev_path; + ret = wd_alloc_dev_group(attrs, drv, + &dparams, + &ctx_idx); + if (!ret) + break; + if (ret != -WD_ENODEV) + goto err_ctxs; + } + } + } + wd_free_list_accels(dev_list); + dev_list = NULL; + } + + *start_idx = ctx_idx; + return ctx_idx == start ? -WD_ENODEV : WD_SUCCESS; + +err_ctxs: + if (dev_list) + wd_free_list_accels(dev_list); + wd_free_ctxs_batch(attrs, ctx_idx); + return ret; +} + /** * wd_alg_ctx_uninit() - Release ctxs, scheduler, ctx_config. * @@ -2880,18 +3055,24 @@ int wd_alg_ctx_init(struct wd_init_attrs *attrs) return -WD_EINVAL; } - /* - * Ensure that contexts (ctx) with the same attributes are allocated first, - * thereby maintaining queue continuity within the contexts. - */ - ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); - if (ret) - return -WD_EINVAL; + if (attrs->sched_type == SCHED_POLICY_DEV) { + ret = wd_alloc_ctxs_batch_dev(attrs, &ctx_idx); + /* DEV tolerates partial: only a truly empty result fails. */ + if (ret == -WD_ENODEV) + return -WD_EINVAL; - /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ - ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); - if (ret) - return -WD_EINVAL; + if (ret) + return ret; + } else { + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; + + /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; + } /* Backfill actual allocated count */ internal_config = attrs->ctx_config_internal; -- 2.50.1.windows.1

1 13

[PATCH 2/4] uadk: unify SEC BD3 paths and fix CE digest/cipher result check
by Chenghai Huang 24 Sep '26

24 Sep '26

From: Wenkai Lin <linwenkai6(a)hisilicon.com> The SEC module has duplicated BD3 fill/parse paths across cipher, digest, and aead, a stream mode scene misconfiguration for common digests, and silently swallowed computation errors in the CE driver. The following three issues are addressed: Unify the BD3 fill/parse paths for cipher, digest, and aead on V3+ hardware. Previously each algorithm duplicated udata handling, scene selection, and DIF detection in the qm_fill/qm_parse entry points. Route everything through fill_xxx_bd3/parse_xxx_bd3, add DIF and udata address helpers, select scene (STORAGE/IPSEC/STREAM) by is_storage, and guard unmap in error paths. BD1/BD2 paths are retained for V2 hardware. The common digest (such as SHA/SM3) that is not GMAC supports streaming hash processing, but SCENE_STREAM was only described for CTR/GCM streaming encryption in the FS, and the hash function for streaming mode in FS014/FS017 was ignored. Use SCENE_STREAM for common digests; GMAC uses SCENE_IPSEC independently. In sm3_ce_drv_send and isa_ce_cipher_send, the return value of the software digest/cipher computation is silently overwritten by the subsequent wd_get_sqe_from_queue call. If the computation fails, the error is swallowed and the function may return success. Add an early return on error before calling wd_get_sqe_from_queue. Signed-off-by: Wenkai Lin <linwenkai6(a)hisilicon.com> --- drv/isa_ce_sm3.c | 2 + drv/isa_ce_sm4.c | 2 + v1/drv/hisi_sec_udrv.c | 465 ++++++++++++++++++++++++++--------------- 3 files changed, 298 insertions(+), 171 deletions(-) diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c index b41f6c5..117b95e 100644 --- a/drv/isa_ce_sm3.c +++ b/drv/isa_ce_sm3.c @@ -374,6 +374,8 @@ static int sm3_ce_drv_send(handle_t ctx, void *digest_msg) WD_ERR("invalid digest mode!\n"); ret = -WD_EINVAL; } + if (ret) + return ret; ret = wd_get_sqe_from_queue(sfctx, msg->tag); if (ret) diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c index 863b99e..7547c00 100644 --- a/drv/isa_ce_sm4.c +++ b/drv/isa_ce_sm4.c @@ -399,6 +399,8 @@ static int isa_ce_cipher_send(handle_t ctx, void *wd_msg) ret = sm4_xts_encrypt(msg, &rkey); else ret = sm4_xts_decrypt(msg, &rkey); + if (ret) + return ret; break; default: WD_ERR("The current block cipher mode is not supported!\n"); diff --git a/v1/drv/hisi_sec_udrv.c b/v1/drv/hisi_sec_udrv.c index cf4bcd9..6ad6c57 100644 --- a/v1/drv/hisi_sec_udrv.c +++ b/v1/drv/hisi_sec_udrv.c @@ -823,11 +823,6 @@ static int fill_cipher_bd3_alg(struct wcrypto_cipher_msg *msg, struct hisi_sec_bd3_sqe *sqe) { int ret = WD_SUCCESS; - - ret = cipher_param_check(msg); - if (unlikely(ret)) - return ret; - __u8 c_key_len = 0; switch (msg->alg) { @@ -971,19 +966,72 @@ static int fill_cipher_bd3_mode(struct wcrypto_cipher_msg *msg, return ret; } +static void fill_cipher_bd3_dif(struct hisi_sec_bd3_sqe *sqe, + struct wd_sec_udata *udata) +{ + sqe->skip_data.gran_num = udata->gran_num; + sqe->skip_data.src_skip_data_len = udata->src_offset; + sqe->skip_data.dst_skip_data_len = udata->dst_offset; + + sqe->storage_scene.gen_ver_val = udata->dif.ver; + sqe->storage_scene.gen_app_val = udata->dif.app; + sqe->storage_scene.gen_page_pad_ctrl = udata->dif.ctrl.gen.page_layout_gen_type; + sqe->storage_scene.gen_grd_ctrl = udata->dif.ctrl.gen.grd_gen_type; + sqe->storage_scene.gen_ver_ctrl = udata->dif.ctrl.gen.ver_gen_type; + sqe->storage_scene.gen_app_ctrl = udata->dif.ctrl.gen.app_gen_type; + sqe->storage_scene.gen_ref_ctrl = udata->dif.ctrl.gen.ref_gen_type; + sqe->storage_scene.page_pad_type = udata->dif.ctrl.gen.page_layout_pad_type; + sqe->storage_scene.block_size = udata->block_size; + sqe->storage_scene.private_info = udata->dif.priv_info; + sqe->storage_scene.chk_grd_ctrl = udata->dif.ctrl.verify.grd_verify_type; + sqe->storage_scene.chk_ref_ctrl = udata->dif.ctrl.verify.ref_verify_type; + sqe->storage_scene.lba_l = udata->dif.lba & QM_L32BITS_MASK; + sqe->storage_scene.lba_h = udata->dif.lba >> QM_HADDR_SHIFT; +} + +static void fill_cipher_bd3_udata_addr(struct wcrypto_cipher_msg *msg, + struct hisi_sec_bd3_sqe *sqe, __u8 is_storage) +{ + uintptr_t phy; + + phy = (uintptr_t)msg->in; + sqe->data_src_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->data_src_addr_h = HI_U32(phy); + phy = (uintptr_t)msg->out; + sqe->data_dst_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->data_dst_addr_h = HI_U32(phy); + phy = (uintptr_t)msg->key; + sqe->c_key_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->c_key_addr_h = HI_U32(phy); + if (msg->iv_bytes) { + phy = (uintptr_t)msg->iv; + if (is_storage) { + sqe->skip_data.c_iv_a_key_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->skip_data.c_iv_a_key_h = HI_U32(phy); + } else { + sqe->ipsec_scene.c_ivin_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->ipsec_scene.c_ivin_addr_h = HI_U32(phy); + } + } +} + static int fill_cipher_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, struct wcrypto_cipher_msg *msg, struct wcrypto_cipher_tag *tag) { + struct wd_sec_udata *udata = tag ? tag->priv : NULL; + __u8 is_storage = udata && udata->gran_num != 0; int ret; + if (!is_storage) { + ret = cipher_param_check(msg); + if (unlikely(ret)) + return ret; + } + sqe->type = BD_TYPE3; - sqe->scene = SCENE_IPSEC; + sqe->scene = is_storage ? SCENE_STORAGE : SCENE_IPSEC; sqe->de = DATA_DST_ADDR_ENABLE; - if (msg->in_bytes > MAX_CIPHER_LENGTH) { - WD_ERR("input data is too large.\n"); - return -WD_EINVAL; - } sqe->c_len = msg->in_bytes; fill_bd3_addr_type(msg->data_fmt, sqe); @@ -1000,10 +1048,19 @@ static int fill_cipher_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, return ret; } - ret = fill_cipher_bd3_area(q, msg, sqe); - if (ret != WD_SUCCESS) { - WD_ERR("fail to fill_cipher_bd3_addr.\n"); - return ret; + if (is_storage && msg->mode == WCRYPTO_CIPHER_XTS) + sqe->ci_gen = CI_GEN_BY_LBA; + + if (udata) { + fill_cipher_bd3_udata_addr(msg, sqe, is_storage); + if (is_storage) + fill_cipher_bd3_dif(sqe, udata); + } else { + ret = fill_cipher_bd3_area(q, msg, sqe); + if (ret != WD_SUCCESS) { + WD_ERR("fail to fill_cipher_bd3_addr.\n"); + return ret; + } } if (tag) @@ -1129,10 +1186,8 @@ int qm_fill_cipher_bd3_sqe(void *message, struct qm_queue_info *info, __u16 i) { struct wcrypto_cipher_msg *msg = message; struct wcrypto_cipher_tag *tag = (void *)(uintptr_t)msg->usr_data; - struct wd_sec_udata *udata = tag->priv; struct wd_queue *q = info->q; struct hisi_sec_bd3_sqe *sqe3; - struct hisi_sec_sqe *sqe; uintptr_t temp; int ret; @@ -1144,35 +1199,17 @@ int qm_fill_cipher_bd3_sqe(void *message, struct qm_queue_info *info, __u16 i) } temp = (uintptr_t)info->sq_base + i * info->sqe_size; + sqe3 = (struct hisi_sec_bd3_sqe *)temp; + memset(sqe3, 0, sizeof(struct hisi_sec_bd3_sqe)); - /* - * For user self-defined data with DIF scence, will fill BD1. - * For user self-defined data without DIF scence, will fill BD2. - * For non user self-defined data scence, will fill BD3. - */ - if (udata) { - sqe = (struct hisi_sec_sqe *)temp; - memset(sqe, 0, sizeof(struct hisi_sec_sqe)); - if (udata->gran_num != 0) - ret = fill_cipher_bd1(q, sqe, msg, tag); - else - ret = fill_cipher_bd2(q, sqe, msg, tag); - } else { - sqe3 = (struct hisi_sec_bd3_sqe *)temp; - memset(sqe3, 0, sizeof(struct hisi_sec_bd3_sqe)); - ret = fill_cipher_bd3(q, sqe3, msg, tag); - } - + ret = fill_cipher_bd3(q, sqe3, msg, tag); if (ret != WD_SUCCESS) return ret; info->req_cache[i] = msg; #ifdef DEBUG_LOG - if (udata) - sec_dump_bd((unsigned char *)sqe, SQE_BYTES_NUMS); - else - sec_dump_bd((unsigned char *)sqe3, SQE_BYTES_NUMS); + sec_dump_bd((unsigned char *)sqe3, SQE_BYTES_NUMS); #endif return ret; @@ -1707,24 +1744,45 @@ static int digest_param_check_v3(struct wcrypto_digest_msg *msg) return WD_SUCCESS; } -static int fill_digest_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, - struct wcrypto_digest_msg *msg, struct wcrypto_digest_tag *tag) +static void fill_digest_bd3_udata_addr(struct wcrypto_digest_msg *msg, + struct hisi_sec_bd3_sqe *sqe) { uintptr_t phy; - int ret; - ret = digest_param_check_v3(msg); - if (unlikely(ret)) - return ret; + phy = (uintptr_t)msg->in; + sqe->data_src_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->data_src_addr_h = HI_U32(phy); + phy = (uintptr_t)msg->out; + sqe->mac_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->mac_addr_h = HI_U32(phy); - sqe->type = BD_TYPE3; - if (msg->alg == WCRYPTO_AES_GMAC) - sqe->scene = SCENE_IPSEC; - else - sqe->scene = SCENE_STREAM; + if (msg->mode == WCRYPTO_DIGEST_HMAC) { + sqe->a_key_len = msg->key_bytes / SEC_SQE_LEN_RATE; + phy = (uintptr_t)msg->key; + sqe->auth_key_iv.a_key_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->auth_key_iv.a_key_addr_h = HI_U32(phy); + } +} + +static void fill_digest_bd3_dif(struct hisi_sec_bd3_sqe *sqe, + struct wd_sec_udata *udata) +{ + sqe->skip_data.gran_num = udata->gran_num; + sqe->skip_data.src_skip_data_len = udata->src_offset; + sqe->storage_scene.block_size = udata->block_size; + sqe->storage_scene.private_info = udata->dif.priv_info; + sqe->storage_scene.chk_grd_ctrl = udata->dif.ctrl.verify.grd_verify_type; + sqe->storage_scene.chk_ref_ctrl = udata->dif.ctrl.verify.ref_verify_type; + sqe->storage_scene.lba_l = udata->dif.lba & QM_L32BITS_MASK; + sqe->storage_scene.lba_h = udata->dif.lba >> QM_HADDR_SHIFT; +} + +static int fill_digest_bd3_map(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, + struct wcrypto_digest_msg *msg) +{ + uintptr_t phy; + int ret; - sqe->auth = AUTH_MAC_CALCULATE; - sqe->a_len = msg->in_bytes; phy = (uintptr_t)drv_iova_map(q, msg->in, msg->in_bytes); if (unlikely(!phy)) { WD_ERR("Get message in dma address fail!\n"); @@ -1740,33 +1798,84 @@ static int fill_digest_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, WD_ERR("Get digest bd3 message out dma address fail!\n"); goto map_out_error; } + + ret = set_hmac_mode_v3(msg, sqe, q); + if (unlikely(ret)) + goto unmap_out; + + return WD_SUCCESS; + +unmap_out: + unmap_addr(q, msg->out, msg->out_bytes, sqe->mac_addr_l, + sqe->mac_addr_h, msg->data_fmt); +map_out_error: + phy = DMA_ADDR(sqe->data_src_addr_h, sqe->data_src_addr_l); + drv_iova_unmap(q, msg->in, (void *)(uintptr_t)phy, msg->in_bytes); + return ret; +} + +static int fill_digest_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, + struct wcrypto_digest_msg *msg, struct wcrypto_digest_tag *tag) +{ + struct wd_sec_udata *udata = tag ? tag->priv : NULL; + __u8 is_storage = udata && udata->gran_num != 0; + uintptr_t phy; + int ret; + + ret = digest_param_check_v3(msg); + if (unlikely(ret)) + return ret; + + sqe->type = BD_TYPE3; + + /* + * GMAC is an IPSEC-specific auth algo (RFC4543); other digests use + * stream scene for long-data hash processing. + */ + if (is_storage) + sqe->scene = SCENE_STORAGE; + else if (msg->alg == WCRYPTO_AES_GMAC) + sqe->scene = SCENE_IPSEC; + else + sqe->scene = SCENE_STREAM; + + sqe->auth = AUTH_MAC_CALCULATE; + sqe->a_len = msg->in_bytes; + + if (udata) { + fill_digest_bd3_udata_addr(msg, sqe); + if (is_storage) + fill_digest_bd3_dif(sqe, udata); + } else { + ret = fill_digest_bd3_map(q, sqe, msg); + if (unlikely(ret)) + return ret; + } + sqe->mac_len = msg->out_bytes / WORD_BYTES; ret = fill_digest_bd3_alg(msg, sqe); if (ret != WD_SUCCESS) { WD_ERR("fill_digest_bd3_alg fail!\n"); - goto map_alg_error; + goto unmap; } ret = qm_fill_digest_long_bd3(msg, sqe); if (ret) - goto map_alg_error; - - ret = set_hmac_mode_v3(msg, sqe, q); - if (ret) - goto map_alg_error; + goto unmap; if (tag) sqe->tag_l = tag->wcrypto_tag.ctx_id; return ret; -map_alg_error: - unmap_addr(q, msg->out, msg->out_bytes, sqe->mac_addr_l, - sqe->mac_addr_h, msg->data_fmt); -map_out_error: - phy = DMA_ADDR(sqe->data_src_addr_h, sqe->data_src_addr_l); - drv_iova_unmap(q, msg->in, (void *)(uintptr_t)phy, msg->in_bytes); +unmap: + if (!udata) { + unmap_addr(q, msg->out, msg->out_bytes, sqe->mac_addr_l, + sqe->mac_addr_h, msg->data_fmt); + phy = DMA_ADDR(sqe->data_src_addr_h, sqe->data_src_addr_l); + drv_iova_unmap(q, msg->in, (void *)(uintptr_t)phy, msg->in_bytes); + } return ret; } @@ -1776,23 +1885,15 @@ int qm_fill_digest_bd3_sqe(void *message, struct qm_queue_info *info, __u16 i) struct wcrypto_digest_tag *tag = (void *)(uintptr_t)msg->usr_data; struct wd_queue *q = info->q; struct hisi_sec_bd3_sqe *sqe; - struct hisi_sec_sqe *sqe2; uintptr_t temp; int ret; temp = (uintptr_t)info->sq_base + i * info->sqe_size; + sqe = (struct hisi_sec_bd3_sqe *)temp; + memset(sqe, 0, sizeof(struct hisi_sec_bd3_sqe)); + fill_bd3_addr_type(msg->data_fmt, sqe); - if (tag->priv) { - sqe2 = (struct hisi_sec_sqe *)temp; - memset(sqe2, 0, sizeof(struct hisi_sec_sqe)); - fill_bd_addr_type(msg->data_fmt, sqe2); - ret = fill_digest_bd_udata(q, sqe2, msg, tag); - } else { - sqe = (struct hisi_sec_bd3_sqe *)temp; - memset(sqe, 0, sizeof(struct hisi_sec_bd3_sqe)); - fill_bd3_addr_type(msg->data_fmt, sqe); - ret = fill_digest_bd3(q, sqe, msg, tag); - } + ret = fill_digest_bd3(q, sqe, msg, tag); if (ret != WD_SUCCESS) return ret; @@ -1890,7 +1991,7 @@ static void parse_cipher_bd2(struct wd_queue *q, struct hisi_sec_sqe *sqe, /* In user self-define data case, may not need addr map, just return */ tag = (void *)(uintptr_t)cipher_msg->usr_data; - if (tag->priv) + if (tag && tag->priv) return; dma_addr = DMA_ADDR(sqe->type2.data_src_addr_h, @@ -1919,16 +2020,24 @@ static void parse_cipher_bd2(struct wd_queue *q, struct hisi_sec_sqe *sqe, static void parse_cipher_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, struct wcrypto_cipher_msg *cipher_msg) { + struct wcrypto_cipher_tag *tag; __u64 dma_addr; if (sqe->done != SEC_HW_TASK_DONE || sqe->error_type) { WD_ERR("Fail to parse SEC BD3 %s, done=0x%x, etype=0x%x\n", "cipher", (__u32)sqe->done, (__u32)sqe->error_type); cipher_msg->result = WD_IN_EPARA; + } else if (sqe->dc == DIF_VERIFY_FAIL) { + cipher_msg->result = WD_VERIFY_ERR; } else { cipher_msg->result = WD_SUCCESS; } + /* In user self-define data case, may not need addr map, just return */ + tag = (void *)(uintptr_t)cipher_msg->usr_data; + if (tag && tag->priv) + return; + dma_addr = DMA_ADDR(sqe->data_src_addr_h, sqe->data_src_addr_l); drv_iova_unmap(q, cipher_msg->in, (void *)(uintptr_t)dma_addr, cipher_msg->in_bytes); @@ -1990,36 +2099,15 @@ int qm_parse_cipher_bd3_sqe(void *msg, const struct qm_queue_info *info, struct wcrypto_cipher_msg *cipher_msg = info->req_cache[i]; struct hisi_sec_bd3_sqe *sqe3 = msg; struct wd_queue *q = info->q; - struct hisi_sec_sqe *sqe; if (unlikely(!cipher_msg)) { WD_ERR("info->req_cache is null at index:%hu\n", i); return 0; } + if (unlikely(usr && sqe3->tag_l != usr)) + return 0; - switch (sqe3->type) { - case BD_TYPE3: - if (unlikely(usr && sqe3->tag_l != usr)) - return 0; - parse_cipher_bd3(q, sqe3, cipher_msg); - break; - case BD_TYPE2: - sqe = (struct hisi_sec_sqe *)sqe3; - if (usr && sqe->type2.tag != usr) - return 0; - parse_cipher_bd2(q, sqe, cipher_msg); - break; - case BD_TYPE1: - sqe = (struct hisi_sec_sqe *)sqe3; - if (usr && sqe->type1.tag != usr) - return 0; - parse_cipher_bd1(q, sqe, cipher_msg); - break; - default: - WD_ERR("SEC BD Type error\n"); - cipher_msg->result = WD_IN_EPARA; - break; - } + parse_cipher_bd3(q, sqe3, cipher_msg); #ifdef DEBUG_LOG sec_dump_bd((unsigned char *)msg, SQE_BYTES_NUMS); @@ -2068,7 +2156,7 @@ static void parse_digest_bd2(struct wd_queue *q, struct hisi_sec_sqe *sqe, digest_msg->result = WD_SUCCESS; tag = (void *)(uintptr_t)digest_msg->usr_data; - if (tag->priv) + if (tag && tag->priv) return; dma_addr = DMA_ADDR(sqe->type2.data_src_addr_h, @@ -2674,16 +2762,80 @@ out: return ret; } +static int fill_aead_bd3_udata_addr(struct wcrypto_aead_msg *msg, + struct hisi_sec_bd3_sqe *sqe, struct wd_aead_udata *udata) +{ + uintptr_t phy; + + sqe->auth_src_offset = udata->src_offset; + sqe->cipher_src_offset = udata->src_offset + msg->assoc_bytes; + + phy = (uintptr_t)msg->in; + sqe->data_src_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->data_src_addr_h = HI_U32(phy); + phy = (uintptr_t)msg->out; + sqe->data_dst_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->data_dst_addr_h = HI_U32(phy); + phy = (uintptr_t)msg->iv; + sqe->ipsec_scene.c_ivin_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->ipsec_scene.c_ivin_addr_h = HI_U32(phy); + + phy = (uintptr_t)udata->ckey; + sqe->c_key_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->c_key_addr_h = HI_U32(phy); + phy = (uintptr_t)udata->mac; + sqe->mac_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->mac_addr_h = HI_U32(phy); + if (msg->cmode == WCRYPTO_CIPHER_CCM || msg->cmode == WCRYPTO_CIPHER_GCM) { + if (unlikely(!udata->aiv)) { + WD_ERR("invalid aead udata: aiv is NULL!\n"); + return -WD_EINVAL; + } + phy = (uintptr_t)udata->aiv; + sqe->auth_key_iv.a_ivin_addr_l = (__u32)(phy & QM_L32BITS_MASK); + sqe->auth_key_iv.a_ivin_addr_h = HI_U32(phy); + } + + return WD_SUCCESS; +} + +static int init_msg_with_udata(struct wcrypto_aead_msg *req, struct wd_aead_udata *udata) +{ + if (!udata->ckey || !udata->mac) { + WD_ERR("invalid udata para!\n"); + return -WD_EINVAL; + } + + if (req->cmode == WCRYPTO_CIPHER_CCM || req->cmode == WCRYPTO_CIPHER_GCM) { + req->ckey_bytes = udata->ckey_bytes; + req->auth_bytes = udata->mac_bytes; + } else { + WD_ERR("invalid cmode para!\n"); + return -WD_EINVAL; + } + + return WD_SUCCESS; +} + static int fill_aead_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, struct wcrypto_aead_msg *msg, struct wcrypto_aead_tag *tag) { + struct wd_aead_udata *udata = tag ? tag->priv : NULL; int ret; + if (udata) { + ret = init_msg_with_udata(msg, udata); + if (ret != WD_SUCCESS) + return ret; + } + sqe->type = BD_TYPE3; sqe->scene = SCENE_IPSEC; sqe->de = DATA_DST_ADDR_ENABLE; sqe->c_len = msg->in_bytes; - sqe->cipher_src_offset = msg->assoc_bytes; + sqe->cipher_src_offset = udata ? udata->src_offset + msg->assoc_bytes + : msg->assoc_bytes; + sqe->auth_src_offset = udata ? udata->src_offset : 0; sqe->a_len = msg->in_bytes + msg->assoc_bytes; ret = fill_aead_bd3_alg(msg, sqe); @@ -2698,15 +2850,21 @@ static int fill_aead_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, return ret; } - ret = fill_aead_bd3_addr(q, msg, sqe); - if (ret != WD_SUCCESS) { - WD_ERR("fail to fill_aead_bd3_addr!\n"); - return ret; - } + if (udata) { + ret = fill_aead_bd3_udata_addr(msg, sqe, udata); + if (unlikely(ret)) + return ret; + } else { + ret = fill_aead_bd3_addr(q, msg, sqe); + if (ret != WD_SUCCESS) { + WD_ERR("fail to fill_aead_bd3_addr!\n"); + return ret; + } - ret = fill_aead_stream_bd3(q, msg, sqe); - if (unlikely(ret)) - return ret; + ret = fill_aead_stream_bd3(q, msg, sqe); + if (unlikely(ret)) + return ret; + } if (tag) sqe->tag_l = tag->wcrypto_tag.ctx_id; @@ -2777,7 +2935,6 @@ int qm_fill_aead_bd3_sqe(void *message, struct qm_queue_info *info, __u16 i) struct wcrypto_aead_tag *tag = (void *)(uintptr_t)msg->usr_data; struct wd_queue *q = info->q; struct hisi_sec_bd3_sqe *sqe; - struct hisi_sec_sqe *sqe2; uintptr_t temp; int ret; @@ -2789,17 +2946,11 @@ int qm_fill_aead_bd3_sqe(void *message, struct qm_queue_info *info, __u16 i) } temp = (uintptr_t)info->sq_base + i * info->sqe_size; - if (tag->priv) { - sqe2 = (struct hisi_sec_sqe *)temp; - memset(sqe2, 0, sizeof(struct hisi_sec_sqe)); - fill_bd_addr_type(msg->data_fmt, sqe2); - ret = fill_aead_bd_udata(q, sqe2, msg, tag); - } else { - sqe = (struct hisi_sec_bd3_sqe *)temp; - memset(sqe, 0, sizeof(struct hisi_sec_bd3_sqe)); - fill_bd3_addr_type(msg->data_fmt, sqe); - ret = fill_aead_bd3(q, sqe, msg, tag); - } + sqe = (struct hisi_sec_bd3_sqe *)temp; + memset(sqe, 0, sizeof(struct hisi_sec_bd3_sqe)); + fill_bd3_addr_type(msg->data_fmt, sqe); + + ret = fill_aead_bd3(q, sqe, msg, tag); if (ret != WD_SUCCESS) return ret; @@ -2815,6 +2966,7 @@ int qm_fill_aead_bd3_sqe(void *message, struct qm_queue_info *info, __u16 i) static void parse_aead_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe3, struct wcrypto_aead_msg *msg) { + struct wcrypto_aead_tag *tag; __u8 mac[AEAD_IV_MAX_BYTES] = { 0 }; __u64 dma_addr; int ret; @@ -2828,6 +2980,11 @@ static void parse_aead_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe3, msg->result = WD_SUCCESS; } + /* In user self-define data case, may not need addr map, just return */ + tag = (void *)(uintptr_t)msg->usr_data; + if (tag && tag->priv) + return; + /* * We obtain a memory from IV SGL as a temporary address space for MAC， * After the encryption is completed, copy the data from this temporary @@ -2883,26 +3040,16 @@ int qm_parse_aead_bd3_sqe(void *msg, const struct qm_queue_info *info, { struct wcrypto_aead_msg *aead_msg = info->req_cache[i]; struct hisi_sec_bd3_sqe *sqe = msg; - struct hisi_sec_sqe *sqe2 = msg; struct wd_queue *q = info->q; if (unlikely(!aead_msg)) { WD_ERR("info->req_cache is null at index:%hu\n", i); return 0; } + if (unlikely(usr && sqe->tag_l != usr)) + return 0; - if (sqe->type == BD_TYPE3) { - if (usr && sqe->tag_l != usr) - return 0; - parse_aead_bd3(q, sqe, aead_msg); - } else if (sqe->type == BD_TYPE2) { - if (usr && sqe2->type2.tag != usr) - return 0; - parse_aead_bd2(q, sqe2, aead_msg); - } else { - WD_ERR("SEC BD Type error\n"); - aead_msg->result = WD_IN_EPARA; - } + parse_aead_bd3(q, sqe, aead_msg); #ifdef DEBUG_LOG sec_dump_bd((unsigned char *)msg, SQE_BYTES_NUMS); @@ -2914,16 +3061,24 @@ int qm_parse_aead_bd3_sqe(void *msg, const struct qm_queue_info *info, static void parse_digest_bd3(struct wd_queue *q, struct hisi_sec_bd3_sqe *sqe, struct wcrypto_digest_msg *digest_msg) { + struct wcrypto_digest_tag *tag; __u64 dma_addr; if (sqe->done != SEC_HW_TASK_DONE || sqe->error_type) { WD_ERR("SEC BD3 %s fail!done=0x%x, etype=0x%x\n", "digest", (__u32)sqe->done, (__u32)sqe->error_type); digest_msg->result = WD_IN_EPARA; + } else if (sqe->dc == DIF_VERIFY_FAIL) { + digest_msg->result = WD_VERIFY_ERR; } else { digest_msg->result = WD_SUCCESS; } + /* In user self-define data case, may not need addr map, just return */ + tag = (void *)(uintptr_t)digest_msg->usr_data; + if (tag && tag->priv) + return; + dma_addr = DMA_ADDR(sqe->data_src_addr_h, sqe->data_src_addr_l); drv_iova_unmap(q, digest_msg->in, (void *)(uintptr_t)dma_addr, digest_msg->in_bytes); @@ -2946,30 +3101,16 @@ int qm_parse_digest_bd3_sqe(void *msg, const struct qm_queue_info *info, { struct wcrypto_digest_msg *digest_msg = info->req_cache[i]; struct hisi_sec_bd3_sqe *sqe = msg; - struct hisi_sec_sqe *sqe2 = msg; struct wd_queue *q = info->q; if (unlikely(!digest_msg)) { WD_ERR("info->req_cache is null at index:%hu\n", i); return 0; } + if (unlikely(usr && sqe->tag_l != usr)) + return 0; - if (sqe->type == BD_TYPE3) { - if (usr && sqe->tag_l != usr) - return 0; - parse_digest_bd3(q, sqe, digest_msg); - } else if (sqe->type == BD_TYPE2) { - if (usr && sqe2->type2.tag != usr) - return 0; - parse_digest_bd2(q, sqe2, digest_msg); - } else if (sqe->type == BD_TYPE1) { - if (usr && sqe2->type1.tag != usr) - return 0; - parse_digest_bd1(q, sqe2, digest_msg); - } else { - WD_ERR("SEC Digest BD Type error\n"); - digest_msg->result = WD_IN_EPARA; - } + parse_digest_bd3(q, sqe, digest_msg); #ifdef DEBUG_LOG sec_dump_bd((unsigned char *)msg, SQE_BYTES_NUMS); @@ -3332,24 +3473,6 @@ static int fill_aead_bd2(struct wd_queue *q, struct hisi_sec_sqe *sqe, return fill_aead_bd2_addr(q, msg, sqe); } -static int init_msg_with_udata(struct wcrypto_aead_msg *req, struct wd_aead_udata *udata) -{ - if (!udata->ckey || !udata->mac) { - WD_ERR("invalid udata para!\n"); - return -WD_EINVAL; - } - - if (req->cmode == WCRYPTO_CIPHER_CCM || req->cmode == WCRYPTO_CIPHER_GCM) { - req->ckey_bytes = udata->ckey_bytes; - req->auth_bytes = udata->mac_bytes; - } else { - WD_ERR("invalid cmode para!\n"); - return -WD_EINVAL; - } - - return WD_SUCCESS; -} - static int fill_aead_bd2_udata(struct wd_queue *q, struct hisi_sec_sqe *sqe, struct wcrypto_aead_msg *msg, struct wcrypto_aead_tag *tag) { @@ -3433,7 +3556,7 @@ static void parse_aead_bd2(struct wd_queue *q, struct hisi_sec_sqe *sqe, } tag = (void *)(uintptr_t)msg->usr_data; - if (tag->priv) + if (tag && tag->priv) return; /* -- 2.28.0.windows.1

1 0

[PATCH 1/4] uadk: fix scheduler ctx allocation and sched_params initialization
by Chenghai Huang 20 Sep '26

20 Sep '26

The scheduler ctx allocation under SCHED_POLICY_DEV and session initialization in several modules contain bugs that cause async process hangs, device starvation, and nondeterministic scheduling behavior. The following four issues are addressed: Under SCHED_POLICY_DEV, allocating sync ctxs then async ctxs in two passes let concurrent contention split them across devices. The DEV scheduler pins the region to ctxs[0]'s device and forbids cross-region fallback, so the async domain ended empty: async picks returned idx 65535, sends failed with -22, and the async process hung. Add a DEV-only path that opens sync/async ctxs interleaved on ONE device, with a preferred_dev_path field to pin later ctxs of the group. Then simplify it to a linear two-pass per device: open all sync then all async; any shortfall rolls back the whole group and tries the next device. wd_hw_alloc_ctx picked the first device whose wd_request_ctx() succeeded. When PF and VFs share the same NUMA node, concurrent businesses all bind to the same leading device, starving the others. Select the device with the most available contexts (wd_get_avail_ctx) instead, with fallback to remaining devices if the chosen one fails to open. wd_agg_alloc_sess, wd_join_gather_alloc_sess and wd_udma_alloc_sess declared a local struct wd_sched_params without initialization, setting only alg_name and ctxs before passing it to wd_sched_set_param, which unconditionally reads pkt_size, data_mode and prio_mode. The uninitialized stack values caused nondeterministic scheduling behavior. Initialize the struct with = {0} at declaration. Signed-off-by: Chenghai Huang <huangchenghai2(a)huawei.com> Co-developed-by: Wenkai Lin <linwenkai6(a)hisilicon.com> Signed-off-by: Wenkai Lin <linwenkai6(a)hisilicon.com> Co-developed-by: Weili Qian <qianweili(a)huawei.com> Signed-off-by: Weili Qian <qianweili(a)huawei.com> --- drv/wd_drv.c | 67 ++++++++++-- include/wd_alg.h | 3 + wd_agg.c | 2 +- wd_join_gather.c | 2 +- wd_udma.c | 2 +- wd_util.c | 279 ++++++++++++++++++++++++++++++++++++++--------- 6 files changed, 291 insertions(+), 64 deletions(-) diff --git a/drv/wd_drv.c b/drv/wd_drv.c index 381cee1..08014dd 100644 --- a/drv/wd_drv.c +++ b/drv/wd_drv.c @@ -181,12 +181,12 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) { struct wd_drv_ctx_params *ctx_params = (struct wd_drv_ctx_params *)params; struct uacce_dev_list *dev_list, *used_list = NULL; + struct uacce_dev_list *curr, *best = NULL; char alg_type[CRYPTO_MAX_ALG_NAME]; - struct uacce_dev_list *curr; + int target_numa, best_avail = 0; struct wd_ctx_h *ctx_h; - int target_numa; + int ret, avail; handle_t hctx; - int ret; if (!params || !ctx) { WD_ERR("invalid: parameters are NULL!\n"); @@ -220,14 +220,61 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) goto out; } - curr = used_list; - while (curr) { - if (curr->dev) { - hctx = wd_request_ctx(curr->dev); - if (hctx) - goto success; + /* + * Preferred-device binding (SCHED_POLICY_DEV): open exactly this device + * and fail strictly if unavailable, so the sync and async ctxs of one + * (driver, op_type, numa) stay on the same device. No NUMA fallback. + */ + if (ctx_params->preferred_dev_path) { + for (curr = used_list; curr; curr = curr->next) { + if (curr->dev && !strcmp(curr->dev->char_dev_path, + ctx_params->preferred_dev_path)) + break; } - curr = curr->next; + + if (!curr) { + ret = -WD_ENODEV; + goto out; + } + + hctx = wd_request_ctx(curr->dev); + if (!hctx) { + ret = -WD_EBUSY; + goto out; + } + + goto success; + } + + /* Pick the device with the most available contexts to balance load. */ + for (curr = used_list; curr; curr = curr->next) { + if (!curr->dev) + continue; + avail = wd_get_avail_ctx(curr->dev); + if (avail > best_avail) { + best_avail = avail; + best = curr; + } + } + + if (!best) { + WD_ERR("failed to request ctx on NUMA node %d for %s\n", + target_numa, alg_name); + ret = -WD_EBUSY; + goto out; + } + + hctx = wd_request_ctx(best->dev); + if (hctx) + goto success; + + /* Fall back to remaining devices if the best one fails to open */ + for (curr = used_list; curr; curr = curr->next) { + if (curr == best || !curr->dev) + continue; + hctx = wd_request_ctx(curr->dev); + if (hctx) + goto success; } WD_ERR("failed to request ctx on NUMA node %d for %s\n", diff --git a/include/wd_alg.h b/include/wd_alg.h index b256a2f..3cc4e43 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -96,6 +96,8 @@ enum alg_drv_type { * @ctx_mode: CTX_MODE_SYNC or CTX_MODE_ASYNC * @op_type: Operation type * @bmp: NUMA node bitmask (optional, NULL if not needed) + * @preferred_dev_path: Preferred device char_dev_path; NULL/empty = auto-select, + * non-empty = open only this device and fail strictly if unavailable */ struct wd_drv_ctx_params { __u8 ctx_mode; @@ -103,6 +105,7 @@ struct wd_drv_ctx_params { int numa_id; bool epoll_en; struct bitmask *bmp; + const char *preferred_dev_path; }; /** diff --git a/wd_agg.c b/wd_agg.c index 95cb869..2c7825d 100644 --- a/wd_agg.c +++ b/wd_agg.c @@ -446,7 +446,7 @@ static int wd_agg_uninit_sess_priv(struct wd_agg_sess *sess) handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) { - struct wd_sched_params params; + struct wd_sched_params params = {0}; struct wd_agg_sess *sess; __u32 out_agg_cols_num = 0; int ret; diff --git a/wd_join_gather.c b/wd_join_gather.c index e43dbf9..27472d3 100644 --- a/wd_join_gather.c +++ b/wd_join_gather.c @@ -533,7 +533,7 @@ static int wd_join_gather_init_sess(struct wd_join_gather_sess *sess, handle_t wd_join_gather_alloc_sess(struct wd_join_gather_sess_setup *setup) { - struct wd_sched_params params; + struct wd_sched_params params = {0}; struct wd_join_gather_sess *sess; int ret; diff --git a/wd_udma.c b/wd_udma.c index 582c8d1..59841bb 100644 --- a/wd_udma.c +++ b/wd_udma.c @@ -78,7 +78,7 @@ void wd_udma_free_sess(handle_t sess) handle_t wd_udma_alloc_sess(struct wd_udma_sess_setup *setup) { - struct wd_sched_params params; + struct wd_sched_params params = {0}; struct wd_udma_sess *sess; int ret; diff --git a/wd_util.c b/wd_util.c index 1b4c74e..0d482f9 100644 --- a/wd_util.c +++ b/wd_util.c @@ -2613,17 +2613,17 @@ static int wd_alg_sched_instance(struct wd_sched *sched, return WD_SUCCESS; } -static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, - __u32 allocated_count) +static void wd_free_ctxs_batch_range(struct wd_init_attrs *attrs, + __u32 start_idx, __u32 end_idx) { struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_alg_driver *drv; __u32 i; - if (!internal_config || !internal_config->ctxs || !allocated_count) + if (!internal_config || !internal_config->ctxs || start_idx >= end_idx) return; - for (i = 0; i < allocated_count; i++) { + for (i = start_idx; i < end_idx; i++) { if (!internal_config->ctxs[i].ctx) continue; @@ -2635,21 +2635,76 @@ static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, } } +static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, + __u32 allocated_count) +{ + wd_free_ctxs_batch_range(attrs, 0, allocated_count); +} + +static void wd_collect_numa_nodes(struct bitmask *bmp, int *numa_nodes, + __u8 *numa_count) +{ + int max_node, n; + + max_node = numa_max_node() + 1; + if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) + max_node = UADK_MAX_NUMA_NODES; + + *numa_count = 0; + + for (n = 0; n < max_node; n++) { + if (numa_bitmask_isbitset(bmp, n)) + numa_nodes[(*numa_count)++] = n; + } + + if (!*numa_count) { + numa_nodes[0] = 0; + *numa_count = 1; + } +} + +static int wd_alloc_one_ctx(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + struct wd_drv_ctx_params *dparams, + __u32 *ctx_idx) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + handle_t ctx; + int ret; + + ret = drv->alloc_ctx(attrs->alg, dparams, &ctx); + if (ret < 0) + return ret; + + if (!ctx) + return -WD_EINVAL; + + internal_config->ctxs[*ctx_idx].ctx = ctx; + internal_config->ctxs[*ctx_idx].op_type = dparams->op_type; + internal_config->ctxs[*ctx_idx].ctx_mode = dparams->ctx_mode; + internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; + internal_config->ctxs[*ctx_idx].drv = drv; + (*ctx_idx)++; + return WD_SUCCESS; +} + static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, struct wd_alg_driver *drv, __u8 ctx_mode, __u8 op_type, __u32 *ctx_idx) { - struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_ctx_params *ctx_params = attrs->ctx_params; - struct wd_drv_ctx_params dparams; + struct wd_drv_ctx_params dparams = {0}; int numa_nodes[UADK_MAX_NUMA_NODES]; - __u32 mode_ctx_num, numa_count = 0; - __u32 numa_idx, j; - int max_node, n; - handle_t ctx; + __u32 mode_ctx_num, numa_idx, j; + __u8 numa_count; int ret; + dparams.ctx_mode = ctx_mode; + dparams.op_type = op_type; + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + if (ctx_mode == CTX_MODE_SYNC) mode_ctx_num = ctx_params->ctx_set_num[op_type].sync_ctx_num; else @@ -2661,46 +2716,26 @@ static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, attrs->sched_type == SCHED_POLICY_SINGLE) && mode_ctx_num > 1) mode_ctx_num = 1; - if (drv->calc_type == UADK_ALG_HW) { - max_node = numa_max_node() + 1; - if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) - max_node = UADK_MAX_NUMA_NODES; - for (n = 0; n < max_node; n++) { - if (numa_bitmask_isbitset(ctx_params->bmp, n)) - numa_nodes[numa_count++] = n; - } - } else { - numa_nodes[0] = 0; - numa_count = 1; - } - if (!numa_count) { + if (drv->calc_type != UADK_ALG_HW) { numa_nodes[0] = 0; numa_count = 1; + } else { + wd_collect_numa_nodes(ctx_params->bmp, numa_nodes, &numa_count); } for (numa_idx = 0; numa_idx < numa_count; numa_idx++) { + dparams.numa_id = numa_nodes[numa_idx]; + for (j = 0; j < mode_ctx_num; j++) { - memset(&dparams, 0, sizeof(dparams)); - dparams.ctx_mode = ctx_mode; - dparams.op_type = op_type; - dparams.numa_id = numa_nodes[numa_idx]; - dparams.bmp = ctx_params->bmp; - dparams.epoll_en = false; - ret = drv->alloc_ctx(attrs->alg, &dparams, &ctx); - if (!ctx || ret < 0) { - if (ret == -WD_ENODEV) - break; + ret = wd_alloc_one_ctx(attrs, drv, &dparams, ctx_idx); + if (ret == -WD_ENODEV) + break; + + if (ret) { WD_ERR("failed to alloc ctx %u from driver %s on numa %d!\n", *ctx_idx, drv->drv_name, numa_nodes[numa_idx]); return ret; } - - internal_config->ctxs[*ctx_idx].ctx = ctx; - internal_config->ctxs[*ctx_idx].op_type = dparams.op_type; - internal_config->ctxs[*ctx_idx].ctx_mode = dparams.ctx_mode; - internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; - internal_config->ctxs[*ctx_idx].drv = drv; - (*ctx_idx)++; } } @@ -2749,6 +2784,142 @@ err_ctxs: return ret; } +/* + * SCHED_POLICY_DEV: open all sync ctxs then all async ctxs of + * (drv, op_type, numa) on ONE device pinned by dparams->preferred_dev_path. + * If the device cannot hold the whole group, roll back every ctx opened on + * it and return -WD_ENODEV so the caller tries the next device. + */ +static int wd_alloc_dev_group(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + struct wd_drv_ctx_params *dparams, + __u32 *ctx_idx) +{ + struct wd_ctx_nums *nums = &attrs->ctx_params->ctx_set_num[dparams->op_type]; + __u32 want[2] = {nums->sync_ctx_num, nums->async_ctx_num}; + __u8 modes[2] = {CTX_MODE_SYNC, CTX_MODE_ASYNC}; + __u32 group_start = *ctx_idx; + __u32 i, j; + int ret; + + for (i = 0; i < 2; i++) { + dparams->ctx_mode = modes[i]; + for (j = 0; j < want[i]; j++) { + ret = wd_alloc_one_ctx(attrs, drv, dparams, ctx_idx); + if (ret) { + wd_free_ctxs_batch_range(attrs, group_start, *ctx_idx); + *ctx_idx = group_start; + if (ret != -WD_EBUSY && ret != -WD_ENODEV) + return ret; + return -WD_ENODEV; + } + } + } + return WD_SUCCESS; +} + +/* + * SCHED_POLICY_DEV: per (drv, op_type, numa) open all sync+async ctxs on one + * device. Enumerates devices on each numa node and pins each group to one + * device via preferred_dev_path, trying the next device if the current one + * cannot hold the whole group. All devices exhausted => error out. + */ +static int wd_alloc_ctxs_batch_dev(struct wd_init_attrs *attrs, __u32 *start_idx) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + struct wd_ctx_params *ctx_params = attrs->ctx_params; + __u32 ctx_idx, drv_idx, start, group_start; + struct uacce_dev_list *dev_list = NULL; + int numa_nodes[UADK_MAX_NUMA_NODES]; + char alg_type[CRYPTO_MAX_ALG_NAME]; + struct wd_drv_ctx_params dparams; + struct wd_alg_driver *drv; + __u8 op_type, numa_count; + struct wd_ctx_nums *nums; + struct uacce_dev_list *p; + int n, ret; + + memset(&dparams, 0, sizeof(dparams)); + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + + start = *start_idx; + ctx_idx = start; + for (drv_idx = 0; drv_idx < internal_config->drv_count; drv_idx++) { + drv = internal_config->drv_array[drv_idx]; + if (!drv || !drv->alloc_ctx) { + WD_ERR("failed to check driver %s alloc_ctx!\n", + drv ? drv->drv_name : "unknown"); + ret = -WD_EINVAL; + goto err_ctxs; + } + if (drv->calc_type != UADK_ALG_HW) + continue; + + ret = wd_get_alg_type(attrs->alg, alg_type); + if (ret) { + ret = -WD_EINVAL; + goto err_ctxs; + } + if (!strcmp(alg_type, "ecc")) + (void)strcpy(alg_type, "sm2"); + if (!strcmp(alg_type, "comp")) + (void)strcpy(alg_type, "zlib"); + + dev_list = wd_get_accel_list(alg_type); + if (!dev_list) + continue; + + wd_collect_numa_nodes(ctx_params->bmp, numa_nodes, &numa_count); + + for (op_type = 0; op_type < ctx_params->op_type_num; op_type++) { + nums = &ctx_params->ctx_set_num[op_type]; + if (!nums->sync_ctx_num && !nums->async_ctx_num) + continue; + + dparams.op_type = op_type; + group_start = ctx_idx; + for (n = 0; n < numa_count; n++) { + dparams.numa_id = numa_nodes[n]; + for (p = dev_list; p; p = p->next) { + if (!p->dev || p->dev->numa_id != numa_nodes[n]) + continue; + + /* Skip devices that cannot hold the whole group */ + if (wd_get_avail_ctx(p->dev) < (int)(nums->sync_ctx_num + + nums->async_ctx_num)) + continue; + + dparams.preferred_dev_path = p->dev->char_dev_path; + ret = wd_alloc_dev_group(attrs, drv, + &dparams, + &ctx_idx); + if (!ret) + break; + if (ret != -WD_ENODEV) + goto err_ctxs; + } + } + + if (ctx_idx == group_start) { + ret = -WD_ENODEV; + goto err_ctxs; + } + } + wd_free_list_accels(dev_list); + dev_list = NULL; + } + + *start_idx = ctx_idx; + return ctx_idx == start ? -WD_ENODEV : WD_SUCCESS; + +err_ctxs: + if (dev_list) + wd_free_list_accels(dev_list); + wd_free_ctxs_batch(attrs, ctx_idx); + return ret; +} + /** * wd_alg_ctx_uninit() - Release ctxs, scheduler, ctx_config. * @@ -2880,18 +3051,24 @@ int wd_alg_ctx_init(struct wd_init_attrs *attrs) return -WD_EINVAL; } - /* - * Ensure that contexts (ctx) with the same attributes are allocated first, - * thereby maintaining queue continuity within the contexts. - */ - ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); - if (ret) - return -WD_EINVAL; + if (attrs->sched_type == SCHED_POLICY_DEV) { + ret = wd_alloc_ctxs_batch_dev(attrs, &ctx_idx); + /* DEV tolerates partial: only a truly empty result fails. */ + if (ret == -WD_ENODEV) + return -WD_EINVAL; - /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ - ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); - if (ret) - return -WD_EINVAL; + if (ret) + return ret; + } else { + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; + + /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; + } /* Backfill actual allocated count */ internal_config = attrs->ctx_config_internal; -- 2.28.0.windows.1

1 3

[PATCH 0/9] uadk_engine: fix several minor issues
by Weili Qian 11 Sep '26

11 Sep '26

Fix several minor issues. Weili Qian (6): uadk_engine: fix double-free in AEAD and async init uadk_engine: use secure BN allocation for signing nonce uadk_engine: fix uninit read in ECDH derive uadk_engine: fix memory leaks in error paths uadk_engine: fix EC key management cleanup and secure allocation uadk_engine: refactor RSA set0 functions and fix keygen BN allocation Zhushuai Yin (3): uadk_provider: fix one-shot cipher padding and stream-mode handling uadk_provider: fix stream mode ciphertext error on non-aligned multi-update uadk_engine: fix stream mode ciphertext error on non-aligned multi-update src/uadk_aead.c | 4 +- src/uadk_async.c | 7 +- src/uadk_cipher.c | 74 ++++++++++++----- src/uadk_digest.c | 11 ++- src/uadk_ec.c | 9 +++ src/uadk_pkey.c | 4 +- src/uadk_prov_aead.c | 3 +- src/uadk_prov_cipher.c | 165 ++++++++++++++++++++++++++++++-------- src/uadk_prov_ec_kmgmt.c | 4 +- src/uadk_prov_ecdh_exch.c | 2 +- src/uadk_prov_ecx.c | 25 +++--- src/uadk_prov_pkey.c | 4 +- src/uadk_prov_rsa_kmgmt.c | 128 +++++++++-------------------- src/uadk_prov_rsa_sign.c | 9 ++- src/uadk_rsa.c | 92 +++++++++++---------- 15 files changed, 314 insertions(+), 227 deletions(-) -- 2.43.0

1 9

[PATCH 00/13] uadk_engine/uadk_provider: bug fixes and AEAD GCM enhancements
by Weili Qian 04 Sep '26

04 Sep '26

From: qianweili <qianweili(a)huawei.com> This series fixes several bugs in the UADK engine and provider, and adds non-block-aligned data handling and TLS 1.x GCM record offload to the provider AES-GCM path. Engine fixes: - Correct error log format and a misleading async macro name (01) - Balance the async recv loop count (02) - Fix wrong return values and missing NULL checks in error paths (03) - Securely release DH/ECDH/SM2 private keys and fix an SM2 order leak (04) - Fall back to software when digest/cipher session allocation fails (05) - Copy md_nid and md_update_status in SM2 ctx copy (06) Provider fixes: - Fix wrong parameter order in wd_xxx_init2_ calls (07) - Split session/key management into the init stage to avoid per-operation alloc/free overhead (08) - Handle non-block-aligned GCM data via an AES-CTR tail soft path (09) - Add TLS 1.x GCM record offload and an IV state machine (10) - Widen the IV buffer and fall back to software for non-default IV lengths (11) - Fix session allocation failure handling in multiple threads (12) - Fix a double-free in digest cleanup under multiple threads (13) Weili Qian (9): uadk_engine: fix error log format coding style and misleading async macro name uadk_engine: fix return value checks in error paths uadk_engine: securely release DH/ECDH/SM2 private key and fix SM2 order leak uadk_engine: copy md_nid and md_update_status in SM2 ctx copy uadk_provider: fix wrong parameter order in wd_xxx_init2_ call uadk_provider: split session/key management into init stage uadk_provider: handle non-block-aligned data via AES-CTR tail soft path uadk_provider: add TLS 1.x GCM record offload and IV state machine uadk_provider: widen IV buffer and fall back to software for non-default IV lengths Wenkai Lin (2): uadk_engine: fall back to software when digest/cipher session alloc fails uadk_provider: fix the double free problem in multiple threads Zhushuai Yin (1): uadk_provider: fix the error of allocating sessions in multiple threads lizhi (1): uadk_engine: adjust async recv loop count to balance async mode behavior src/uadk.h | 2 +- src/uadk_cipher.c | 3 +- src/uadk_dh.c | 8 +- src/uadk_digest.c | 40 +- src/uadk_ec.c | 52 +- src/uadk_ecx.c | 4 +- src/uadk_prov.h | 2 +- src/uadk_prov_aead.c | 1337 +++++++++++++++++++++++++------------ src/uadk_prov_cipher.c | 2 +- src/uadk_prov_dh.c | 6 +- src/uadk_prov_digest.c | 97 ++- src/uadk_prov_ecdh_exch.c | 23 +- src/uadk_prov_ecx.c | 10 +- src/uadk_prov_hmac.c | 4 +- src/uadk_prov_pkey.c | 8 +- src/uadk_prov_rsa_enc.c | 2 +- src/uadk_prov_sm2_enc.c | 2 +- src/uadk_prov_sm2_kmgmt.c | 11 +- src/uadk_prov_sm2_sign.c | 4 +- src/uadk_sm2.c | 18 +- 20 files changed, 1135 insertions(+), 500 deletions(-) -- 2.53.0.windows.2

1 13

[PATCH] uadk: fix -Wmaybe-uninitialized and -Wsign-compare warnings
by Chenghai Huang 02 Sep '26

02 Sep '26

From: Junchong Pan <panjunchong(a)h-partners.com> Initialize ret to 0 in wd_ctx_param_init() to fix -Wmaybe-uninitialized warning. Change ret type from int to size_t in get_str_attr() to fix -Wsign-compare warning. Move sched_uninit and set_param to the end of struct wd_sched to maintain ABI compatibility. Signed-off-by: Junchong Pan <panjunchong(a)h-partners.com> --- include/wd_alg_common.h | 4 ++-- wd.c | 2 +- wd_util.c | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index 05cc5c0..f0e13bb 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -165,13 +165,13 @@ struct wd_sched { const char *name; int sched_policy; handle_t (*sched_init)(handle_t h_sched_ctx, void *sched_param); - void (*sched_uninit)(handle_t h_sched_ctx, handle_t h_sched_key); __u32 (*pick_next_ctx)(handle_t h_sched_ctx, void *sched_key, const int sched_mode); int (*poll_policy)(handle_t h_sched_ctx, __u32 expect, __u32 *count); - void (*set_param)(handle_t h_sched_ctx, void *sched_key, void *sched_param); handle_t h_sched_ctx; + void (*sched_uninit)(handle_t h_sched_ctx, handle_t h_sched_key); + void (*set_param)(handle_t h_sched_ctx, void *sched_key, void *sched_param); }; typedef int (*wd_alg_init)(struct wd_ctx_config *config, struct wd_sched *sched); diff --git a/wd.c b/wd.c index f973c70..1c28bb3 100644 --- a/wd.c +++ b/wd.c @@ -203,7 +203,7 @@ static int get_int_attr(struct uacce_dev *dev, const char *attr, int *val) static int get_str_attr(struct uacce_dev *dev, const char *attr, char *buf, size_t buf_sz) { - int ret; + size_t ret; int size; size = get_raw_attr(dev->dev_root, attr, buf, buf_sz); diff --git a/wd_util.c b/wd_util.c index 1b4c74e..bf6d380 100644 --- a/wd_util.c +++ b/wd_util.c @@ -1765,7 +1765,7 @@ static int wd_env_set_ctx_nums(const char *alg_name, const char *name, const cha { char *left, *section, *start; int is_comp; - int ret; + int ret = 0; /* COMP environment variable's format is different, mark it */ is_comp = strncmp(name, "WD_COMP_CTX_NUM", sizeof("WD_COMP_CTX_NUM") - 1) ? 0 : 1; @@ -1799,7 +1799,8 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, { const char *env_name = wd_env_name[type]; const char *var_s; - int i, ret; + int ret = 0; + int i; ctx_params->bmp = numa_allocate_nodemask(); if (!ctx_params->bmp) { -- 2.43.0

1 0

[PATCH] uadk_tool: fix segfault in digest/aead init1 from uninitialized wd_sched
by Chenghai Huang 29 Aug '26

29 Aug '26

From: Weili Qian <qianweili(a)huawei.com> digest_init1() and aead_init1() in uadk_tool/test/test_sec.c declared "struct wd_sched sched;" on the stack without initialization, leaving sched_policy and callback fields as garbage. This bypassed the SCHED_POLICY_NONE/SINGLE rejection in wd_digest_init/wd_aead_init and led to a segfault on the first wd_do_digest_sync/async call. Zero-initialize the struct so sched_policy defaults to SCHED_POLICY_RR and all callbacks/handles are NULL. Signed-off-by: Weili Qian <qianweili(a)huawei.com> --- uadk_tool/test/test_sec.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uadk_tool/test/test_sec.c b/uadk_tool/test/test_sec.c index 462ba7a..6e1b990 100644 --- a/uadk_tool/test/test_sec.c +++ b/uadk_tool/test/test_sec.c @@ -1416,7 +1416,7 @@ static __u32 sched_digest_pick_next_ctx(handle_t h_sched_ctx, static int digest_init1(int type, int mode) { struct uacce_dev_list *list; - struct wd_sched sched; + struct wd_sched sched = {0}; int ret; if (g_use_env) @@ -2685,7 +2685,7 @@ static __u32 sched_aead_pick_next_ctx(handle_t h_sched_ctx, static int aead_init1(int type, int mode) { struct uacce_dev_list *list; - struct wd_sched sched; + struct wd_sched sched = {0}; int ret; if (g_use_env) -- 2.33.0

1 0

[PATCH 01/16] uadk: rewrite scheduler with scheduling domain solution
by Chenghai Huang 25 Aug '26

25 Aug '26

From: Longfang Liu <liulongfang(a)huawei.com> Conduct a comprehensive update of the scheduler's internal triple-array-based pooling scheme. Adopt new data organization methods such as hash buckets and segment linked lists to achieve efficient insertion and query operations. Implement pre-fetch queue processing for business thread sessions, creating thread-level sub-queue pools for fast packet reception queries. Provide new scheduling algorithms for the new UADK framework. Signed-off-by: Longfang Liu <liulongfang(a)huawei.com> Signed-off-by: Chenghai Huang <huangchenghai2(a)huawei.com> Signed-off-by: Wenkai Lin <linwenkai6(a)hisilicon.com> --- include/wd_alg.h | 11 +- include/wd_alg_common.h | 5 +- include/wd_internal.h | 18 +- include/wd_sched.h | 10 +- wd_sched.c | 2304 ++++++++++++++++++++++++++++++--------- 5 files changed, 1832 insertions(+), 516 deletions(-) diff --git a/include/wd_alg.h b/include/wd_alg.h index 18503ca..1ae1dae 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -64,10 +64,13 @@ extern "C" { #endif enum alg_dev_type { - UADK_ALG_SOFT = 0x0, - UADK_ALG_CE_INSTR = 0x1, - UADK_ALG_SVE_INSTR = 0x2, - UADK_ALG_HW = 0x3 + UADK_ALG_HW = 0x0, + UADK_ALG_CE_INSTR = 0x1, + UADK_ALG_SVE_INSTR = 0x2, + UADK_ALG_SOFT = 0x3, + UADK_ALG_NPU = 0x4, + UADK_ALG_GPU = 0x5, + UADK_ALG_TYPE_MAX, }; /* diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index a294877..ebba7f8 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -41,6 +41,7 @@ extern "C" { /* Key size of digest */ #define MAX_HMAC_KEY_SIZE 128U +#define STATUS_ENABLE (void *)0x1 enum alg_task_type { TASK_MIX = 0x0, @@ -140,7 +141,7 @@ struct wd_ctx_params { }; /* - * struct wd_comp_sched - Define a scheduler. + * struct wd_sched - Define a scheduler. * @name: Name of this scheduler. * @sched_policy: Method for scheduler to perform scheduling * @sched_init: inited the scheduler input parameters. @@ -157,10 +158,12 @@ struct wd_sched { const char *name; int sched_policy; handle_t (*sched_init)(handle_t h_sched_ctx, void *sched_param); + void (*sched_uninit)(handle_t h_sched_ctx, handle_t h_sched_key); __u32 (*pick_next_ctx)(handle_t h_sched_ctx, void *sched_key, const int sched_mode); int (*poll_policy)(handle_t h_sched_ctx, __u32 expect, __u32 *count); + void (*set_param)(handle_t h_sched_ctx, void *sched_key, void *sched_param); handle_t h_sched_ctx; }; diff --git a/include/wd_internal.h b/include/wd_internal.h index 4f19d3a..62cf1a9 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -7,8 +7,10 @@ #define WD_INTERNAL_H #include <pthread.h> +#include <stdatomic.h> #include <stdbool.h> #include "wd.h" +#include "wd_alg.h" #ifdef __cplusplus extern "C" { @@ -41,11 +43,14 @@ struct wd_ce_ctx { }; struct wd_ctx_internal { - handle_t ctx; __u8 op_type; __u8 ctx_mode; + __u8 ctx_type; + handle_t ctx; __u16 sqn; pthread_spinlock_t lock; + struct wd_alg_driver *drv; + __u32 hw_load; }; struct wd_ctx_config_internal { @@ -64,6 +69,17 @@ struct wd_datalist { struct wd_datalist *next; }; +struct wd_sched_params { + __u32 pkt_size; + /* block mode or stream mode */ + __u16 data_mode; + __u16 prio_mode; + + /* Compat filtering parameters for session-ctx matching */ + const char *alg_name; + struct wd_ctx_internal *ctxs; +}; + int memcmp_consttime(const void *s1, const void *s2, size_t n); #ifdef __cplusplus diff --git a/include/wd_sched.h b/include/wd_sched.h index 5baecd3..f81025f 100644 --- a/include/wd_sched.h +++ b/include/wd_sched.h @@ -13,7 +13,8 @@ extern "C" { #endif -#define INVALID_POS 0xFFFFFFFF +#define INVALID_POS 0xFFFF +#define QUEUE_FULL_POS 0x1FFF /* The global policy type */ enum sched_policy_type { @@ -25,6 +26,12 @@ enum sched_policy_type { SCHED_POLICY_SINGLE, /* requests will be sent to ctxs and dev_id */ SCHED_POLICY_DEV, + /* Hard calculation and soft calculation interval loop call */ + SCHED_POLICY_LOOP, + /* Perform heterogeneous calculations through ctx of session key */ + SCHED_POLICY_HUNGRY, + /* Instructions to accelerate heterogeneous computing */ + SCHED_POLICY_INSTR, SCHED_POLICY_BUTT, }; @@ -35,6 +42,7 @@ struct sched_params { __u32 begin; __u32 end; __u32 dev_id; + int ctx_prop; }; typedef int (*user_poll_func)(__u32 pos, __u32 expect, __u32 *count); diff --git a/wd_sched.c b/wd_sched.c index 19936fd..2efa2aa 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -1,374 +1,1442 @@ // SPDX-License-Identifier: Apache-2.0 /* - * Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved. * Copyright 2020-2021 Linaro ltd. + * + * Scheduler: Simplified Pure Hash Table with Dynamic Context Expansion + * + * Key improvements: + * - Single global hash table with (region_id, mode, op_type, prop) dimensions + * - Segment list for non-contiguous ctx ranges + * - Dual-domain queues for session key. + * - Dynamic ctx expansion in HUNGRY mode based on load threshold + * - Packet reception is handled through the active queues in the session key. + * - Simplified sched_init: only allocate one sync + one async ctx + * - Removed redundant wd_sched_info layer */ #define _GNU_SOURCE #include <stdlib.h> #include <stdbool.h> +#include <string.h> #include <sched.h> #include <numa.h> +#include <limits.h> +#include <pthread.h> #include "wd_sched.h" +#include "wd_alg.h" +#include "wd_internal.h" -#define MAX_POLL_TIMES 1000 +#define MAX_POLL_TIMES 1000 +#define HUNGRY_LOAD_THRESHOLD 256 +#define SKEY_CTX_MAX_NUM 16 +#define SKEY_MAX_THREAD_NUM 64 +#define SKEY_LOAD_UPDATE_INTERVAL 1 +#define HW_QUEUE_FULL_DEPTH 1024 +#define MAX_NUMA_NODES (NUMA_NUM_NODES >> 5) + +/* ============================================================================ + * Hash Table Configuration + * ============================================================================ + */ +#define WD_SCHED_MAX_BUCKETS 512 +#define WD_SCHED_MIN_BUCKETS 32 +#define WD_SCHED_LOAD_FACTOR 0.75f +#define HASH_PRIME1 73 +#define HASH_PRIME2 13 +#define HASH_PRIME3 7 +#define HASH_PRIME4 11 + +/* ============================================================================ + * Scheduling Region Mode + * ============================================================================ + */ enum sched_region_mode { SCHED_MODE_SYNC = 0, SCHED_MODE_ASYNC = 1, SCHED_MODE_BUTT }; -/* - * sched_key - The key if schedule region. - * @numa_id: The schedule numa region id. - * @mode: Sync mode:0, async_mode:1 - * @type: Service type , the value must smaller than type_num. - * @sync_ctxid: alloc ctx id for sync mode - * @async_ctxid: alloc ctx id for async mode - */ -struct sched_key { - int numa_id; - __u8 type; - __u8 mode; - __u32 sync_ctxid; - __u32 async_ctxid; - __u32 dev_id; -}; +/* ============================================================================ + * Segment List for Domain Index Organization + * ============================================================================ + */ -/* - * struct sched_ctx_range - define one ctx pos. - * @begin: the start pos in ctxs of config. - * @end: the end pos in ctxx of config. - * @last: the last one which be distributed. - * @valid: the region used flag. - * @lock: lock the currentscheduling region. - */ -struct sched_ctx_region { +/** + * wd_sched_ctx_segment - Contiguous segment of ctx indices in domain + * @begin: Start index of this segment + * @end: End index of this segment (inclusive) + * @next: Pointer to next segment in the linked list + * + * Supports non-contiguous ctx ranges via segment list. + */ +struct wd_sched_ctx_segment { __u32 begin; __u32 end; - __u32 last; + struct wd_sched_ctx_segment *next; +}; + +/* ============================================================================ + * Session key domain cache processing. + * ============================================================================ + */ + +/** + * wd_sched_domain_idx_cache - Simplified fixed array cache for skey domains + * + * Design principles: + * - Fixed array for cache-friendly memory layout + * - Atomic operations for lock-free load tracking + * - Simple RR and load balancing strategies + * - Maximum 16 queues per thread (typical usage) + */ +struct wd_sched_domain_idx_cache { + /* Queue index array */ + __u32 idx_list[SKEY_CTX_MAX_NUM]; /* Array of ctx indices */ + __u32 load_values[SKEY_CTX_MAX_NUM]; /* Atomic load counters */ + __u32 valid_count; /* Number of valid queues */ + + /* Scheduling state */ + __u32 rr_ptr; /* Round-robin pointer */ + __u32 min_load_idx; /* Cached min load index */ + __u32 op_counter; /* Operation counter for updates */ + __u8 load_decreased; /* Non-zero if poll ever decremented load */ + + /* Configuration */ + __u32 update_interval; /* Min load update interval */ + __u8 policy; /* Scheduling policy */ + + /* Synchronization */ + pthread_mutex_t cache_lock; /* Lock for structure modifications */ +}; + +/** + * wd_sched_ctx_domain - Scheduling domain with four dimensions + * @region_id: Region identifier (numa_id or device_id) + * @mode: Context mode (SYNC/ASYNC) + * @op_type: Operation type + * @prop: Property (e.g., device type: HW, CE, SOFT) + * @segments: Linked list of context ranges + * @segment_count: Number of segments + * @total_ctx_count: Total contexts across all segments + * @current_segment: Current segment pointer for round-robin + * @current_pos: Current position within segment + * @valid: Domain validity flag + * @lock: Synchronization spinlock + */ +struct wd_sched_ctx_domain { + int region_id; + __u8 mode; + __u32 op_type; + __u8 prop; + + struct wd_sched_ctx_segment *segments; + __u32 segment_count; + __u32 total_ctx_count; + + struct wd_sched_ctx_segment *current_segment; + __u32 current_pos; bool valid; + pthread_mutex_t lock; }; -/* - * wd_sched_info - define the context of the scheduler. - * @ctx_region: define the map for the comp ctxs, using for quickly search. - * the x range: two(sync and async), the y range: - * two(e.g. comp and uncomp) the map[x][y]'s value is the ctx - * begin and end pos. - * @valid: the region used flag. - */ -struct wd_sched_info { - struct sched_ctx_region *ctx_region[SCHED_MODE_BUTT]; - bool valid; +/** + * wd_sched_domain_hash_node - Hash table collision chain node + */ +struct wd_sched_domain_hash_node { + struct wd_sched_ctx_domain domain; + struct wd_sched_domain_hash_node *next; }; -struct dev_region_map { +/** + * wd_sched_domain_hash_table - Pure dynamic hash table for scheduling domains + * @buckets: Hash table bucket array + * @bucket_size: Number of buckets + * @lock: Read-write lock for concurrent access + */ +struct wd_sched_domain_hash_table { + struct wd_sched_domain_hash_node **buckets; + __u32 bucket_size; + pthread_mutex_t lock; +}; + +/* ============================================================================ + * Dual-Domain Structure for Session Key + * ============================================================================ + */ + +/** + * wd_sched_key_domain - Session domain with min-heap + * @idx_cache: Index cache with min-heap for load-based selection + * @lock: Synchronization spinlock + * @expanded_count: Track how many times ctx has been expanded + */ +struct wd_sched_key_domain { + struct wd_sched_domain_idx_cache idx_cache; + pthread_mutex_t lock; + __u32 expanded_count; +}; + +/** + * wd_sched_key - Session-level scheduling key + * @region_id: Region identifier + * @type: Operation type + * @mode: Current mode (SYNC/ASYNC) + * @dev_id: Device identifier (for SCHED_POLICY_DEV) + * @ctx_prop: Context property + * @is_stream: Stream mode flag + * @prio_mode: Priority mode + * @pkt_size: Current packet size + * @sync_domain: Min-heap domain for sync contexts + * @async_domain: Min-heap domain for async contexts + * @lock: Synchronization spinlock + */ +struct wd_sched_key { + int region_id; + __u8 type; + __u8 mode; __u32 dev_id; - __u32 region_id; + __u8 ctx_prop; + __u16 is_stream; + __u16 prio_mode; + __u32 pkt_size; + + struct wd_sched_key_domain sync_domain; + struct wd_sched_key_domain async_domain; + + pthread_mutex_t lock; + __u32 poll_lock; + __u32 refcount; + + /* Compat filtering parameters for session-ctx matching */ + const char *alg_name; + struct wd_ctx_internal *ctxs; }; -/* - * wd_sched_ctx - define the context of the scheduler. - * @policy: define the policy of the scheduler. - * @numa_num: the max numa numbers of the scheduler. - * @type_num: the max operation types of the scheduler. - * @poll_func: the task's poll operation function. - * @numa_map: a map of cpus to devices. - * @sched_info: the context of the scheduler. + +/** + * wd_sched_ctx - Main scheduler context + * @policy: Scheduling policy type + * @type_num: Number of operation types + * @mode_num: Number of modes (SYNC/ASYNC) + * @region_num: Number of regions (numa or devices) + * @poll_func: Poll function for receiving responses + * @domain_hash_table: Global hash table for all domains + * @skey_num: Number of active session keys + * @skey_lock: Lock for skey array + * @skey: Array of session keys */ struct wd_sched_ctx { __u32 policy; __u32 type_num; - __u16 numa_num; - __u16 dev_num; + __u32 mode_num; + __u16 region_num; + user_poll_func poll_func; - int numa_map[NUMA_NUM_NODES]; - struct dev_region_map dev_id_map[DEVICE_REGION_MAX]; - struct wd_sched_info sched_info[0]; + struct wd_sched_domain_hash_table *domain_hash_table; + + __u32 skey_num; + pthread_mutex_t skey_lock; + struct wd_sched_key *skey[SKEY_MAX_THREAD_NUM]; }; -static bool sched_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +/* ============================================================================ + * Hash Table Core Operations + * ============================================================================ + */ + +static bool wd_sched_is_prime(__u32 n) { - if (key->numa_id >= sched_ctx->numa_num || key->mode >= SCHED_MODE_BUTT || - key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's numa: %d, mode: %u, type: %u!\n", - key->numa_id, key->mode, key->type); + __u32 i; + + if (n <= 1) return false; + if (n <= 3) + return true; + if (n % 2 == 0 || n % 3 == 0) + return false; + + for (i = 5; i * i <= n; i += 6) { + if (n % i == 0 || n % (i + 2) == 0) + return false; } return true; } -/* - * sched_get_ctx_range - Get ctx range from ctx_map by the wd comp arg +static __u32 wd_sched_find_prime(__u32 n) +{ + while (!wd_sched_is_prime(n)) + n++; + return n; +} + +static __u32 wd_sched_compute_bucket_size(__u32 estimated_entries) +{ + __u32 target_size; + + target_size = (estimated_entries * 4) / 3; + + if (target_size < WD_SCHED_MIN_BUCKETS) + target_size = WD_SCHED_MIN_BUCKETS; + if (target_size > WD_SCHED_MAX_BUCKETS) + target_size = WD_SCHED_MAX_BUCKETS; + + return wd_sched_find_prime(target_size); +} + +/** + * wd_sched_hash_compute - Compute hash value for four-dimensional domain key + * @region_id: Region identifier + * @mode: Context mode + * @op_type: Operation type + * @prop: Property + * @bucket_size: Hash table bucket count + * + * Combines four dimensions using prime number multipliers. */ -static struct sched_ctx_region *sched_get_ctx_range(struct wd_sched_ctx *sched_ctx, - const struct sched_key *key) +static inline __u32 wd_sched_hash_compute(int region_id, __u8 mode, + __u32 op_type, __u8 prop, __u32 bucket_size) { - struct wd_sched_info *sched_info; - int numa_id; + __u32 hash; - sched_info = sched_ctx->sched_info; - if (key->numa_id >= 0 && - sched_info[key->numa_id].ctx_region[key->mode][key->type].valid) - return &sched_info[key->numa_id].ctx_region[key->mode][key->type]; + hash = (region_id * HASH_PRIME1) + (mode * HASH_PRIME2) + + (op_type * HASH_PRIME3) + (prop * HASH_PRIME4); + return hash % bucket_size; +} + +static inline bool wd_sched_domain_key_match( + int region_id1, __u8 mode1, __u32 op_type1, __u8 prop1, + int region_id2, __u8 mode2, __u32 op_type2, __u8 prop2) +{ + return (region_id1 == region_id2 && mode1 == mode2 && + op_type1 == op_type2 && prop1 == prop2); +} + +/** + * wd_sched_hash_table_create - Create hash table + * @estimated_entries: Estimated number of entries + * + * Returns: Initialized hash table or NULL on error + */ +static struct wd_sched_domain_hash_table * +wd_sched_hash_table_create(__u32 estimated_entries) +{ + struct wd_sched_domain_hash_table *table; + __u32 bucket_size; + int ret; - /* If the key->numa_id is not exist, we should scan for a region */ - for (numa_id = 0; numa_id < sched_ctx->numa_num; numa_id++) { - if (sched_info[numa_id].ctx_region[key->mode][key->type].valid) - return &sched_info[numa_id].ctx_region[key->mode][key->type]; + table = calloc(1, sizeof(*table)); + if (!table) + return NULL; + + bucket_size = wd_sched_compute_bucket_size(estimated_entries); + + table->buckets = calloc(bucket_size, sizeof(*table->buckets)); + if (!table->buckets) { + free(table); + return NULL; } - return NULL; + table->bucket_size = bucket_size; + ret = pthread_mutex_init(&table->lock, NULL); + if (ret) { + free(table->buckets); + free(table); + return NULL; + } + + return table; } -/* - * sched_get_next_pos_rr - Get next resource pos by RR schedule. - * The second para is reserved for future. +static void wd_sched_hash_table_destroy(struct wd_sched_domain_hash_table *table) +{ + struct wd_sched_domain_hash_node *node, *next; + struct wd_sched_ctx_segment *seg, *next_seg; + __u32 i; + + if (!table) + return; + + for (i = 0; i < table->bucket_size; i++) { + node = table->buckets[i]; + while (node) { + next = node->next; + + /* Release segment linked list */ + seg = node->domain.segments; + while (seg) { + next_seg = seg->next; + free(seg); + seg = next_seg; + } + + pthread_mutex_destroy(&node->domain.lock); + free(node); + node = next; + } + } + + pthread_mutex_destroy(&table->lock); + free(table->buckets); + free(table); +} + +static struct wd_sched_ctx_domain * +wd_sched_hash_table_lookup(struct wd_sched_domain_hash_table *table, + int region_id, __u8 mode, __u32 op_type, __u8 prop) +{ + struct wd_sched_domain_hash_node *node; + struct wd_sched_ctx_domain *domain = NULL; + __u32 hash_idx; + + if (!table) + return NULL; + + pthread_mutex_lock(&table->lock); + hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size); + node = table->buckets[hash_idx]; + while (node) { + if (wd_sched_domain_key_match( + node->domain.region_id, node->domain.mode, node->domain.op_type, + node->domain.prop, region_id, mode, op_type, prop)) { + domain = &node->domain; + break; + } + node = node->next; + } + pthread_mutex_unlock(&table->lock); + + return domain; +} + +static struct wd_sched_ctx_domain * +wd_sched_hash_table_insert(struct wd_sched_domain_hash_table *table, + int region_id, __u8 mode, __u32 op_type, __u8 prop) +{ + struct wd_sched_domain_hash_node *new_node; + struct wd_sched_ctx_domain *existing; + __u32 hash_idx; + int ret; + + if (!table) + return NULL; + + existing = wd_sched_hash_table_lookup(table, region_id, mode, op_type, prop); + if (existing) + return existing; + + pthread_mutex_lock(&table->lock); + hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size); + /* Alloc and initialize new domain */ + new_node = calloc(1, sizeof(*new_node)); + if (!new_node) { + pthread_mutex_unlock(&table->lock); + return NULL; + } + + /* Initialize new domain */ + new_node->domain.region_id = region_id; + new_node->domain.mode = mode; + new_node->domain.op_type = op_type; + new_node->domain.prop = prop; + new_node->domain.segments = NULL; + new_node->domain.segment_count = 0; + new_node->domain.total_ctx_count = 0; + new_node->domain.current_segment = NULL; + new_node->domain.current_pos = 0; + new_node->domain.valid = false; + + ret = pthread_mutex_init(&new_node->domain.lock, NULL); + if (ret) { + pthread_mutex_unlock(&table->lock); + free(new_node); + return NULL; + } + + new_node->next = table->buckets[hash_idx]; + table->buckets[hash_idx] = new_node; + pthread_mutex_unlock(&table->lock); + + return &new_node->domain; +} + +/* ============================================================================ + * Segment List Operations + * ============================================================================ + */ + +/** + * wd_sched_domain_add_segment - Add context range segment to domain + * @domain: Target domain + * @begin: Start context index + * @end: End context index (inclusive) + * + * Supports non-contiguous context ranges via segment list. */ -static __u32 sched_get_next_pos_rr(struct sched_ctx_region *region, void *para) +static int wd_sched_domain_add_segment(struct wd_sched_ctx_domain *domain, + __u32 begin, __u32 end) { + struct wd_sched_ctx_segment *seg, *new_seg; + + if (!domain || begin > end) + return -WD_EINVAL; + + new_seg = calloc(1, sizeof(*new_seg)); + if (!new_seg) + return -WD_ENOMEM; + + new_seg->begin = begin; + new_seg->end = end; + new_seg->next = NULL; + + pthread_mutex_lock(&domain->lock); + + /* Append to segment list tail */ + if (!domain->segments) { + domain->segments = new_seg; + } else { + seg = domain->segments; + while (seg->next) + seg = seg->next; + seg->next = new_seg; + } + + domain->segment_count++; + domain->total_ctx_count += (end - begin + 1); + + /* Initialize polling state */ + if (!domain->current_segment) + domain->current_segment = domain->segments; + + pthread_mutex_unlock(&domain->lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_domain_get_next_rr - Get next context via round-robin from domain + * @domain: Source domain + * + * Returns: Next global queue index in round-robin order + * Time complexity: O(1) + */ +static __u32 wd_sched_domain_get_next_rr(struct wd_sched_ctx_domain *domain) +{ + __u32 ctx_idx; __u32 pos; - pthread_mutex_lock(&region->lock); + if (!domain || !domain->segments || !domain->total_ctx_count) + return INVALID_POS; - pos = region->last; + pthread_mutex_lock(&domain->lock); + if (!domain->current_segment) + domain->current_segment = domain->segments; - if (pos < region->end) - region->last++; - else - region->last = region->begin; + pos = domain->current_pos; + + /* Calculate global queue number: segment.begin + relative position */ + ctx_idx = domain->current_segment->begin + pos; - pthread_mutex_unlock(&region->lock); + /* Move to next position */ + if (pos + 1 < domain->current_segment->end - domain->current_segment->begin + 1) { + /* Within same segment */ + domain->current_pos = pos + 1; + } else if (domain->current_segment->next) { + /* Move to next segment */ + domain->current_segment = domain->current_segment->next; + domain->current_pos = 0; + } else { + /* Loop back to beginning */ + domain->current_segment = domain->segments; + domain->current_pos = 0; + } + + pthread_mutex_unlock(&domain->lock); - return pos; + return ctx_idx; } -/* - * session_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/* ============================================================================ + * SKey Domain Cache Management Functions + * ============================================================================ + */ +/** + * wd_sched_skey_cache_init - Initialize skey domain cache + * @cache: Pointer to cache structure + * @policy: Scheduling policy * @sched_type: Scheduling policy type (cannot modify per + * API contract) + * + * Initialize fixed array cache with invalid positions and zero loads. + */ +static int wd_sched_skey_cache_init(struct wd_sched_domain_idx_cache *cache, + __u8 policy) +{ + int i; + + if (!cache) { + WD_ERR("invalid: cache pointer is NULL!\n"); + return -WD_EINVAL; + } + + /* Initialize array with invalid positions */ + for (i = 0; i < SKEY_CTX_MAX_NUM; i++) { + cache->idx_list[i] = INVALID_POS; + __atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED); + } + + /* Initialize atomic counters */ + __atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->op_counter, 0, __ATOMIC_RELAXED); + cache->load_decreased = 0; + + /* Set configuration */ + cache->valid_count = 0; + cache->update_interval = SKEY_LOAD_UPDATE_INTERVAL; + cache->policy = policy; + + /* Initialize structure lock */ + if (pthread_mutex_init(&cache->cache_lock, NULL)) { + WD_ERR("failed to init cache lock!\n"); + return -WD_EINVAL; + } + + return WD_SUCCESS; +} +/** + * wd_sched_skey_cache_uninit - Cleanup skey domain cache + * @cache: Pointer to cache structure + * + * Release resources and reset cache state. + */ +static void wd_sched_skey_cache_uninit(struct wd_sched_domain_idx_cache *cache) +{ + if (!cache) + return; + + pthread_mutex_destroy(&cache->cache_lock); + + /* Reset cache state */ + for (int i = 0; i < SKEY_CTX_MAX_NUM; i++) { + cache->idx_list[i] = INVALID_POS; + __atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED); + } + + cache->valid_count = 0; + cache->load_decreased = 0; +} +/** + * wd_sched_skey_add_ctx - Add ctx to skey domain cache + * @cache: Pointer to cache structure + * @ctx_id: Context ID to add + * + * Add ctx to next available position in fixed array. + * Returns 0 on success, negative error code on failure. + */ +static int wd_sched_skey_add_ctx(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_id) +{ + __u32 i; + + if (!cache || ctx_id == INVALID_POS) { + WD_ERR("invalid: parameters are NULL!\n"); + return -WD_EINVAL; + } + + pthread_mutex_lock(&cache->cache_lock); + /* Check if cache is full */ + if (cache->valid_count >= SKEY_CTX_MAX_NUM) { + pthread_mutex_unlock(&cache->cache_lock); + WD_ERR("invalid: skey cache full, cannot add more queues!\n"); + return -WD_EINVAL; + } + + /* Check for duplicate ctx_id */ + for (i = 0; i < cache->valid_count; i++) { + if (cache->idx_list[i] == ctx_id) { + WD_ERR("invalid: context %u already exists in skey cache at pos %u!\n", + ctx_id, i); + pthread_mutex_unlock(&cache->cache_lock); + return -WD_EEXIST; + } + } + + /* Update min load index if as the new ctx */ + __atomic_store_n(&cache->min_load_idx, cache->valid_count, __ATOMIC_RELAXED); + + /* Add to next available position */ + cache->idx_list[cache->valid_count] = ctx_id; + __atomic_store_n(&cache->load_values[cache->valid_count], 0, __ATOMIC_RELAXED); + cache->valid_count++; + pthread_mutex_unlock(&cache->cache_lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_skey_remove_ctx - Remove ctx from skey domain cache + * @cache: Pointer to cache structure + * @ctx_id: Context ID to remove + * + * Remove ctx by shifting array elements to maintain continuity. + * Returns 0 on success, negative error code if not found. + */ +static int wd_sched_skey_remove_ctx(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_id) +{ + __u32 i, current_min; + int found = 0; + + if (!cache) { + WD_ERR("invalid: cache pointer is NULL!\n"); + return -WD_EINVAL; + } + + pthread_mutex_lock(&cache->cache_lock); + /* Find and remove the ctx */ + for (i = 0; i < cache->valid_count; i++) { + if (cache->idx_list[i] == ctx_id) { + found = 1; + break; + } + } + + if (!found) { + WD_ERR("invalid: context %u not found in skey cache!\n", ctx_id); + pthread_mutex_unlock(&cache->cache_lock); + return -WD_ENODEV; + } + + /* Shift remaining elements to fill the gap */ + for (; i < cache->valid_count - 1; i++) { + cache->idx_list[i] = cache->idx_list[i + 1]; + __atomic_store_n(&cache->load_values[i], + __atomic_load_n(&cache->load_values[i + 1], __ATOMIC_RELAXED), + __ATOMIC_RELAXED); + } + + /* Clear last position */ + cache->idx_list[cache->valid_count - 1] = INVALID_POS; + __atomic_store_n(&cache->load_values[cache->valid_count - 1], 0, __ATOMIC_RELAXED); + cache->valid_count--; + + /* Reset pointers if cache becomes empty */ + if (!cache->valid_count) { + __atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + } else { + /* Adjust min load index if necessary */ + current_min = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED); + if (current_min >= cache->valid_count) + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + } + pthread_mutex_unlock(&cache->cache_lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_update_min_load - Update cached min load index + * @cache: Pointer to cache structure + * @ctxs: Context array for hw_load lookup, or NULL to use cache->load_values * - * The user must init the schedule info through wd_sched_rr_instance + * Scan valid queues to find the one with minimum load. + * When ctxs is provided, reads per-ctx hw_load for cross-session accuracy. */ -static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, - const int sched_mode) +static void wd_sched_update_min_load(struct wd_sched_domain_idx_cache *cache, + struct wd_ctx_internal *ctxs) { - struct sched_ctx_region *region = NULL; - bool ret; + __u32 min_load = UINT_MAX; + __u32 min_idx = 0; + __u32 i, load; - key->mode = sched_mode; - ret = sched_key_valid(sched_ctx, key); - if (!ret) + if (!cache->valid_count) + return; + + for (i = 0; i < cache->valid_count; i++) { + if (ctxs) + load = __atomic_load_n(&ctxs[cache->idx_list[i]].hw_load, + __ATOMIC_RELAXED); + else + load = __atomic_load_n(&cache->load_values[i], __ATOMIC_RELAXED); + if (load < min_load) { + min_load = load; + min_idx = i; + } + } + + __atomic_store_n(&cache->min_load_idx, min_idx, __ATOMIC_RELAXED); +} + +/** + * wd_sched_skey_pick_next - Pick next ctx from skey domain cache + * @cache: Pointer to cache structure + * @ctx_idx: Output index within cache array + * @ctxs: Context array for hw_load lookup, or NULL + * + * Select next ctx based on scheduling policy: + * - RR: Simple round-robin selection + * - HUNGRY: Choose ctx with minimum load (hw_load if ctxs available) + * + * Returns selected ctx index, or INVALID_POS if no valid ctx. + */ +static __u32 wd_sched_skey_pick_next(struct wd_sched_domain_idx_cache *cache, + __u32 *ctx_idx, struct wd_ctx_internal *ctxs) +{ + __u32 selected_idx; + __u32 op_count; + + if (!cache || !cache->valid_count) return INVALID_POS; - region = sched_get_ctx_range(sched_ctx, key); - if (!region) + switch (cache->policy) { + case SCHED_POLICY_RR: + case SCHED_POLICY_NONE: + case SCHED_POLICY_SINGLE: + case SCHED_POLICY_DEV: + case SCHED_POLICY_LOOP: + case SCHED_POLICY_INSTR: + /* Round-robin: atomic increment and module */ + selected_idx = __atomic_fetch_add(&cache->rr_ptr, 1, __ATOMIC_RELAXED) % + cache->valid_count; + break; + case SCHED_POLICY_HUNGRY: + /* Update min load periodically */ + op_count = __atomic_fetch_add(&cache->op_counter, 1, __ATOMIC_RELAXED); + if (op_count % cache->update_interval == 0) + wd_sched_update_min_load(cache, ctxs); + + /* Load balancing: use cached min load index */ + selected_idx = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED); + break; + default: + WD_ERR("invalid: unknown scheduling policy %d!\n", cache->policy); + selected_idx = INVALID_POS; + break; + } + + /* Ensure index is within valid range */ + if (selected_idx >= cache->valid_count) return INVALID_POS; - return sched_get_next_pos_rr(region, NULL); + *ctx_idx = selected_idx; + return cache->idx_list[selected_idx]; } -static handle_t session_sched_init(handle_t h_sched_ctx, void *sched_param) +/** + * wd_sched_skey_update_load - Update load for a specific ctx + * @cache: Pointer to cache structure + * @ctx_idx: Context index in list + * @delta: Load delta (positive for send, negative for receive) + * + * Atomically update load counter for the specified ctx. + * Returns 0 on success, negative error code if ctx not found. + */ +static int wd_sched_skey_update_load(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_idx, int delta) { - struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - struct sched_params *param = (struct sched_params *)sched_param; - struct sched_key *skey; + /* Atomic update without locking, ctx_idx's value is guaranteed by the caller. */ + if (delta > 0) + __atomic_fetch_add(&cache->load_values[ctx_idx], delta, __ATOMIC_RELAXED); + else { + __atomic_fetch_sub(&cache->load_values[ctx_idx], -delta, __ATOMIC_RELAXED); + if (unlikely(!cache->load_decreased)) + cache->load_decreased = 1; + } + return WD_SUCCESS; +} + +/* ============================================================================ + * Session Key Domain Initialization + * ============================================================================ + */ + +/** + * wd_sched_skey_domain_init - Initialize session domain with min-heap + * @key_domain: Target key domain + * @ctx_idx: context indices idx + * @policy: current session's policy + * + * Initializes dual-domain structure for session. + */ +static int wd_sched_skey_domain_init(struct wd_sched_key_domain *key_domain, + __u32 ctx_idx, __u8 policy) +{ + int ret; + + if (!key_domain) + return -WD_EINVAL; + + ret = wd_sched_skey_cache_init(&key_domain->idx_cache, policy); + if (ret) + return ret; + + ret = wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx); + if (ret) + goto init_err; + + ret = pthread_mutex_init(&key_domain->lock, NULL); + if (ret) + goto add_ctx_err; + + key_domain->expanded_count = 0; + + return WD_SUCCESS; + +add_ctx_err: + wd_sched_skey_remove_ctx(&key_domain->idx_cache, ctx_idx); +init_err: + wd_sched_skey_cache_uninit(&key_domain->idx_cache); + return ret; +} + +/** + * wd_sched_skey_domain_destroy - Release session domain resources + */ +static void wd_sched_skey_domain_destroy(struct wd_sched_key_domain *key_domain) +{ + if (!key_domain) + return; + + pthread_mutex_destroy(&key_domain->lock); + wd_sched_skey_cache_uninit(&key_domain->idx_cache); +} + + +/** + * wd_sched_poll_skey - Poll contexts for scheduler session + * @sched_ctx: Scheduler context + * @skey: Session key + * @expect: Expected number of responses + * @count: Actual response count (output) + * + * Polls all contexts in session domains and updates load values. + */ +static int wd_sched_poll_skey(struct wd_sched_ctx *sched_ctx, struct wd_sched_key *skey, + __u32 expect, __u32 *count) +{ + struct wd_sched_domain_idx_cache *cache; + __u32 ctx_list_num = 0; + __u32 sum_poll_num = 0; + bool hungry_policy; + __u32 poll_num; + __u32 idx, i; + int ret = 0; + + /* Get cache pointer and check if HUNGRY policy */ + cache = &skey->async_domain.idx_cache; + ctx_list_num = cache->valid_count; + hungry_policy = (cache->policy == SCHED_POLICY_HUNGRY); + + /* Poll async domain contexts */ + for (i = 0; i < ctx_list_num; i++) { + idx = cache->idx_list[i]; + if (idx == INVALID_POS) + continue; + poll_num = 0; + ret = sched_ctx->poll_func(idx, expect, &poll_num); + if (poll_num > 0) + sum_poll_num += poll_num; + + /* Update load value for this context */ + if (hungry_policy && poll_num > 0) { + if (skey->ctxs) + __atomic_fetch_sub(&skey->ctxs[idx].hw_load, + poll_num, __ATOMIC_RELAXED); + wd_sched_skey_update_load(cache, i, -poll_num); + } + + if (ret < 0 && ret != -WD_EAGAIN) + break; + } + *count = sum_poll_num; + + return ret; +} + +/* ============================================================================ + * Utility Functions + * ============================================================================ + */ +static inline bool sched_skey_get_ref(struct wd_sched_key *skey) +{ + if (!skey) + return false; + return __atomic_fetch_add(&skey->refcount, 1, __ATOMIC_ACQUIRE) > 0; +} + +static inline bool sched_skey_put_ref(struct wd_sched_key *skey) +{ + return __atomic_fetch_sub(&skey->refcount, 1, __ATOMIC_RELEASE) == 1; +} + +static int sched_skey_param_init(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey) +{ + __u32 i; + + pthread_mutex_lock(&sched_ctx->skey_lock); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + if (!sched_ctx->skey[i]) { + sched_ctx->skey[i] = skey; + if (sched_ctx->skey_num < SKEY_MAX_THREAD_NUM) + sched_ctx->skey_num++; + pthread_mutex_unlock(&sched_ctx->skey_lock); + return 0; + } + } + pthread_mutex_unlock(&sched_ctx->skey_lock); + WD_ERR("invalid: skey node number exceeds SKEY_MAX_THREAD_NUM(%d)!\n", + SKEY_MAX_THREAD_NUM); + return -WD_ENOMEM; +} + +static void sched_skey_param_uninit(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey) +{ + __u32 i; + + if (!sched_ctx || !skey) + return; + + pthread_mutex_lock(&sched_ctx->skey_lock); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + if (sched_ctx->skey[i] == skey) { + sched_ctx->skey[i] = NULL; + pthread_mutex_unlock(&sched_ctx->skey_lock); + return; + } + } + pthread_mutex_unlock(&sched_ctx->skey_lock); + WD_ERR("warning: skey %p not found in sched_ctx array\n", skey); +} + +static handle_t sched_session_common_init(struct wd_sched_ctx *sched_ctx, + struct sched_params *param) +{ + struct wd_sched_key *skey; unsigned int node; if (getcpu(NULL, &node)) { WD_ERR("failed to get node, errno %d!\n", errno); return (handle_t)(-errno); } - if (node == (unsigned int)NUMA_NO_NODE) { - WD_ERR("invalid: failed to get numa node!\n"); - return (handle_t)(-WD_EINVAL); - } if (!sched_ctx) { WD_ERR("invalid: sched ctx is NULL!\n"); return (handle_t)(-WD_EINVAL); } - skey = malloc(sizeof(struct sched_key)); + skey = malloc(sizeof(struct wd_sched_key)); if (!skey) { WD_ERR("failed to alloc memory for session sched key!\n"); return (handle_t)(-WD_ENOMEM); } + memset(skey, 0, sizeof(struct wd_sched_key)); if (!param) { - memset(skey, 0, sizeof(struct sched_key)); - skey->numa_id = sched_ctx->numa_map[node]; + skey->region_id = node; if (wd_need_debug()) WD_DEBUG("session don't set scheduler parameters!\n"); - } else if (param->numa_id < 0) { - skey->type = param->type; - skey->numa_id = sched_ctx->numa_map[node]; } else { + if (sched_ctx->policy == SCHED_POLICY_DEV) + skey->region_id = param->dev_id; + else if (param->numa_id >= 0) + skey->region_id = param->numa_id; + else + skey->region_id = node; skey->type = param->type; - skey->numa_id = param->numa_id; + skey->ctx_prop = param->ctx_prop; } + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + __atomic_store_n(&skey->refcount, 1, __ATOMIC_RELAXED); - if (skey->numa_id < 0) { - WD_ERR("failed to get valid sched numa region!\n"); - goto out; - } + return (handle_t)skey; +} - skey->sync_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); - skey->async_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); - if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { - WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); - goto out; +static __u16 sched_get_poll_skey_idx(struct wd_sched_ctx *sched_ctx) +{ + /* Thread-local sequence number with atomic global counter */ + static __thread __u32 thread_seq = UINT32_MAX; + static __u32 global_seq_counter; + __u32 skey_num = sched_ctx->skey_num; + __u16 start_pos; + + if (unlikely(!sched_ctx || !skey_num)) + return skey_num; + + /* Assign unique sequence number on first call */ + if (unlikely(thread_seq == UINT32_MAX)) { + thread_seq = __atomic_fetch_add(&global_seq_counter, 1, __ATOMIC_RELAXED); + + /* Basic overflow protection */ + if (thread_seq >= UINT32_MAX - 1) + WD_DEBUG("Thread sequence counter approaching limit: %u\n", thread_seq); } - return (handle_t)skey; + /* Calculate start_pos based on thread_seq */ + start_pos = thread_seq % skey_num; -out: - free(skey); - return (handle_t)(-WD_EINVAL); + return start_pos; } -/* - * session_pick_next_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/** + * session_sched_init_ctx - Pre-fetch single context from domain for session + * @sched_ctx: Scheduler context + * @skey: session scheduler param + * @sched_mode: Mode (SYNC/ASYNC) * - * The user must init the schedule info through session_sched_init + * Returns: Context index from domain (compatible with alg_name if skey provided) */ -static __u32 session_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, - const int sched_mode) +static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, int sched_mode) { - struct sched_key *key = (struct sched_key *)sched_key; + struct wd_sched_ctx_domain *domain = NULL; + int region_id = skey->region_id; + __u32 op_type = skey->type; + __u8 prop = skey->ctx_prop; + __u32 ctx_idx; + __u16 r; + + if (sched_mode >= SCHED_MODE_BUTT || + op_type >= sched_ctx->type_num || prop >= UADK_ALG_TYPE_MAX) { + WD_ERR("invalid: region: %d, mode: %d, type: %u!, prop: %u\n", + region_id, sched_mode, op_type, prop); + return INVALID_POS; + } - if (unlikely(!h_sched_ctx || !key)) { - WD_ERR("invalid: sched ctx or key is NULL!\n"); + if (!sched_ctx->domain_hash_table) + return INVALID_POS; + + /* Try current region first */ + if (region_id >= 0) { + if (sched_ctx->policy == SCHED_POLICY_DEV || + region_id < sched_ctx->region_num) { + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + region_id, sched_mode, op_type, prop); + if (domain && domain->valid) + return wd_sched_domain_get_next_rr(domain); + } + } + + /* DEV policy must not cross region */ + if (sched_ctx->policy == SCHED_POLICY_DEV) return INVALID_POS; + + /* Cross-region fallback for other policies */ + for (r = 0; r < sched_ctx->region_num; r++) { + if ((int)r == region_id) + continue; + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + r, sched_mode, op_type, prop); + if (domain && domain->valid) { + ctx_idx = wd_sched_domain_get_next_rr(domain); + if (ctx_idx != INVALID_POS) + return ctx_idx; + } } - /* return in do task */ - if (sched_mode == CTX_MODE_SYNC) - return key->sync_ctxid; - return key->async_ctxid; + return INVALID_POS; } -static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, - __u32 end, __u32 expect, __u32 *count) +/** + * session_sched_domain_destroy - Destroy session domains + * @skey: Session key to destroy domains for + * + * Releases all resources associated with session domains. + */ +static void session_sched_domain_destroy(struct wd_sched_key *skey) { - __u32 poll_num = 0; - __u32 i; - int ret; + if (!skey) + return; - /* i is the pos of sched_ctxs, the max is end */ - for (i = begin; i <= end; i++) { - /* - * RR schedule, one time poll one package, - * poll_num is always not more than one here. - */ - ret = sched_ctx->poll_func(i, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) - return ret; - else if (ret == -EAGAIN) + /* Destroy both sync and async domains */ + wd_sched_skey_domain_destroy(&skey->sync_domain); + wd_sched_skey_domain_destroy(&skey->async_domain); +} + +static inline void sched_skey_poll_release(struct wd_sched_key *skey) +{ + __u32 prev; + + if (!skey) + return; + + prev = __atomic_fetch_sub(&skey->refcount, 1, __ATOMIC_ACQ_REL); + if (prev == 1) { + session_sched_domain_destroy(skey); + free(skey); + } +} + +static void sched_session_uninit(handle_t h_sched_ctx, handle_t h_sched_key) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_key *skey = (struct wd_sched_key *)h_sched_key; + + if (!skey) + return; + + /* Remove from skey array first to prevent new poll discovery. + * Poll threads already holding a pointer are protected by refcount. + */ + if (sched_ctx) + sched_skey_param_uninit(sched_ctx, skey); + + /* Drop creator reference. If no poll holds a ref, free now. + * Otherwise the last poll thread will cleanup via poll_release. + */ + if (sched_skey_put_ref(skey)) { + session_sched_domain_destroy(skey); + free(skey); + } +} + +/** + * session_sched_init_ctx_with_fallback - Pre-fetch a ctx for one sched_mode, + * trying user_prop first then falling + * back through other props in enum order. + * @sched_ctx: Scheduler context + * @skey: Session key (skey->ctx_prop is the user-specified prop; restored on return) + * @sched_mode: SCHED_MODE_SYNC or SCHED_MODE_ASYNC + * + * Returns: ctx index, or INVALID_POS if no prop has a usable ctx. + * + * Falls back only when the user-specified prop's domain is unavailable + * (e.g. HW driver unloaded). Stops at the first prop that yields a ctx; + * does NOT accumulate ctxs across props. + */ +static __u32 session_sched_init_ctx_with_fallback(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + int sched_mode) +{ + __u8 user_prop = skey->ctx_prop; + __u32 ctx_idx; + __u8 p; + + /* Try user-specified prop first */ + ctx_idx = session_sched_init_ctx(sched_ctx, skey, sched_mode); + if (ctx_idx != INVALID_POS) + return ctx_idx; + + /* Fallback: target prop unavailable, try remaining props in enum order */ + for (p = 0; p < UADK_ALG_TYPE_MAX; p++) { + if (p == user_prop) continue; - *count += poll_num; - if (*count == expect) + skey->ctx_prop = p; + ctx_idx = session_sched_init_ctx(sched_ctx, skey, sched_mode); + if (ctx_idx != INVALID_POS) break; } - return 0; + /* Restore user_prop so skey state is not mutated */ + skey->ctx_prop = user_prop; + return ctx_idx; } -static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, - __u32 expect, __u32 *count) +/** + * session_sched_domain_init - Initialize session domains with sync/async + * @sched_ctx: Scheduler context + * @skey: Session key to initialize + * @allow_fallback: If true, fall back to other props when skey->ctx_prop has + * no usable domain; if false, only try skey->ctx_prop. + * + * Pre-fetches sync and async contexts and initializes corresponding domains. + * Returns: 0 on success, negative error code on failure. + */ +static int session_sched_domain_init(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + bool allow_fallback) { - struct sched_ctx_region **region = sched_ctx->sched_info[numa_id].ctx_region; - __u32 begin, end; - __u32 i; - int ret; + __u32 sync_ctx, async_ctx; - for (i = 0; i < sched_ctx->type_num; i++) { - if (!region[SCHED_MODE_ASYNC][i].valid) - continue; + if (!sched_ctx || !skey) { + WD_ERR("invalid: sched_ctx or skey is NULL!\n"); + return -WD_EINVAL; + } + + if (allow_fallback) { + /* Pre-fetch with per-mode prop fallback */ + sync_ctx = session_sched_init_ctx_with_fallback(sched_ctx, skey, + SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx_with_fallback(sched_ctx, skey, + SCHED_MODE_ASYNC); + } else { + sync_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_ASYNC); + } + + if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) { + WD_ERR("invalid: no valid sync_ctx or async_ctx domain!\n"); + return -WD_EINVAL; + } + + /* Initialize sync domain if context is valid */ + if (sync_ctx != INVALID_POS) { + if (wd_sched_skey_domain_init(&skey->sync_domain, sync_ctx, sched_ctx->policy)) { + WD_ERR("failed to init sync domain!\n"); + return -WD_EINVAL; + } + } + + /* Initialize async domain if context is valid */ + if (async_ctx != INVALID_POS) { + if (wd_sched_skey_domain_init(&skey->async_domain, async_ctx, sched_ctx->policy)) { + WD_ERR("failed to init async domain!\n"); + /* Cleanup sync domain if async domain init failed */ + if (sync_ctx != INVALID_POS) + wd_sched_skey_domain_destroy(&skey->sync_domain); + return -WD_EINVAL; + } + } + + return WD_SUCCESS; +} + +/* ============================================================================ + * Scheduler Policy Functions + * ============================================================================ + */ +/** + * round_robin_sched_init - Initialize session with single sync and async ctx + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_param: Scheduling parameters (cannot modify per API contract) + * + * Allocates session key and pre-fetches one sync and one async context. + */ +static handle_t round_robin_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + struct wd_sched_key *skey; + handle_t hskey; + int ret = 0; + + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; + } + + skey = (struct wd_sched_key *)hskey; + /* RR: allow prop fallback so session creation survives target prop unload */ + ret = session_sched_domain_init(sched_ctx, skey, true); + if (ret) { + WD_ERR("failed to initialize session domains!\n"); + free(skey); + return (handle_t)(-WD_EINVAL); + } + + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); + return (handle_t)(-WD_ENOMEM); + } + + return hskey; +} + +/** + * round_robin_pick_next_ctx - Pick context with load-based selection + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) + * + * Returns: Context index with minimum load + * Time complexity: O(1) + */ +static __u32 round_robin_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_key_domain *domain; + __u32 min_ctx, ctx_idx; - begin = region[SCHED_MODE_ASYNC][i].begin; - end = region[SCHED_MODE_ASYNC][i].end; - ret = session_poll_region(sched_ctx, begin, end, expect, count); - if (unlikely(ret)) - return ret; + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; } - return 0; + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; + + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) + return INVALID_POS; + + return min_ctx; } -/* - * session_poll_policy - The polling policy matches the pick next ctx. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @cfg: The global resoure info. - * @expect: User expect poll msg num. - * @count: The actually poll num. +/** + * round_robin_poll_policy - Poll policy for session scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) * - * The user must init the schedule info through wd_sched_rr_instance, the - * func interval will not check the valid, becouse it will affect performance. + * Returns: Status code */ -static int session_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +static int round_robin_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - struct wd_sched_info *sched_info; - __u32 loop_time = 0; - __u32 last_count = 0; - __u16 i, region_mum; - int ret; + struct wd_sched_key *skey; + __u16 i, tpos, start_pos; + __u32 poll_num, skey_num; + int ret = -WD_EAGAIN; + __u32 sum_count = 0; if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); return -WD_EINVAL; } - if (unlikely(sched_ctx->numa_num > NUMA_NUM_NODES)) { - WD_ERR("invalid: ctx's numa number is %u!\n", sched_ctx->numa_num); - return -WD_EINVAL; + /* Randomize the initial query position. */ + skey_num = sched_ctx->skey_num; + start_pos = sched_get_poll_skey_idx(sched_ctx); + if (!skey_num || start_pos >= sched_ctx->skey_num) { + *count = 0; + return -WD_EAGAIN; } - sched_info = sched_ctx->sched_info; - if (sched_ctx->policy == SCHED_POLICY_DEV) - region_mum = sched_ctx->dev_num; - else - region_mum = sched_ctx->numa_num; - - /* - * Try different region's ctx if we can't receive any - * package last time, it is more efficient. In most - * bad situation, poll ends after MAX_POLL_TIMES loop. - */ - while (++loop_time < MAX_POLL_TIMES) { - for (i = 0; i < region_mum;) { - /* If current numa is not valid, find next. */ - if (!sched_info[i].valid) { - i++; - continue; - } + /* Query the queues on each skey separately. */ + for (i = 0; i < skey_num; i++) { + tpos = (start_pos + i) % skey_num; + skey = sched_ctx->skey[tpos]; - last_count = *count; - ret = session_poll_policy_rr(sched_ctx, i, expect, count); - if (unlikely(ret)) - return ret; + if (unlikely(!skey)) + continue; - if (expect == *count) - return 0; + if (!sched_skey_get_ref(skey)) + continue; - /* - * If no package is received, find next numa, - * otherwise, keep receiving packets at this node. - */ - if (last_count == *count) - i++; + if (__atomic_test_and_set(&skey->poll_lock, __ATOMIC_ACQUIRE)) { + sched_skey_poll_release(skey); + continue; } + + ret = wd_sched_poll_skey(sched_ctx, skey, expect, &poll_num); + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + sched_skey_poll_release(skey); + + sum_count += poll_num; + if (unlikely(ret && ret != -WD_EAGAIN)) + goto poll_err; + + if (sum_count >= expect) + break; } - return 0; +poll_err: + *count = sum_count; + if (ret == -WD_EAGAIN) + return 0; + return ret; } static handle_t sched_none_init(handle_t h_sched_ctx, void *sched_param) @@ -396,12 +1464,11 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, } while (loop_times > 0) { - /* Default use ctx 0 */ loop_times--; ret = sched_ctx->poll_func(0, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) + if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; - else if (ret == -EAGAIN) + else if (ret == -WD_EAGAIN) continue; *count += poll_num; @@ -409,7 +1476,7 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, break; } - return 0; + return WD_SUCCESS; } static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) @@ -420,13 +1487,10 @@ static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) static __u32 sched_single_pick_next_ctx(handle_t sched_ctx, void *sched_key, const int sched_mode) { -#define CTX_ASYNC 1 -#define CTX_SYNC 0 - if (sched_mode) - return CTX_ASYNC; + return 1; else - return CTX_SYNC; + return 0; } static int sched_single_poll_policy(handle_t h_sched_ctx, @@ -443,12 +1507,11 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, } while (loop_times > 0) { - /* Default async mode use ctx 1 */ loop_times--; ret = sched_ctx->poll_func(1, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) + if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; - else if (ret == -EAGAIN) + else if (ret == -WD_EAGAIN) continue; *count += poll_num; @@ -456,260 +1519,483 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, break; } - return 0; + return WD_SUCCESS; } -static bool sched_dev_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +/** + * sched_skey_domain_fill - Initialize or append ctx to session domain + * @key_domain: Target domain (sync or async) + * @ctx_idx: Context index to add + * @policy: Scheduler policy + * @inited: Pointer to boolean tracking whether domain has been initialized + * + * First call: full domain init via wd_sched_skey_domain_init(). + * Subsequent calls: append via wd_sched_skey_add_ctx(). + */ +static void sched_skey_domain_fill(struct wd_sched_key_domain *key_domain, + __u32 ctx_idx, __u8 policy, bool *inited) { - bool found = false; - int i; + int ret; - if (key->mode >= SCHED_MODE_BUTT || key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's device id: %u, mode: %u, type: %u!\n", - key->dev_id, key->mode, key->type); - return false; + if (ctx_idx == INVALID_POS) + return; + + if (!*inited) { + ret = wd_sched_skey_domain_init(key_domain, ctx_idx, policy); + if (!ret) + *inited = true; + return; } - for (i = 0; i < sched_ctx->dev_num; i++) { - if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { - found = true; - break; + (void)wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx); +} + +/** + * sched_skey_common_init - Common scheduler init with init-once + add-rest pattern + * @h_sched_ctx: Scheduler handle + * @sched_param: Scheduling parameters + * @prop_begin: First prop type to iterate (inclusive) + * @prop_end: Last prop type to iterate (inclusive) + * + * Shared by Loop, Hungry, and Instr scheduler init functions. + * Iterates [prop_begin, prop_end], fetching sync/async ctxs per prop type. + * First valid ctx initializes the domain, subsequent ctxs are appended. + */ +static handle_t sched_skey_common_init(handle_t h_sched_ctx, void *sched_param, + __u8 prop_begin, __u8 prop_end) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + struct wd_sched_key *skey; + bool sync_inited = false; + bool async_inited = false; + __u32 sync_ctx, async_ctx; + __u32 req_ctx_num = 0; + handle_t hskey; + __u8 def_prop; + int ret, i; + + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; + } + + skey = (struct wd_sched_key *)hskey; + def_prop = skey->ctx_prop; + for (i = prop_begin; i <= prop_end; i++) { + skey->ctx_prop = i; + sync_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_ASYNC); + if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) + continue; + + if (sync_ctx != INVALID_POS) { + sched_skey_domain_fill(&skey->sync_domain, sync_ctx, + sched_ctx->policy, &sync_inited); } + if (async_ctx != INVALID_POS) + sched_skey_domain_fill(&skey->async_domain, async_ctx, + sched_ctx->policy, &async_inited); + + req_ctx_num += 2; + } + if (!req_ctx_num) { + free(skey); + return (handle_t)(-WD_EINVAL); } - if (!found) { - WD_ERR("invalid: dev_id %u is not registered!\n", key->dev_id); - return false; + skey->ctx_prop = def_prop; + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); + return (handle_t)(-WD_ENOMEM); } - return true; + return hskey; } -/* - * sched_dev_get_region - Get ctx region from ctx_map by the wd comp arg +/** + * skey_sched_init - Initialize Hungry scheduler session + */ +static handle_t skey_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + 0, UADK_ALG_TYPE_MAX - 1); + return hskey; +} + +/** + * skey_sched_pick_next_ctx - Pick context from hungry scheduler with load awareness + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) + * + * Returns: Context with minimum load, or expands if threshold exceeded + * Time complexity: O(1) for selection, O(n) if expansion needed */ -static struct sched_ctx_region *sched_dev_get_region(struct wd_sched_ctx *sched_ctx, - const struct sched_key *key) +static __u32 skey_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) { - struct wd_sched_info *sched_info; - int i, region_id; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_domain_idx_cache *idx_cache; + struct wd_sched_key_domain *domain; + __u32 min_ctx, min_load, ctx_idx; - for (i = 0; i < sched_ctx->dev_num; i++) { - if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { - region_id = sched_ctx->dev_id_map[i].region_id; - sched_info = &sched_ctx->sched_info[region_id]; - if (sched_info->ctx_region[key->mode][key->type].valid) - return &sched_info->ctx_region[key->mode][key->type]; + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; + } + + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; + + idx_cache = &domain->idx_cache; + if (sched_mode == SCHED_MODE_SYNC) { + /* Sync: send+recv atomic, spinlock serialized, use RR */ + if (!idx_cache->valid_count) + return INVALID_POS; + + ctx_idx = __atomic_fetch_add(&idx_cache->rr_ptr, 1, __ATOMIC_RELAXED) % + idx_cache->valid_count; + return idx_cache->idx_list[ctx_idx]; + } + + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) + return INVALID_POS; + + if (skey->ctxs) { + min_load = __atomic_load_n(&skey->ctxs[min_ctx].hw_load, __ATOMIC_RELAXED); + if (min_load >= HW_QUEUE_FULL_DEPTH) + return QUEUE_FULL_POS; + __atomic_fetch_add(&skey->ctxs[min_ctx].hw_load, 1, __ATOMIC_RELAXED); + wd_sched_skey_update_load(idx_cache, ctx_idx, 1); + } else { + min_load = __atomic_load_n(&idx_cache->load_values[ctx_idx], __ATOMIC_RELAXED); + if (min_load >= HW_QUEUE_FULL_DEPTH && idx_cache->load_decreased > 0) + return QUEUE_FULL_POS; + wd_sched_skey_update_load(idx_cache, ctx_idx, 1); + } + /* Check if we need to expand context pool */ + + return min_ctx; +} + +/** + * skey_sched_poll_policy - Poll policy for hungry scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) + * + * Returns: Status code + */ +static int skey_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_key *skey; + __u16 i, tpos, start_pos; + __u32 poll_num, skey_num; + int ret = -WD_EAGAIN; + __u32 sum_count = 0; + + if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { + WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); + return -WD_EINVAL; + } + + /* Randomize the initial query position. */ + skey_num = sched_ctx->skey_num; + start_pos = sched_get_poll_skey_idx(sched_ctx); + if (!skey_num || start_pos >= sched_ctx->skey_num) { + *count = 0; + return -WD_EAGAIN; + } + + /* Query the queues on each skey separately. */ + for (i = 0; i < skey_num; i++) { + tpos = (start_pos + i) % skey_num; + skey = sched_ctx->skey[tpos]; + + if (unlikely(!skey)) + continue; + + if (!sched_skey_get_ref(skey)) + continue; + + if (__atomic_test_and_set(&skey->poll_lock, __ATOMIC_ACQUIRE)) { + sched_skey_poll_release(skey); + continue; } + + ret = wd_sched_poll_skey(sched_ctx, skey, expect, &poll_num); + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + sched_skey_poll_release(skey); + /* + * This query returned 0, indicating the hardware + * likely hasn't finished processing yet. + * Implementing a delay and releasing the CPU is a predictive optimization + */ + if (!poll_num) + usleep(1); + + sum_count += poll_num; + if (ret == -WD_EAGAIN) + continue; + if (unlikely(ret) || sum_count >= expect) + break; } + *count = sum_count; - /* - * If the scheduling domain of dev_id does not exist, - * taskes operations cannot be executed using queues from other devices; - * otherwise, an SMMU error will occur. - */ - return NULL; + return ret; } -/* - * session_dev_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/** + * loop_sched_init - Initialize Loop scheduler session + */ +static handle_t loop_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + 0, UADK_ALG_TYPE_MAX - 1); + return hskey; +} + +/** + * loop_sched_pick_next_ctx - Pick context for loop scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) * - * The user must init the schedule info through wd_sched_rr_instance + * Returns: Context index with minimum load + * Time complexity: O(1) + */ +static __u32 loop_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + return round_robin_pick_next_ctx(h_sched_ctx, sched_key, sched_mode); +} + +static int loop_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + return round_robin_poll_policy(h_sched_ctx, expect, count); +} + +/** + * instr_sched_init - Initialize Instr scheduler session */ -static __u32 session_dev_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, - const int sched_mode) +static handle_t instr_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + UADK_ALG_CE_INSTR, + UADK_ALG_SVE_INSTR); + return hskey; +} + +static __u32 instr_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) { - struct sched_ctx_region *region = NULL; - bool ret; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_key_domain *domain; + __u32 min_ctx, ctx_idx; - key->mode = sched_mode; - ret = sched_dev_key_valid(sched_ctx, key); - if (!ret) + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); return INVALID_POS; + } + + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; - region = sched_dev_get_region(sched_ctx, key); - if (!region) + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) return INVALID_POS; - return sched_get_next_pos_rr(region, NULL); + return min_ctx; +} + +/** + * instr_sched_poll_policy - Poll policy for instruction scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) + * + * Returns: Status code + */ +static int instr_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + return round_robin_poll_policy(h_sched_ctx, expect, count); } static handle_t session_dev_sched_init(handle_t h_sched_ctx, void *sched_param) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; struct sched_params *param = (struct sched_params *)sched_param; - struct sched_key *skey; - unsigned int node; + struct wd_sched_key *skey; + handle_t hskey; + int ret = 0; - if (getcpu(NULL, &node)) { - WD_ERR("failed to get numa node, errno %d!\n", errno); - return (handle_t)(-errno); - } - if (node == (unsigned int)NUMA_NO_NODE) { - WD_ERR("invalid: failed to get numa node for dev sched init!\n"); + if (!param) { + WD_ERR("invalid: dev sched param is NULL!\n"); return (handle_t)(-WD_EINVAL); } - if (!sched_ctx) { - WD_ERR("invalid: sched ctx is NULL!\n"); - return (handle_t)(-WD_EINVAL); + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; } - if (!param) { - WD_DEBUG("no-sva session don't set scheduler parameters!\n"); + skey = (struct wd_sched_key *)hskey; + skey->type = param->type; + skey->dev_id = param->dev_id; + + /* DEV: pinned to a specific device, do not cross prop families */ + ret = session_sched_domain_init(sched_ctx, skey, false); + if (ret) { + WD_ERR("failed to initialize session domains!\n"); + free(skey); return (handle_t)(-WD_EINVAL); } - skey = malloc(sizeof(struct sched_key)); - if (!skey) { - WD_ERR("failed to alloc memory for session sched key!\n"); + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); return (handle_t)(-WD_ENOMEM); } - skey->type = param->type; - skey->dev_id = param->dev_id; + return (handle_t)skey; +} + +/** + * wd_sched_set_param - Set scheduler parameters + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_param: Scheduling parameters (cannot modify per API contract) + */ + +static void wd_sched_set_param(handle_t h_sched_ctx, + void *sched_key, void *sched_param) +{ + struct wd_sched_params *params = (struct wd_sched_params *)sched_param; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; - skey->sync_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); - skey->async_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); - if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { - WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); - goto out; + if (unlikely(!params || !skey)) { + WD_ERR("invalid: sched parmas or skey is NULL!\n"); + return; } - return (handle_t)skey; + skey->pkt_size = params->pkt_size; + skey->is_stream = params->data_mode; + skey->prio_mode = params->prio_mode; + + /* Store compat filtering parameters */ + skey->alg_name = params->alg_name; + skey->ctxs = params->ctxs; -out: - free(skey); - return (handle_t)(-WD_EINVAL); } static struct wd_sched sched_table[SCHED_POLICY_BUTT] = { { .name = "RR scheduler", .sched_policy = SCHED_POLICY_RR, - .sched_init = session_sched_init, - .pick_next_ctx = session_sched_pick_next_ctx, - .poll_policy = session_sched_poll_policy, + .sched_init = round_robin_sched_init, + .pick_next_ctx = round_robin_pick_next_ctx, + .poll_policy = round_robin_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "None scheduler", .sched_policy = SCHED_POLICY_NONE, .sched_init = sched_none_init, .pick_next_ctx = sched_none_pick_next_ctx, .poll_policy = sched_none_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "Single scheduler", .sched_policy = SCHED_POLICY_SINGLE, .sched_init = sched_single_init, .pick_next_ctx = sched_single_pick_next_ctx, .poll_policy = sched_single_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "Device RR scheduler", .sched_policy = SCHED_POLICY_DEV, .sched_init = session_dev_sched_init, - .pick_next_ctx = session_sched_pick_next_ctx, - .poll_policy = session_sched_poll_policy, - } + .pick_next_ctx = round_robin_pick_next_ctx, + .poll_policy = round_robin_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Loop scheduler", + .sched_policy = SCHED_POLICY_LOOP, + .sched_init = loop_sched_init, + .pick_next_ctx = loop_sched_pick_next_ctx, + .poll_policy = loop_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Hungry scheduler", + .sched_policy = SCHED_POLICY_HUNGRY, + .sched_init = skey_sched_init, + .pick_next_ctx = skey_sched_pick_next_ctx, + .poll_policy = skey_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Instr scheduler", + .sched_policy = SCHED_POLICY_INSTR, + .sched_init = instr_sched_init, + .pick_next_ctx = instr_sched_pick_next_ctx, + .poll_policy = instr_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, }; -static int wd_sched_get_nearby_numa_id(struct wd_sched_info *sched_info, int node, int numa_num) -{ - int dis = INT32_MAX; - int valid_id = -1; - int i, tmp; - - for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) { - tmp = numa_distance(node, i); - if (dis > tmp) { - valid_id = i; - dis = tmp; - } - } - } - - return valid_id; -} - -static void wd_sched_map_cpus_to_dev(struct wd_sched_ctx *sched_ctx) -{ - struct wd_sched_info *sched_info = sched_ctx->sched_info; - int i, numa_num = sched_ctx->numa_num; - int *numa_map = sched_ctx->numa_map; - - for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) - numa_map[i] = i; - else - numa_map[i] = wd_sched_get_nearby_numa_id(sched_info, i, numa_num); - } -} - -static int wd_instance_dev_region(struct wd_sched_ctx *sched_ctx, - struct sched_params *param) +static int numa_num_check(__u16 region_num) { - struct wd_sched_info *sched_info; - __u32 region_idx = INVALID_POS; - __u8 type, mode; - __u32 dev_id; - int i; - - dev_id = param->dev_id; - type = param->type; - mode = param->mode; - - /* Check whether dev_id has already been registered. */ - for (i = 0; i < sched_ctx->dev_num; i++) { - if (sched_ctx->dev_id_map[i].dev_id == dev_id) { - region_idx = sched_ctx->dev_id_map[i].region_id; - break; - } - } - - /* If not registered, allocate a new region. */ - if (region_idx == INVALID_POS) { - if (sched_ctx->dev_num >= DEVICE_REGION_MAX) { - WD_ERR("too many devices registered!\n"); - return -WD_EINVAL; - } - - region_idx = sched_ctx->dev_num; - sched_ctx->dev_id_map[region_idx].dev_id = dev_id; - sched_ctx->dev_id_map[region_idx].region_id = region_idx; - sched_ctx->dev_num++; + int max_node; - sched_info = &sched_ctx->sched_info[region_idx]; - } else { - sched_info = &sched_ctx->sched_info[region_idx]; + max_node = numa_max_node() + 1; + if (max_node <= 0) { + WD_ERR("invalid: numa max node is %d!\n", max_node); + return -WD_EINVAL; } - /* Check whether the mode and type have already been registered. */ - if (sched_info->ctx_region[mode][type].valid) { - WD_INFO("device %u mode %u type %u already registered\n", - dev_id, mode, type); - return WD_SUCCESS; + if (!region_num || region_num > max_node) { + WD_ERR("invalid: region number is %u!\n", region_num); + return -WD_EINVAL; } - /* Initialize the scheduling region for this mode and type */ - sched_info->ctx_region[mode][type].begin = param->begin; - sched_info->ctx_region[mode][type].end = param->end; - sched_info->ctx_region[mode][type].last = param->begin; - sched_info->ctx_region[mode][type].valid = true; - sched_info->valid = true; - - pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); - - return WD_SUCCESS; + return 0; } +/** + * wd_sched_rr_instance - External API for scheduling region instance + * @sched: Scheduler (cannot modify per API contract) + * @param: Scheduling parameters (cannot modify per API contract) + * + * Creates scheduling region for given parameters. + */ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *param) { - struct wd_sched_info *sched_info = NULL; struct wd_sched_ctx *sched_ctx = NULL; - __u8 type, mode; - int numa_id; + struct wd_sched_ctx_domain *domain; + int region_key; + __u8 mode; + int ret; if (!sched || !sched->h_sched_ctx || !param) { WD_ERR("invalid: sched or sched_params is NULL!\n"); @@ -721,57 +2007,68 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para return -WD_EINVAL; } - numa_id = param->numa_id; - type = param->type; mode = param->mode; sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; - if (sched_ctx->numa_num > 0 && (numa_id >= sched_ctx->numa_num || - numa_id < 0)) { - WD_ERR("invalid: sched_ctx's numa_id is %d, numa_num is %u!\n", - numa_id, sched_ctx->numa_num); - return -WD_EINVAL; + if (sched_ctx->policy == SCHED_POLICY_DEV) { + region_key = param->dev_id; + if (region_key < 0) { + WD_ERR("invalid: dev_id is %d!\n", region_key); + return -WD_EINVAL; + } + } else { + region_key = param->numa_id; + if (region_key >= sched_ctx->region_num || region_key < 0) { + WD_ERR("invalid: region_key is %d, region_num is %u!\n", + region_key, sched_ctx->region_num); + return -WD_EINVAL; + } } - if (type >= sched_ctx->type_num) { - WD_ERR("invalid: sched_ctx's type is %u, type_num is %u!\n", - type, sched_ctx->type_num); + if (param->type >= sched_ctx->type_num) { + WD_ERR("invalid: type is %u, type_num is %u!\n", + param->type, sched_ctx->type_num); return -WD_EINVAL; } if (mode >= SCHED_MODE_BUTT) { - WD_ERR("invalid: sched_ctx's mode is %u, mode_num is %d!\n", - mode, SCHED_MODE_BUTT); + WD_ERR("invalid: mode is %u, mode_num is %u!\n", + mode, sched_ctx->mode_num); return -WD_EINVAL; } - if (sched_ctx->policy == SCHED_POLICY_DEV) - return wd_instance_dev_region(sched_ctx, param); - - sched_info = &sched_ctx->sched_info[numa_id]; - if (!sched_info->ctx_region[mode]) { - WD_ERR("invalid: ctx_region is NULL, numa: %d, mode: %u!\n", - numa_id, mode); - return -WD_EINVAL; + if (param->ctx_prop < 0 || param->ctx_prop >= UADK_ALG_TYPE_MAX) { + WD_INFO("Info: ctx_prop %d exceeds max type!\n", param->ctx_prop); + param->ctx_prop = UADK_ALG_HW; } - sched_info->ctx_region[mode][type].begin = param->begin; - sched_info->ctx_region[mode][type].end = param->end; - sched_info->ctx_region[mode][type].last = param->begin; - sched_info->ctx_region[mode][type].valid = true; - sched_info->valid = true; + domain = wd_sched_hash_table_insert(sched_ctx->domain_hash_table, + region_key, mode, param->type, + param->ctx_prop); + if (!domain) + return -WD_ENOMEM; - wd_sched_map_cpus_to_dev(sched_ctx); - pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); + ret = wd_sched_domain_add_segment(domain, param->begin, param->end); + if (ret) { + WD_ERR("failed to add segment to domain!\n"); + return ret; + } + domain->valid = true; return WD_SUCCESS; } +/** + * wd_sched_rr_release - External API for scheduler release + * @sched: Scheduler to release (cannot modify per API contract) + * + * Releases all scheduler resources. + */ void wd_sched_rr_release(struct wd_sched *sched) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; - int i, j, region_num; + struct wd_sched_key *skey; + __u32 i; if (!sched) return; @@ -780,59 +2077,49 @@ void wd_sched_rr_release(struct wd_sched *sched) if (!sched_ctx) goto ctx_out; - /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ - if (sched_ctx->policy == SCHED_POLICY_DEV) - region_num = DEVICE_REGION_MAX; - else - region_num = sched_ctx->numa_num; + /* Release all session keys - iterate full array to catch residual entries */ + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + skey = sched_ctx->skey[i]; + if (!skey) + continue; - sched_info = sched_ctx->sched_info; - if (!sched_info) - goto info_out; + sched_ctx->skey[i] = NULL; + __atomic_store_n(&skey->refcount, 0, __ATOMIC_RELAXED); + session_sched_domain_destroy(skey); + free(skey); + } + sched_ctx->skey_num = 0; - for (i = 0; i < region_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - if (sched_info[i].ctx_region[j]) { - free(sched_info[i].ctx_region[j]); - sched_info[i].ctx_region[j] = NULL; - } - } + /* Release hash table */ + if (sched_ctx->domain_hash_table) { + wd_sched_hash_table_destroy(sched_ctx->domain_hash_table); + sched_ctx->domain_hash_table = NULL; } -info_out: + pthread_mutex_destroy(&sched_ctx->skey_lock); free(sched_ctx); + ctx_out: free(sched); - return; } -static int numa_num_check(__u16 numa_num) -{ - int max_node; - - max_node = numa_max_node() + 1; - if (max_node <= 0) { - WD_ERR("invalid: numa max node is %d!\n", max_node); - return -WD_EINVAL; - } - - if (!numa_num || numa_num > max_node) { - WD_ERR("invalid: numa number is %u!\n", numa_num); - return -WD_EINVAL; - } - - return 0; -} - +/** + * wd_sched_rr_alloc - External API for scheduler allocation + * @sched_type: Scheduling policy type (cannot modify per API contract) + * @type_num: Number of operation types (cannot modify per API contract) + * @region_num: Number of regions (cannot modify per API contract) + * @func: Poll function (cannot modify per API contract) + * + * Allocates and initializes scheduler with single global hash table. + */ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, - __u16 numa_num, user_poll_func func) + __u16 region_num, user_poll_func func) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; struct wd_sched *sched; - int region_num; - int i, j; + __u32 estimated_entries; + __u32 i; if (sched_type >= SCHED_POLICY_BUTT || !type_num) { WD_ERR("invalid: sched_type is %u or type_num is %u!\n", @@ -846,63 +2133,62 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, return NULL; } - if (sched_type == SCHED_POLICY_DEV) - region_num = DEVICE_REGION_MAX; - else - region_num = numa_num; - - sched_ctx = calloc(1, sizeof(struct wd_sched_ctx) + - sizeof(struct wd_sched_info) * region_num); + sched_ctx = calloc(1, sizeof(struct wd_sched_ctx)); if (!sched_ctx) { WD_ERR("failed to alloc memory for sched_ctx!\n"); goto err_out; } - /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ + /* Cache dimension parameters */ + sched_ctx->type_num = type_num; + sched_ctx->mode_num = SCHED_MODE_BUTT; + sched_ctx->region_num = region_num; + sched_ctx->policy = sched_type; + if (sched_type == SCHED_POLICY_DEV) { - sched_ctx->numa_num = 0; - sched_ctx->dev_num = 0; - for (i = 0; i < DEVICE_REGION_MAX; i++) { - sched_ctx->dev_id_map[i].dev_id = INVALID_POS; - sched_ctx->dev_id_map[i].region_id = INVALID_POS; - } + /* Device mode: region_num is actually device count */ + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } else { - sched_ctx->numa_num = numa_num; - sched_ctx->dev_num = 0; - if (numa_num_check(sched_ctx->numa_num)) + /* NUMA mode: validate region_num */ + if (numa_num_check(region_num)) goto err_out; + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } - sched->h_sched_ctx = (handle_t)sched_ctx; - if (sched_type == SCHED_POLICY_NONE || - sched_type == SCHED_POLICY_SINGLE) - goto simple_ok; - - sched_info = sched_ctx->sched_info; - for (i = 0; i < region_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - sched_info[i].ctx_region[j] = - calloc(1, sizeof(struct sched_ctx_region) * type_num); - if (!sched_info[i].ctx_region[j]) - goto err_out; - } + /* Create single global hash table */ + sched_ctx->domain_hash_table = wd_sched_hash_table_create(estimated_entries); + if (!sched_ctx->domain_hash_table) { + WD_ERR("failed to create hash table!\n"); + goto ctx_out; } -simple_ok: sched_ctx->poll_func = func; - sched_ctx->policy = sched_type; - sched_ctx->type_num = type_num; - memset(sched_ctx->numa_map, -1, sizeof(int) * NUMA_NUM_NODES); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) + sched_ctx->skey[i] = NULL; + + if (pthread_mutex_init(&sched_ctx->skey_lock, NULL)) { + WD_ERR("failed to init skey_lock!\n"); + goto err_destroy_hash; + } + sched_ctx->skey_num = 0; + + sched->h_sched_ctx = (handle_t)sched_ctx; sched->sched_init = sched_table[sched_type].sched_init; + sched->sched_uninit = sched_table[sched_type].sched_uninit; sched->pick_next_ctx = sched_table[sched_type].pick_next_ctx; sched->poll_policy = sched_table[sched_type].poll_policy; sched->sched_policy = sched_type; sched->name = sched_table[sched_type].name; + sched->set_param = sched_table[sched_type].set_param; return sched; +err_destroy_hash: + wd_sched_hash_table_destroy(sched_ctx->domain_hash_table); +ctx_out: + free(sched_ctx); err_out: - wd_sched_rr_release(sched); + free(sched); return NULL; } -- 2.43.0

1 15

[PATCH 01/16] uadk: rewrite scheduler with scheduling domain solution
by Chenghai Huang 25 Aug '26

25 Aug '26

From: Longfang Liu <liulongfang(a)huawei.com> Conduct a comprehensive update of the scheduler's internal triple-array-based pooling scheme. Adopt new data organization methods such as hash buckets and segment linked lists to achieve efficient insertion and query operations. Implement pre-fetch queue processing for business thread sessions, creating thread-level sub-queue pools for fast packet reception queries. Provide new scheduling algorithms for the new UADK framework. Signed-off-by: Longfang Liu <liulongfang(a)huawei.com> Signed-off-by: Chenghai Huang <huangchenghai2(a)huawei.com> Signed-off-by: Wenkai Lin <linwenkai6(a)hisilicon.com> --- include/wd_alg.h | 11 +- include/wd_alg_common.h | 5 +- include/wd_internal.h | 18 +- include/wd_sched.h | 10 +- wd_sched.c | 2304 ++++++++++++++++++++++++++++++--------- 5 files changed, 1832 insertions(+), 516 deletions(-) diff --git a/include/wd_alg.h b/include/wd_alg.h index 18503ca..1ae1dae 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -64,10 +64,13 @@ extern "C" { #endif enum alg_dev_type { - UADK_ALG_SOFT = 0x0, - UADK_ALG_CE_INSTR = 0x1, - UADK_ALG_SVE_INSTR = 0x2, - UADK_ALG_HW = 0x3 + UADK_ALG_HW = 0x0, + UADK_ALG_CE_INSTR = 0x1, + UADK_ALG_SVE_INSTR = 0x2, + UADK_ALG_SOFT = 0x3, + UADK_ALG_NPU = 0x4, + UADK_ALG_GPU = 0x5, + UADK_ALG_TYPE_MAX, }; /* diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index a294877..ebba7f8 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -41,6 +41,7 @@ extern "C" { /* Key size of digest */ #define MAX_HMAC_KEY_SIZE 128U +#define STATUS_ENABLE (void *)0x1 enum alg_task_type { TASK_MIX = 0x0, @@ -140,7 +141,7 @@ struct wd_ctx_params { }; /* - * struct wd_comp_sched - Define a scheduler. + * struct wd_sched - Define a scheduler. * @name: Name of this scheduler. * @sched_policy: Method for scheduler to perform scheduling * @sched_init: inited the scheduler input parameters. @@ -157,10 +158,12 @@ struct wd_sched { const char *name; int sched_policy; handle_t (*sched_init)(handle_t h_sched_ctx, void *sched_param); + void (*sched_uninit)(handle_t h_sched_ctx, handle_t h_sched_key); __u32 (*pick_next_ctx)(handle_t h_sched_ctx, void *sched_key, const int sched_mode); int (*poll_policy)(handle_t h_sched_ctx, __u32 expect, __u32 *count); + void (*set_param)(handle_t h_sched_ctx, void *sched_key, void *sched_param); handle_t h_sched_ctx; }; diff --git a/include/wd_internal.h b/include/wd_internal.h index 4f19d3a..62cf1a9 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -7,8 +7,10 @@ #define WD_INTERNAL_H #include <pthread.h> +#include <stdatomic.h> #include <stdbool.h> #include "wd.h" +#include "wd_alg.h" #ifdef __cplusplus extern "C" { @@ -41,11 +43,14 @@ struct wd_ce_ctx { }; struct wd_ctx_internal { - handle_t ctx; __u8 op_type; __u8 ctx_mode; + __u8 ctx_type; + handle_t ctx; __u16 sqn; pthread_spinlock_t lock; + struct wd_alg_driver *drv; + __u32 hw_load; }; struct wd_ctx_config_internal { @@ -64,6 +69,17 @@ struct wd_datalist { struct wd_datalist *next; }; +struct wd_sched_params { + __u32 pkt_size; + /* block mode or stream mode */ + __u16 data_mode; + __u16 prio_mode; + + /* Compat filtering parameters for session-ctx matching */ + const char *alg_name; + struct wd_ctx_internal *ctxs; +}; + int memcmp_consttime(const void *s1, const void *s2, size_t n); #ifdef __cplusplus diff --git a/include/wd_sched.h b/include/wd_sched.h index 5baecd3..f81025f 100644 --- a/include/wd_sched.h +++ b/include/wd_sched.h @@ -13,7 +13,8 @@ extern "C" { #endif -#define INVALID_POS 0xFFFFFFFF +#define INVALID_POS 0xFFFF +#define QUEUE_FULL_POS 0x1FFF /* The global policy type */ enum sched_policy_type { @@ -25,6 +26,12 @@ enum sched_policy_type { SCHED_POLICY_SINGLE, /* requests will be sent to ctxs and dev_id */ SCHED_POLICY_DEV, + /* Hard calculation and soft calculation interval loop call */ + SCHED_POLICY_LOOP, + /* Perform heterogeneous calculations through ctx of session key */ + SCHED_POLICY_HUNGRY, + /* Instructions to accelerate heterogeneous computing */ + SCHED_POLICY_INSTR, SCHED_POLICY_BUTT, }; @@ -35,6 +42,7 @@ struct sched_params { __u32 begin; __u32 end; __u32 dev_id; + int ctx_prop; }; typedef int (*user_poll_func)(__u32 pos, __u32 expect, __u32 *count); diff --git a/wd_sched.c b/wd_sched.c index 19936fd..2efa2aa 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -1,374 +1,1442 @@ // SPDX-License-Identifier: Apache-2.0 /* - * Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved. * Copyright 2020-2021 Linaro ltd. + * + * Scheduler: Simplified Pure Hash Table with Dynamic Context Expansion + * + * Key improvements: + * - Single global hash table with (region_id, mode, op_type, prop) dimensions + * - Segment list for non-contiguous ctx ranges + * - Dual-domain queues for session key. + * - Dynamic ctx expansion in HUNGRY mode based on load threshold + * - Packet reception is handled through the active queues in the session key. + * - Simplified sched_init: only allocate one sync + one async ctx + * - Removed redundant wd_sched_info layer */ #define _GNU_SOURCE #include <stdlib.h> #include <stdbool.h> +#include <string.h> #include <sched.h> #include <numa.h> +#include <limits.h> +#include <pthread.h> #include "wd_sched.h" +#include "wd_alg.h" +#include "wd_internal.h" -#define MAX_POLL_TIMES 1000 +#define MAX_POLL_TIMES 1000 +#define HUNGRY_LOAD_THRESHOLD 256 +#define SKEY_CTX_MAX_NUM 16 +#define SKEY_MAX_THREAD_NUM 64 +#define SKEY_LOAD_UPDATE_INTERVAL 1 +#define HW_QUEUE_FULL_DEPTH 1024 +#define MAX_NUMA_NODES (NUMA_NUM_NODES >> 5) + +/* ============================================================================ + * Hash Table Configuration + * ============================================================================ + */ +#define WD_SCHED_MAX_BUCKETS 512 +#define WD_SCHED_MIN_BUCKETS 32 +#define WD_SCHED_LOAD_FACTOR 0.75f +#define HASH_PRIME1 73 +#define HASH_PRIME2 13 +#define HASH_PRIME3 7 +#define HASH_PRIME4 11 + +/* ============================================================================ + * Scheduling Region Mode + * ============================================================================ + */ enum sched_region_mode { SCHED_MODE_SYNC = 0, SCHED_MODE_ASYNC = 1, SCHED_MODE_BUTT }; -/* - * sched_key - The key if schedule region. - * @numa_id: The schedule numa region id. - * @mode: Sync mode:0, async_mode:1 - * @type: Service type , the value must smaller than type_num. - * @sync_ctxid: alloc ctx id for sync mode - * @async_ctxid: alloc ctx id for async mode - */ -struct sched_key { - int numa_id; - __u8 type; - __u8 mode; - __u32 sync_ctxid; - __u32 async_ctxid; - __u32 dev_id; -}; +/* ============================================================================ + * Segment List for Domain Index Organization + * ============================================================================ + */ -/* - * struct sched_ctx_range - define one ctx pos. - * @begin: the start pos in ctxs of config. - * @end: the end pos in ctxx of config. - * @last: the last one which be distributed. - * @valid: the region used flag. - * @lock: lock the currentscheduling region. - */ -struct sched_ctx_region { +/** + * wd_sched_ctx_segment - Contiguous segment of ctx indices in domain + * @begin: Start index of this segment + * @end: End index of this segment (inclusive) + * @next: Pointer to next segment in the linked list + * + * Supports non-contiguous ctx ranges via segment list. + */ +struct wd_sched_ctx_segment { __u32 begin; __u32 end; - __u32 last; + struct wd_sched_ctx_segment *next; +}; + +/* ============================================================================ + * Session key domain cache processing. + * ============================================================================ + */ + +/** + * wd_sched_domain_idx_cache - Simplified fixed array cache for skey domains + * + * Design principles: + * - Fixed array for cache-friendly memory layout + * - Atomic operations for lock-free load tracking + * - Simple RR and load balancing strategies + * - Maximum 16 queues per thread (typical usage) + */ +struct wd_sched_domain_idx_cache { + /* Queue index array */ + __u32 idx_list[SKEY_CTX_MAX_NUM]; /* Array of ctx indices */ + __u32 load_values[SKEY_CTX_MAX_NUM]; /* Atomic load counters */ + __u32 valid_count; /* Number of valid queues */ + + /* Scheduling state */ + __u32 rr_ptr; /* Round-robin pointer */ + __u32 min_load_idx; /* Cached min load index */ + __u32 op_counter; /* Operation counter for updates */ + __u8 load_decreased; /* Non-zero if poll ever decremented load */ + + /* Configuration */ + __u32 update_interval; /* Min load update interval */ + __u8 policy; /* Scheduling policy */ + + /* Synchronization */ + pthread_mutex_t cache_lock; /* Lock for structure modifications */ +}; + +/** + * wd_sched_ctx_domain - Scheduling domain with four dimensions + * @region_id: Region identifier (numa_id or device_id) + * @mode: Context mode (SYNC/ASYNC) + * @op_type: Operation type + * @prop: Property (e.g., device type: HW, CE, SOFT) + * @segments: Linked list of context ranges + * @segment_count: Number of segments + * @total_ctx_count: Total contexts across all segments + * @current_segment: Current segment pointer for round-robin + * @current_pos: Current position within segment + * @valid: Domain validity flag + * @lock: Synchronization spinlock + */ +struct wd_sched_ctx_domain { + int region_id; + __u8 mode; + __u32 op_type; + __u8 prop; + + struct wd_sched_ctx_segment *segments; + __u32 segment_count; + __u32 total_ctx_count; + + struct wd_sched_ctx_segment *current_segment; + __u32 current_pos; bool valid; + pthread_mutex_t lock; }; -/* - * wd_sched_info - define the context of the scheduler. - * @ctx_region: define the map for the comp ctxs, using for quickly search. - * the x range: two(sync and async), the y range: - * two(e.g. comp and uncomp) the map[x][y]'s value is the ctx - * begin and end pos. - * @valid: the region used flag. - */ -struct wd_sched_info { - struct sched_ctx_region *ctx_region[SCHED_MODE_BUTT]; - bool valid; +/** + * wd_sched_domain_hash_node - Hash table collision chain node + */ +struct wd_sched_domain_hash_node { + struct wd_sched_ctx_domain domain; + struct wd_sched_domain_hash_node *next; }; -struct dev_region_map { +/** + * wd_sched_domain_hash_table - Pure dynamic hash table for scheduling domains + * @buckets: Hash table bucket array + * @bucket_size: Number of buckets + * @lock: Read-write lock for concurrent access + */ +struct wd_sched_domain_hash_table { + struct wd_sched_domain_hash_node **buckets; + __u32 bucket_size; + pthread_mutex_t lock; +}; + +/* ============================================================================ + * Dual-Domain Structure for Session Key + * ============================================================================ + */ + +/** + * wd_sched_key_domain - Session domain with min-heap + * @idx_cache: Index cache with min-heap for load-based selection + * @lock: Synchronization spinlock + * @expanded_count: Track how many times ctx has been expanded + */ +struct wd_sched_key_domain { + struct wd_sched_domain_idx_cache idx_cache; + pthread_mutex_t lock; + __u32 expanded_count; +}; + +/** + * wd_sched_key - Session-level scheduling key + * @region_id: Region identifier + * @type: Operation type + * @mode: Current mode (SYNC/ASYNC) + * @dev_id: Device identifier (for SCHED_POLICY_DEV) + * @ctx_prop: Context property + * @is_stream: Stream mode flag + * @prio_mode: Priority mode + * @pkt_size: Current packet size + * @sync_domain: Min-heap domain for sync contexts + * @async_domain: Min-heap domain for async contexts + * @lock: Synchronization spinlock + */ +struct wd_sched_key { + int region_id; + __u8 type; + __u8 mode; __u32 dev_id; - __u32 region_id; + __u8 ctx_prop; + __u16 is_stream; + __u16 prio_mode; + __u32 pkt_size; + + struct wd_sched_key_domain sync_domain; + struct wd_sched_key_domain async_domain; + + pthread_mutex_t lock; + __u32 poll_lock; + __u32 refcount; + + /* Compat filtering parameters for session-ctx matching */ + const char *alg_name; + struct wd_ctx_internal *ctxs; }; -/* - * wd_sched_ctx - define the context of the scheduler. - * @policy: define the policy of the scheduler. - * @numa_num: the max numa numbers of the scheduler. - * @type_num: the max operation types of the scheduler. - * @poll_func: the task's poll operation function. - * @numa_map: a map of cpus to devices. - * @sched_info: the context of the scheduler. + +/** + * wd_sched_ctx - Main scheduler context + * @policy: Scheduling policy type + * @type_num: Number of operation types + * @mode_num: Number of modes (SYNC/ASYNC) + * @region_num: Number of regions (numa or devices) + * @poll_func: Poll function for receiving responses + * @domain_hash_table: Global hash table for all domains + * @skey_num: Number of active session keys + * @skey_lock: Lock for skey array + * @skey: Array of session keys */ struct wd_sched_ctx { __u32 policy; __u32 type_num; - __u16 numa_num; - __u16 dev_num; + __u32 mode_num; + __u16 region_num; + user_poll_func poll_func; - int numa_map[NUMA_NUM_NODES]; - struct dev_region_map dev_id_map[DEVICE_REGION_MAX]; - struct wd_sched_info sched_info[0]; + struct wd_sched_domain_hash_table *domain_hash_table; + + __u32 skey_num; + pthread_mutex_t skey_lock; + struct wd_sched_key *skey[SKEY_MAX_THREAD_NUM]; }; -static bool sched_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +/* ============================================================================ + * Hash Table Core Operations + * ============================================================================ + */ + +static bool wd_sched_is_prime(__u32 n) { - if (key->numa_id >= sched_ctx->numa_num || key->mode >= SCHED_MODE_BUTT || - key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's numa: %d, mode: %u, type: %u!\n", - key->numa_id, key->mode, key->type); + __u32 i; + + if (n <= 1) return false; + if (n <= 3) + return true; + if (n % 2 == 0 || n % 3 == 0) + return false; + + for (i = 5; i * i <= n; i += 6) { + if (n % i == 0 || n % (i + 2) == 0) + return false; } return true; } -/* - * sched_get_ctx_range - Get ctx range from ctx_map by the wd comp arg +static __u32 wd_sched_find_prime(__u32 n) +{ + while (!wd_sched_is_prime(n)) + n++; + return n; +} + +static __u32 wd_sched_compute_bucket_size(__u32 estimated_entries) +{ + __u32 target_size; + + target_size = (estimated_entries * 4) / 3; + + if (target_size < WD_SCHED_MIN_BUCKETS) + target_size = WD_SCHED_MIN_BUCKETS; + if (target_size > WD_SCHED_MAX_BUCKETS) + target_size = WD_SCHED_MAX_BUCKETS; + + return wd_sched_find_prime(target_size); +} + +/** + * wd_sched_hash_compute - Compute hash value for four-dimensional domain key + * @region_id: Region identifier + * @mode: Context mode + * @op_type: Operation type + * @prop: Property + * @bucket_size: Hash table bucket count + * + * Combines four dimensions using prime number multipliers. */ -static struct sched_ctx_region *sched_get_ctx_range(struct wd_sched_ctx *sched_ctx, - const struct sched_key *key) +static inline __u32 wd_sched_hash_compute(int region_id, __u8 mode, + __u32 op_type, __u8 prop, __u32 bucket_size) { - struct wd_sched_info *sched_info; - int numa_id; + __u32 hash; - sched_info = sched_ctx->sched_info; - if (key->numa_id >= 0 && - sched_info[key->numa_id].ctx_region[key->mode][key->type].valid) - return &sched_info[key->numa_id].ctx_region[key->mode][key->type]; + hash = (region_id * HASH_PRIME1) + (mode * HASH_PRIME2) + + (op_type * HASH_PRIME3) + (prop * HASH_PRIME4); + return hash % bucket_size; +} + +static inline bool wd_sched_domain_key_match( + int region_id1, __u8 mode1, __u32 op_type1, __u8 prop1, + int region_id2, __u8 mode2, __u32 op_type2, __u8 prop2) +{ + return (region_id1 == region_id2 && mode1 == mode2 && + op_type1 == op_type2 && prop1 == prop2); +} + +/** + * wd_sched_hash_table_create - Create hash table + * @estimated_entries: Estimated number of entries + * + * Returns: Initialized hash table or NULL on error + */ +static struct wd_sched_domain_hash_table * +wd_sched_hash_table_create(__u32 estimated_entries) +{ + struct wd_sched_domain_hash_table *table; + __u32 bucket_size; + int ret; - /* If the key->numa_id is not exist, we should scan for a region */ - for (numa_id = 0; numa_id < sched_ctx->numa_num; numa_id++) { - if (sched_info[numa_id].ctx_region[key->mode][key->type].valid) - return &sched_info[numa_id].ctx_region[key->mode][key->type]; + table = calloc(1, sizeof(*table)); + if (!table) + return NULL; + + bucket_size = wd_sched_compute_bucket_size(estimated_entries); + + table->buckets = calloc(bucket_size, sizeof(*table->buckets)); + if (!table->buckets) { + free(table); + return NULL; } - return NULL; + table->bucket_size = bucket_size; + ret = pthread_mutex_init(&table->lock, NULL); + if (ret) { + free(table->buckets); + free(table); + return NULL; + } + + return table; } -/* - * sched_get_next_pos_rr - Get next resource pos by RR schedule. - * The second para is reserved for future. +static void wd_sched_hash_table_destroy(struct wd_sched_domain_hash_table *table) +{ + struct wd_sched_domain_hash_node *node, *next; + struct wd_sched_ctx_segment *seg, *next_seg; + __u32 i; + + if (!table) + return; + + for (i = 0; i < table->bucket_size; i++) { + node = table->buckets[i]; + while (node) { + next = node->next; + + /* Release segment linked list */ + seg = node->domain.segments; + while (seg) { + next_seg = seg->next; + free(seg); + seg = next_seg; + } + + pthread_mutex_destroy(&node->domain.lock); + free(node); + node = next; + } + } + + pthread_mutex_destroy(&table->lock); + free(table->buckets); + free(table); +} + +static struct wd_sched_ctx_domain * +wd_sched_hash_table_lookup(struct wd_sched_domain_hash_table *table, + int region_id, __u8 mode, __u32 op_type, __u8 prop) +{ + struct wd_sched_domain_hash_node *node; + struct wd_sched_ctx_domain *domain = NULL; + __u32 hash_idx; + + if (!table) + return NULL; + + pthread_mutex_lock(&table->lock); + hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size); + node = table->buckets[hash_idx]; + while (node) { + if (wd_sched_domain_key_match( + node->domain.region_id, node->domain.mode, node->domain.op_type, + node->domain.prop, region_id, mode, op_type, prop)) { + domain = &node->domain; + break; + } + node = node->next; + } + pthread_mutex_unlock(&table->lock); + + return domain; +} + +static struct wd_sched_ctx_domain * +wd_sched_hash_table_insert(struct wd_sched_domain_hash_table *table, + int region_id, __u8 mode, __u32 op_type, __u8 prop) +{ + struct wd_sched_domain_hash_node *new_node; + struct wd_sched_ctx_domain *existing; + __u32 hash_idx; + int ret; + + if (!table) + return NULL; + + existing = wd_sched_hash_table_lookup(table, region_id, mode, op_type, prop); + if (existing) + return existing; + + pthread_mutex_lock(&table->lock); + hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size); + /* Alloc and initialize new domain */ + new_node = calloc(1, sizeof(*new_node)); + if (!new_node) { + pthread_mutex_unlock(&table->lock); + return NULL; + } + + /* Initialize new domain */ + new_node->domain.region_id = region_id; + new_node->domain.mode = mode; + new_node->domain.op_type = op_type; + new_node->domain.prop = prop; + new_node->domain.segments = NULL; + new_node->domain.segment_count = 0; + new_node->domain.total_ctx_count = 0; + new_node->domain.current_segment = NULL; + new_node->domain.current_pos = 0; + new_node->domain.valid = false; + + ret = pthread_mutex_init(&new_node->domain.lock, NULL); + if (ret) { + pthread_mutex_unlock(&table->lock); + free(new_node); + return NULL; + } + + new_node->next = table->buckets[hash_idx]; + table->buckets[hash_idx] = new_node; + pthread_mutex_unlock(&table->lock); + + return &new_node->domain; +} + +/* ============================================================================ + * Segment List Operations + * ============================================================================ + */ + +/** + * wd_sched_domain_add_segment - Add context range segment to domain + * @domain: Target domain + * @begin: Start context index + * @end: End context index (inclusive) + * + * Supports non-contiguous context ranges via segment list. */ -static __u32 sched_get_next_pos_rr(struct sched_ctx_region *region, void *para) +static int wd_sched_domain_add_segment(struct wd_sched_ctx_domain *domain, + __u32 begin, __u32 end) { + struct wd_sched_ctx_segment *seg, *new_seg; + + if (!domain || begin > end) + return -WD_EINVAL; + + new_seg = calloc(1, sizeof(*new_seg)); + if (!new_seg) + return -WD_ENOMEM; + + new_seg->begin = begin; + new_seg->end = end; + new_seg->next = NULL; + + pthread_mutex_lock(&domain->lock); + + /* Append to segment list tail */ + if (!domain->segments) { + domain->segments = new_seg; + } else { + seg = domain->segments; + while (seg->next) + seg = seg->next; + seg->next = new_seg; + } + + domain->segment_count++; + domain->total_ctx_count += (end - begin + 1); + + /* Initialize polling state */ + if (!domain->current_segment) + domain->current_segment = domain->segments; + + pthread_mutex_unlock(&domain->lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_domain_get_next_rr - Get next context via round-robin from domain + * @domain: Source domain + * + * Returns: Next global queue index in round-robin order + * Time complexity: O(1) + */ +static __u32 wd_sched_domain_get_next_rr(struct wd_sched_ctx_domain *domain) +{ + __u32 ctx_idx; __u32 pos; - pthread_mutex_lock(&region->lock); + if (!domain || !domain->segments || !domain->total_ctx_count) + return INVALID_POS; - pos = region->last; + pthread_mutex_lock(&domain->lock); + if (!domain->current_segment) + domain->current_segment = domain->segments; - if (pos < region->end) - region->last++; - else - region->last = region->begin; + pos = domain->current_pos; + + /* Calculate global queue number: segment.begin + relative position */ + ctx_idx = domain->current_segment->begin + pos; - pthread_mutex_unlock(&region->lock); + /* Move to next position */ + if (pos + 1 < domain->current_segment->end - domain->current_segment->begin + 1) { + /* Within same segment */ + domain->current_pos = pos + 1; + } else if (domain->current_segment->next) { + /* Move to next segment */ + domain->current_segment = domain->current_segment->next; + domain->current_pos = 0; + } else { + /* Loop back to beginning */ + domain->current_segment = domain->segments; + domain->current_pos = 0; + } + + pthread_mutex_unlock(&domain->lock); - return pos; + return ctx_idx; } -/* - * session_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/* ============================================================================ + * SKey Domain Cache Management Functions + * ============================================================================ + */ +/** + * wd_sched_skey_cache_init - Initialize skey domain cache + * @cache: Pointer to cache structure + * @policy: Scheduling policy * @sched_type: Scheduling policy type (cannot modify per + * API contract) + * + * Initialize fixed array cache with invalid positions and zero loads. + */ +static int wd_sched_skey_cache_init(struct wd_sched_domain_idx_cache *cache, + __u8 policy) +{ + int i; + + if (!cache) { + WD_ERR("invalid: cache pointer is NULL!\n"); + return -WD_EINVAL; + } + + /* Initialize array with invalid positions */ + for (i = 0; i < SKEY_CTX_MAX_NUM; i++) { + cache->idx_list[i] = INVALID_POS; + __atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED); + } + + /* Initialize atomic counters */ + __atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->op_counter, 0, __ATOMIC_RELAXED); + cache->load_decreased = 0; + + /* Set configuration */ + cache->valid_count = 0; + cache->update_interval = SKEY_LOAD_UPDATE_INTERVAL; + cache->policy = policy; + + /* Initialize structure lock */ + if (pthread_mutex_init(&cache->cache_lock, NULL)) { + WD_ERR("failed to init cache lock!\n"); + return -WD_EINVAL; + } + + return WD_SUCCESS; +} +/** + * wd_sched_skey_cache_uninit - Cleanup skey domain cache + * @cache: Pointer to cache structure + * + * Release resources and reset cache state. + */ +static void wd_sched_skey_cache_uninit(struct wd_sched_domain_idx_cache *cache) +{ + if (!cache) + return; + + pthread_mutex_destroy(&cache->cache_lock); + + /* Reset cache state */ + for (int i = 0; i < SKEY_CTX_MAX_NUM; i++) { + cache->idx_list[i] = INVALID_POS; + __atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED); + } + + cache->valid_count = 0; + cache->load_decreased = 0; +} +/** + * wd_sched_skey_add_ctx - Add ctx to skey domain cache + * @cache: Pointer to cache structure + * @ctx_id: Context ID to add + * + * Add ctx to next available position in fixed array. + * Returns 0 on success, negative error code on failure. + */ +static int wd_sched_skey_add_ctx(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_id) +{ + __u32 i; + + if (!cache || ctx_id == INVALID_POS) { + WD_ERR("invalid: parameters are NULL!\n"); + return -WD_EINVAL; + } + + pthread_mutex_lock(&cache->cache_lock); + /* Check if cache is full */ + if (cache->valid_count >= SKEY_CTX_MAX_NUM) { + pthread_mutex_unlock(&cache->cache_lock); + WD_ERR("invalid: skey cache full, cannot add more queues!\n"); + return -WD_EINVAL; + } + + /* Check for duplicate ctx_id */ + for (i = 0; i < cache->valid_count; i++) { + if (cache->idx_list[i] == ctx_id) { + WD_ERR("invalid: context %u already exists in skey cache at pos %u!\n", + ctx_id, i); + pthread_mutex_unlock(&cache->cache_lock); + return -WD_EEXIST; + } + } + + /* Update min load index if as the new ctx */ + __atomic_store_n(&cache->min_load_idx, cache->valid_count, __ATOMIC_RELAXED); + + /* Add to next available position */ + cache->idx_list[cache->valid_count] = ctx_id; + __atomic_store_n(&cache->load_values[cache->valid_count], 0, __ATOMIC_RELAXED); + cache->valid_count++; + pthread_mutex_unlock(&cache->cache_lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_skey_remove_ctx - Remove ctx from skey domain cache + * @cache: Pointer to cache structure + * @ctx_id: Context ID to remove + * + * Remove ctx by shifting array elements to maintain continuity. + * Returns 0 on success, negative error code if not found. + */ +static int wd_sched_skey_remove_ctx(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_id) +{ + __u32 i, current_min; + int found = 0; + + if (!cache) { + WD_ERR("invalid: cache pointer is NULL!\n"); + return -WD_EINVAL; + } + + pthread_mutex_lock(&cache->cache_lock); + /* Find and remove the ctx */ + for (i = 0; i < cache->valid_count; i++) { + if (cache->idx_list[i] == ctx_id) { + found = 1; + break; + } + } + + if (!found) { + WD_ERR("invalid: context %u not found in skey cache!\n", ctx_id); + pthread_mutex_unlock(&cache->cache_lock); + return -WD_ENODEV; + } + + /* Shift remaining elements to fill the gap */ + for (; i < cache->valid_count - 1; i++) { + cache->idx_list[i] = cache->idx_list[i + 1]; + __atomic_store_n(&cache->load_values[i], + __atomic_load_n(&cache->load_values[i + 1], __ATOMIC_RELAXED), + __ATOMIC_RELAXED); + } + + /* Clear last position */ + cache->idx_list[cache->valid_count - 1] = INVALID_POS; + __atomic_store_n(&cache->load_values[cache->valid_count - 1], 0, __ATOMIC_RELAXED); + cache->valid_count--; + + /* Reset pointers if cache becomes empty */ + if (!cache->valid_count) { + __atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + } else { + /* Adjust min load index if necessary */ + current_min = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED); + if (current_min >= cache->valid_count) + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + } + pthread_mutex_unlock(&cache->cache_lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_update_min_load - Update cached min load index + * @cache: Pointer to cache structure + * @ctxs: Context array for hw_load lookup, or NULL to use cache->load_values * - * The user must init the schedule info through wd_sched_rr_instance + * Scan valid queues to find the one with minimum load. + * When ctxs is provided, reads per-ctx hw_load for cross-session accuracy. */ -static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, - const int sched_mode) +static void wd_sched_update_min_load(struct wd_sched_domain_idx_cache *cache, + struct wd_ctx_internal *ctxs) { - struct sched_ctx_region *region = NULL; - bool ret; + __u32 min_load = UINT_MAX; + __u32 min_idx = 0; + __u32 i, load; - key->mode = sched_mode; - ret = sched_key_valid(sched_ctx, key); - if (!ret) + if (!cache->valid_count) + return; + + for (i = 0; i < cache->valid_count; i++) { + if (ctxs) + load = __atomic_load_n(&ctxs[cache->idx_list[i]].hw_load, + __ATOMIC_RELAXED); + else + load = __atomic_load_n(&cache->load_values[i], __ATOMIC_RELAXED); + if (load < min_load) { + min_load = load; + min_idx = i; + } + } + + __atomic_store_n(&cache->min_load_idx, min_idx, __ATOMIC_RELAXED); +} + +/** + * wd_sched_skey_pick_next - Pick next ctx from skey domain cache + * @cache: Pointer to cache structure + * @ctx_idx: Output index within cache array + * @ctxs: Context array for hw_load lookup, or NULL + * + * Select next ctx based on scheduling policy: + * - RR: Simple round-robin selection + * - HUNGRY: Choose ctx with minimum load (hw_load if ctxs available) + * + * Returns selected ctx index, or INVALID_POS if no valid ctx. + */ +static __u32 wd_sched_skey_pick_next(struct wd_sched_domain_idx_cache *cache, + __u32 *ctx_idx, struct wd_ctx_internal *ctxs) +{ + __u32 selected_idx; + __u32 op_count; + + if (!cache || !cache->valid_count) return INVALID_POS; - region = sched_get_ctx_range(sched_ctx, key); - if (!region) + switch (cache->policy) { + case SCHED_POLICY_RR: + case SCHED_POLICY_NONE: + case SCHED_POLICY_SINGLE: + case SCHED_POLICY_DEV: + case SCHED_POLICY_LOOP: + case SCHED_POLICY_INSTR: + /* Round-robin: atomic increment and module */ + selected_idx = __atomic_fetch_add(&cache->rr_ptr, 1, __ATOMIC_RELAXED) % + cache->valid_count; + break; + case SCHED_POLICY_HUNGRY: + /* Update min load periodically */ + op_count = __atomic_fetch_add(&cache->op_counter, 1, __ATOMIC_RELAXED); + if (op_count % cache->update_interval == 0) + wd_sched_update_min_load(cache, ctxs); + + /* Load balancing: use cached min load index */ + selected_idx = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED); + break; + default: + WD_ERR("invalid: unknown scheduling policy %d!\n", cache->policy); + selected_idx = INVALID_POS; + break; + } + + /* Ensure index is within valid range */ + if (selected_idx >= cache->valid_count) return INVALID_POS; - return sched_get_next_pos_rr(region, NULL); + *ctx_idx = selected_idx; + return cache->idx_list[selected_idx]; } -static handle_t session_sched_init(handle_t h_sched_ctx, void *sched_param) +/** + * wd_sched_skey_update_load - Update load for a specific ctx + * @cache: Pointer to cache structure + * @ctx_idx: Context index in list + * @delta: Load delta (positive for send, negative for receive) + * + * Atomically update load counter for the specified ctx. + * Returns 0 on success, negative error code if ctx not found. + */ +static int wd_sched_skey_update_load(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_idx, int delta) { - struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - struct sched_params *param = (struct sched_params *)sched_param; - struct sched_key *skey; + /* Atomic update without locking, ctx_idx's value is guaranteed by the caller. */ + if (delta > 0) + __atomic_fetch_add(&cache->load_values[ctx_idx], delta, __ATOMIC_RELAXED); + else { + __atomic_fetch_sub(&cache->load_values[ctx_idx], -delta, __ATOMIC_RELAXED); + if (unlikely(!cache->load_decreased)) + cache->load_decreased = 1; + } + return WD_SUCCESS; +} + +/* ============================================================================ + * Session Key Domain Initialization + * ============================================================================ + */ + +/** + * wd_sched_skey_domain_init - Initialize session domain with min-heap + * @key_domain: Target key domain + * @ctx_idx: context indices idx + * @policy: current session's policy + * + * Initializes dual-domain structure for session. + */ +static int wd_sched_skey_domain_init(struct wd_sched_key_domain *key_domain, + __u32 ctx_idx, __u8 policy) +{ + int ret; + + if (!key_domain) + return -WD_EINVAL; + + ret = wd_sched_skey_cache_init(&key_domain->idx_cache, policy); + if (ret) + return ret; + + ret = wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx); + if (ret) + goto init_err; + + ret = pthread_mutex_init(&key_domain->lock, NULL); + if (ret) + goto add_ctx_err; + + key_domain->expanded_count = 0; + + return WD_SUCCESS; + +add_ctx_err: + wd_sched_skey_remove_ctx(&key_domain->idx_cache, ctx_idx); +init_err: + wd_sched_skey_cache_uninit(&key_domain->idx_cache); + return ret; +} + +/** + * wd_sched_skey_domain_destroy - Release session domain resources + */ +static void wd_sched_skey_domain_destroy(struct wd_sched_key_domain *key_domain) +{ + if (!key_domain) + return; + + pthread_mutex_destroy(&key_domain->lock); + wd_sched_skey_cache_uninit(&key_domain->idx_cache); +} + + +/** + * wd_sched_poll_skey - Poll contexts for scheduler session + * @sched_ctx: Scheduler context + * @skey: Session key + * @expect: Expected number of responses + * @count: Actual response count (output) + * + * Polls all contexts in session domains and updates load values. + */ +static int wd_sched_poll_skey(struct wd_sched_ctx *sched_ctx, struct wd_sched_key *skey, + __u32 expect, __u32 *count) +{ + struct wd_sched_domain_idx_cache *cache; + __u32 ctx_list_num = 0; + __u32 sum_poll_num = 0; + bool hungry_policy; + __u32 poll_num; + __u32 idx, i; + int ret = 0; + + /* Get cache pointer and check if HUNGRY policy */ + cache = &skey->async_domain.idx_cache; + ctx_list_num = cache->valid_count; + hungry_policy = (cache->policy == SCHED_POLICY_HUNGRY); + + /* Poll async domain contexts */ + for (i = 0; i < ctx_list_num; i++) { + idx = cache->idx_list[i]; + if (idx == INVALID_POS) + continue; + poll_num = 0; + ret = sched_ctx->poll_func(idx, expect, &poll_num); + if (poll_num > 0) + sum_poll_num += poll_num; + + /* Update load value for this context */ + if (hungry_policy && poll_num > 0) { + if (skey->ctxs) + __atomic_fetch_sub(&skey->ctxs[idx].hw_load, + poll_num, __ATOMIC_RELAXED); + wd_sched_skey_update_load(cache, i, -poll_num); + } + + if (ret < 0 && ret != -WD_EAGAIN) + break; + } + *count = sum_poll_num; + + return ret; +} + +/* ============================================================================ + * Utility Functions + * ============================================================================ + */ +static inline bool sched_skey_get_ref(struct wd_sched_key *skey) +{ + if (!skey) + return false; + return __atomic_fetch_add(&skey->refcount, 1, __ATOMIC_ACQUIRE) > 0; +} + +static inline bool sched_skey_put_ref(struct wd_sched_key *skey) +{ + return __atomic_fetch_sub(&skey->refcount, 1, __ATOMIC_RELEASE) == 1; +} + +static int sched_skey_param_init(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey) +{ + __u32 i; + + pthread_mutex_lock(&sched_ctx->skey_lock); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + if (!sched_ctx->skey[i]) { + sched_ctx->skey[i] = skey; + if (sched_ctx->skey_num < SKEY_MAX_THREAD_NUM) + sched_ctx->skey_num++; + pthread_mutex_unlock(&sched_ctx->skey_lock); + return 0; + } + } + pthread_mutex_unlock(&sched_ctx->skey_lock); + WD_ERR("invalid: skey node number exceeds SKEY_MAX_THREAD_NUM(%d)!\n", + SKEY_MAX_THREAD_NUM); + return -WD_ENOMEM; +} + +static void sched_skey_param_uninit(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey) +{ + __u32 i; + + if (!sched_ctx || !skey) + return; + + pthread_mutex_lock(&sched_ctx->skey_lock); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + if (sched_ctx->skey[i] == skey) { + sched_ctx->skey[i] = NULL; + pthread_mutex_unlock(&sched_ctx->skey_lock); + return; + } + } + pthread_mutex_unlock(&sched_ctx->skey_lock); + WD_ERR("warning: skey %p not found in sched_ctx array\n", skey); +} + +static handle_t sched_session_common_init(struct wd_sched_ctx *sched_ctx, + struct sched_params *param) +{ + struct wd_sched_key *skey; unsigned int node; if (getcpu(NULL, &node)) { WD_ERR("failed to get node, errno %d!\n", errno); return (handle_t)(-errno); } - if (node == (unsigned int)NUMA_NO_NODE) { - WD_ERR("invalid: failed to get numa node!\n"); - return (handle_t)(-WD_EINVAL); - } if (!sched_ctx) { WD_ERR("invalid: sched ctx is NULL!\n"); return (handle_t)(-WD_EINVAL); } - skey = malloc(sizeof(struct sched_key)); + skey = malloc(sizeof(struct wd_sched_key)); if (!skey) { WD_ERR("failed to alloc memory for session sched key!\n"); return (handle_t)(-WD_ENOMEM); } + memset(skey, 0, sizeof(struct wd_sched_key)); if (!param) { - memset(skey, 0, sizeof(struct sched_key)); - skey->numa_id = sched_ctx->numa_map[node]; + skey->region_id = node; if (wd_need_debug()) WD_DEBUG("session don't set scheduler parameters!\n"); - } else if (param->numa_id < 0) { - skey->type = param->type; - skey->numa_id = sched_ctx->numa_map[node]; } else { + if (sched_ctx->policy == SCHED_POLICY_DEV) + skey->region_id = param->dev_id; + else if (param->numa_id >= 0) + skey->region_id = param->numa_id; + else + skey->region_id = node; skey->type = param->type; - skey->numa_id = param->numa_id; + skey->ctx_prop = param->ctx_prop; } + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + __atomic_store_n(&skey->refcount, 1, __ATOMIC_RELAXED); - if (skey->numa_id < 0) { - WD_ERR("failed to get valid sched numa region!\n"); - goto out; - } + return (handle_t)skey; +} - skey->sync_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); - skey->async_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); - if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { - WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); - goto out; +static __u16 sched_get_poll_skey_idx(struct wd_sched_ctx *sched_ctx) +{ + /* Thread-local sequence number with atomic global counter */ + static __thread __u32 thread_seq = UINT32_MAX; + static __u32 global_seq_counter; + __u32 skey_num = sched_ctx->skey_num; + __u16 start_pos; + + if (unlikely(!sched_ctx || !skey_num)) + return skey_num; + + /* Assign unique sequence number on first call */ + if (unlikely(thread_seq == UINT32_MAX)) { + thread_seq = __atomic_fetch_add(&global_seq_counter, 1, __ATOMIC_RELAXED); + + /* Basic overflow protection */ + if (thread_seq >= UINT32_MAX - 1) + WD_DEBUG("Thread sequence counter approaching limit: %u\n", thread_seq); } - return (handle_t)skey; + /* Calculate start_pos based on thread_seq */ + start_pos = thread_seq % skey_num; -out: - free(skey); - return (handle_t)(-WD_EINVAL); + return start_pos; } -/* - * session_pick_next_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/** + * session_sched_init_ctx - Pre-fetch single context from domain for session + * @sched_ctx: Scheduler context + * @skey: session scheduler param + * @sched_mode: Mode (SYNC/ASYNC) * - * The user must init the schedule info through session_sched_init + * Returns: Context index from domain (compatible with alg_name if skey provided) */ -static __u32 session_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, - const int sched_mode) +static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, int sched_mode) { - struct sched_key *key = (struct sched_key *)sched_key; + struct wd_sched_ctx_domain *domain = NULL; + int region_id = skey->region_id; + __u32 op_type = skey->type; + __u8 prop = skey->ctx_prop; + __u32 ctx_idx; + __u16 r; + + if (sched_mode >= SCHED_MODE_BUTT || + op_type >= sched_ctx->type_num || prop >= UADK_ALG_TYPE_MAX) { + WD_ERR("invalid: region: %d, mode: %d, type: %u!, prop: %u\n", + region_id, sched_mode, op_type, prop); + return INVALID_POS; + } - if (unlikely(!h_sched_ctx || !key)) { - WD_ERR("invalid: sched ctx or key is NULL!\n"); + if (!sched_ctx->domain_hash_table) + return INVALID_POS; + + /* Try current region first */ + if (region_id >= 0) { + if (sched_ctx->policy == SCHED_POLICY_DEV || + region_id < sched_ctx->region_num) { + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + region_id, sched_mode, op_type, prop); + if (domain && domain->valid) + return wd_sched_domain_get_next_rr(domain); + } + } + + /* DEV policy must not cross region */ + if (sched_ctx->policy == SCHED_POLICY_DEV) return INVALID_POS; + + /* Cross-region fallback for other policies */ + for (r = 0; r < sched_ctx->region_num; r++) { + if ((int)r == region_id) + continue; + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + r, sched_mode, op_type, prop); + if (domain && domain->valid) { + ctx_idx = wd_sched_domain_get_next_rr(domain); + if (ctx_idx != INVALID_POS) + return ctx_idx; + } } - /* return in do task */ - if (sched_mode == CTX_MODE_SYNC) - return key->sync_ctxid; - return key->async_ctxid; + return INVALID_POS; } -static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, - __u32 end, __u32 expect, __u32 *count) +/** + * session_sched_domain_destroy - Destroy session domains + * @skey: Session key to destroy domains for + * + * Releases all resources associated with session domains. + */ +static void session_sched_domain_destroy(struct wd_sched_key *skey) { - __u32 poll_num = 0; - __u32 i; - int ret; + if (!skey) + return; - /* i is the pos of sched_ctxs, the max is end */ - for (i = begin; i <= end; i++) { - /* - * RR schedule, one time poll one package, - * poll_num is always not more than one here. - */ - ret = sched_ctx->poll_func(i, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) - return ret; - else if (ret == -EAGAIN) + /* Destroy both sync and async domains */ + wd_sched_skey_domain_destroy(&skey->sync_domain); + wd_sched_skey_domain_destroy(&skey->async_domain); +} + +static inline void sched_skey_poll_release(struct wd_sched_key *skey) +{ + __u32 prev; + + if (!skey) + return; + + prev = __atomic_fetch_sub(&skey->refcount, 1, __ATOMIC_ACQ_REL); + if (prev == 1) { + session_sched_domain_destroy(skey); + free(skey); + } +} + +static void sched_session_uninit(handle_t h_sched_ctx, handle_t h_sched_key) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_key *skey = (struct wd_sched_key *)h_sched_key; + + if (!skey) + return; + + /* Remove from skey array first to prevent new poll discovery. + * Poll threads already holding a pointer are protected by refcount. + */ + if (sched_ctx) + sched_skey_param_uninit(sched_ctx, skey); + + /* Drop creator reference. If no poll holds a ref, free now. + * Otherwise the last poll thread will cleanup via poll_release. + */ + if (sched_skey_put_ref(skey)) { + session_sched_domain_destroy(skey); + free(skey); + } +} + +/** + * session_sched_init_ctx_with_fallback - Pre-fetch a ctx for one sched_mode, + * trying user_prop first then falling + * back through other props in enum order. + * @sched_ctx: Scheduler context + * @skey: Session key (skey->ctx_prop is the user-specified prop; restored on return) + * @sched_mode: SCHED_MODE_SYNC or SCHED_MODE_ASYNC + * + * Returns: ctx index, or INVALID_POS if no prop has a usable ctx. + * + * Falls back only when the user-specified prop's domain is unavailable + * (e.g. HW driver unloaded). Stops at the first prop that yields a ctx; + * does NOT accumulate ctxs across props. + */ +static __u32 session_sched_init_ctx_with_fallback(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + int sched_mode) +{ + __u8 user_prop = skey->ctx_prop; + __u32 ctx_idx; + __u8 p; + + /* Try user-specified prop first */ + ctx_idx = session_sched_init_ctx(sched_ctx, skey, sched_mode); + if (ctx_idx != INVALID_POS) + return ctx_idx; + + /* Fallback: target prop unavailable, try remaining props in enum order */ + for (p = 0; p < UADK_ALG_TYPE_MAX; p++) { + if (p == user_prop) continue; - *count += poll_num; - if (*count == expect) + skey->ctx_prop = p; + ctx_idx = session_sched_init_ctx(sched_ctx, skey, sched_mode); + if (ctx_idx != INVALID_POS) break; } - return 0; + /* Restore user_prop so skey state is not mutated */ + skey->ctx_prop = user_prop; + return ctx_idx; } -static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, - __u32 expect, __u32 *count) +/** + * session_sched_domain_init - Initialize session domains with sync/async + * @sched_ctx: Scheduler context + * @skey: Session key to initialize + * @allow_fallback: If true, fall back to other props when skey->ctx_prop has + * no usable domain; if false, only try skey->ctx_prop. + * + * Pre-fetches sync and async contexts and initializes corresponding domains. + * Returns: 0 on success, negative error code on failure. + */ +static int session_sched_domain_init(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + bool allow_fallback) { - struct sched_ctx_region **region = sched_ctx->sched_info[numa_id].ctx_region; - __u32 begin, end; - __u32 i; - int ret; + __u32 sync_ctx, async_ctx; - for (i = 0; i < sched_ctx->type_num; i++) { - if (!region[SCHED_MODE_ASYNC][i].valid) - continue; + if (!sched_ctx || !skey) { + WD_ERR("invalid: sched_ctx or skey is NULL!\n"); + return -WD_EINVAL; + } + + if (allow_fallback) { + /* Pre-fetch with per-mode prop fallback */ + sync_ctx = session_sched_init_ctx_with_fallback(sched_ctx, skey, + SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx_with_fallback(sched_ctx, skey, + SCHED_MODE_ASYNC); + } else { + sync_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_ASYNC); + } + + if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) { + WD_ERR("invalid: no valid sync_ctx or async_ctx domain!\n"); + return -WD_EINVAL; + } + + /* Initialize sync domain if context is valid */ + if (sync_ctx != INVALID_POS) { + if (wd_sched_skey_domain_init(&skey->sync_domain, sync_ctx, sched_ctx->policy)) { + WD_ERR("failed to init sync domain!\n"); + return -WD_EINVAL; + } + } + + /* Initialize async domain if context is valid */ + if (async_ctx != INVALID_POS) { + if (wd_sched_skey_domain_init(&skey->async_domain, async_ctx, sched_ctx->policy)) { + WD_ERR("failed to init async domain!\n"); + /* Cleanup sync domain if async domain init failed */ + if (sync_ctx != INVALID_POS) + wd_sched_skey_domain_destroy(&skey->sync_domain); + return -WD_EINVAL; + } + } + + return WD_SUCCESS; +} + +/* ============================================================================ + * Scheduler Policy Functions + * ============================================================================ + */ +/** + * round_robin_sched_init - Initialize session with single sync and async ctx + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_param: Scheduling parameters (cannot modify per API contract) + * + * Allocates session key and pre-fetches one sync and one async context. + */ +static handle_t round_robin_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + struct wd_sched_key *skey; + handle_t hskey; + int ret = 0; + + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; + } + + skey = (struct wd_sched_key *)hskey; + /* RR: allow prop fallback so session creation survives target prop unload */ + ret = session_sched_domain_init(sched_ctx, skey, true); + if (ret) { + WD_ERR("failed to initialize session domains!\n"); + free(skey); + return (handle_t)(-WD_EINVAL); + } + + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); + return (handle_t)(-WD_ENOMEM); + } + + return hskey; +} + +/** + * round_robin_pick_next_ctx - Pick context with load-based selection + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) + * + * Returns: Context index with minimum load + * Time complexity: O(1) + */ +static __u32 round_robin_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_key_domain *domain; + __u32 min_ctx, ctx_idx; - begin = region[SCHED_MODE_ASYNC][i].begin; - end = region[SCHED_MODE_ASYNC][i].end; - ret = session_poll_region(sched_ctx, begin, end, expect, count); - if (unlikely(ret)) - return ret; + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; } - return 0; + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; + + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) + return INVALID_POS; + + return min_ctx; } -/* - * session_poll_policy - The polling policy matches the pick next ctx. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @cfg: The global resoure info. - * @expect: User expect poll msg num. - * @count: The actually poll num. +/** + * round_robin_poll_policy - Poll policy for session scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) * - * The user must init the schedule info through wd_sched_rr_instance, the - * func interval will not check the valid, becouse it will affect performance. + * Returns: Status code */ -static int session_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +static int round_robin_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - struct wd_sched_info *sched_info; - __u32 loop_time = 0; - __u32 last_count = 0; - __u16 i, region_mum; - int ret; + struct wd_sched_key *skey; + __u16 i, tpos, start_pos; + __u32 poll_num, skey_num; + int ret = -WD_EAGAIN; + __u32 sum_count = 0; if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); return -WD_EINVAL; } - if (unlikely(sched_ctx->numa_num > NUMA_NUM_NODES)) { - WD_ERR("invalid: ctx's numa number is %u!\n", sched_ctx->numa_num); - return -WD_EINVAL; + /* Randomize the initial query position. */ + skey_num = sched_ctx->skey_num; + start_pos = sched_get_poll_skey_idx(sched_ctx); + if (!skey_num || start_pos >= sched_ctx->skey_num) { + *count = 0; + return -WD_EAGAIN; } - sched_info = sched_ctx->sched_info; - if (sched_ctx->policy == SCHED_POLICY_DEV) - region_mum = sched_ctx->dev_num; - else - region_mum = sched_ctx->numa_num; - - /* - * Try different region's ctx if we can't receive any - * package last time, it is more efficient. In most - * bad situation, poll ends after MAX_POLL_TIMES loop. - */ - while (++loop_time < MAX_POLL_TIMES) { - for (i = 0; i < region_mum;) { - /* If current numa is not valid, find next. */ - if (!sched_info[i].valid) { - i++; - continue; - } + /* Query the queues on each skey separately. */ + for (i = 0; i < skey_num; i++) { + tpos = (start_pos + i) % skey_num; + skey = sched_ctx->skey[tpos]; - last_count = *count; - ret = session_poll_policy_rr(sched_ctx, i, expect, count); - if (unlikely(ret)) - return ret; + if (unlikely(!skey)) + continue; - if (expect == *count) - return 0; + if (!sched_skey_get_ref(skey)) + continue; - /* - * If no package is received, find next numa, - * otherwise, keep receiving packets at this node. - */ - if (last_count == *count) - i++; + if (__atomic_test_and_set(&skey->poll_lock, __ATOMIC_ACQUIRE)) { + sched_skey_poll_release(skey); + continue; } + + ret = wd_sched_poll_skey(sched_ctx, skey, expect, &poll_num); + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + sched_skey_poll_release(skey); + + sum_count += poll_num; + if (unlikely(ret && ret != -WD_EAGAIN)) + goto poll_err; + + if (sum_count >= expect) + break; } - return 0; +poll_err: + *count = sum_count; + if (ret == -WD_EAGAIN) + return 0; + return ret; } static handle_t sched_none_init(handle_t h_sched_ctx, void *sched_param) @@ -396,12 +1464,11 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, } while (loop_times > 0) { - /* Default use ctx 0 */ loop_times--; ret = sched_ctx->poll_func(0, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) + if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; - else if (ret == -EAGAIN) + else if (ret == -WD_EAGAIN) continue; *count += poll_num; @@ -409,7 +1476,7 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, break; } - return 0; + return WD_SUCCESS; } static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) @@ -420,13 +1487,10 @@ static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) static __u32 sched_single_pick_next_ctx(handle_t sched_ctx, void *sched_key, const int sched_mode) { -#define CTX_ASYNC 1 -#define CTX_SYNC 0 - if (sched_mode) - return CTX_ASYNC; + return 1; else - return CTX_SYNC; + return 0; } static int sched_single_poll_policy(handle_t h_sched_ctx, @@ -443,12 +1507,11 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, } while (loop_times > 0) { - /* Default async mode use ctx 1 */ loop_times--; ret = sched_ctx->poll_func(1, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) + if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; - else if (ret == -EAGAIN) + else if (ret == -WD_EAGAIN) continue; *count += poll_num; @@ -456,260 +1519,483 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, break; } - return 0; + return WD_SUCCESS; } -static bool sched_dev_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +/** + * sched_skey_domain_fill - Initialize or append ctx to session domain + * @key_domain: Target domain (sync or async) + * @ctx_idx: Context index to add + * @policy: Scheduler policy + * @inited: Pointer to boolean tracking whether domain has been initialized + * + * First call: full domain init via wd_sched_skey_domain_init(). + * Subsequent calls: append via wd_sched_skey_add_ctx(). + */ +static void sched_skey_domain_fill(struct wd_sched_key_domain *key_domain, + __u32 ctx_idx, __u8 policy, bool *inited) { - bool found = false; - int i; + int ret; - if (key->mode >= SCHED_MODE_BUTT || key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's device id: %u, mode: %u, type: %u!\n", - key->dev_id, key->mode, key->type); - return false; + if (ctx_idx == INVALID_POS) + return; + + if (!*inited) { + ret = wd_sched_skey_domain_init(key_domain, ctx_idx, policy); + if (!ret) + *inited = true; + return; } - for (i = 0; i < sched_ctx->dev_num; i++) { - if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { - found = true; - break; + (void)wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx); +} + +/** + * sched_skey_common_init - Common scheduler init with init-once + add-rest pattern + * @h_sched_ctx: Scheduler handle + * @sched_param: Scheduling parameters + * @prop_begin: First prop type to iterate (inclusive) + * @prop_end: Last prop type to iterate (inclusive) + * + * Shared by Loop, Hungry, and Instr scheduler init functions. + * Iterates [prop_begin, prop_end], fetching sync/async ctxs per prop type. + * First valid ctx initializes the domain, subsequent ctxs are appended. + */ +static handle_t sched_skey_common_init(handle_t h_sched_ctx, void *sched_param, + __u8 prop_begin, __u8 prop_end) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + struct wd_sched_key *skey; + bool sync_inited = false; + bool async_inited = false; + __u32 sync_ctx, async_ctx; + __u32 req_ctx_num = 0; + handle_t hskey; + __u8 def_prop; + int ret, i; + + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; + } + + skey = (struct wd_sched_key *)hskey; + def_prop = skey->ctx_prop; + for (i = prop_begin; i <= prop_end; i++) { + skey->ctx_prop = i; + sync_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_ASYNC); + if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) + continue; + + if (sync_ctx != INVALID_POS) { + sched_skey_domain_fill(&skey->sync_domain, sync_ctx, + sched_ctx->policy, &sync_inited); } + if (async_ctx != INVALID_POS) + sched_skey_domain_fill(&skey->async_domain, async_ctx, + sched_ctx->policy, &async_inited); + + req_ctx_num += 2; + } + if (!req_ctx_num) { + free(skey); + return (handle_t)(-WD_EINVAL); } - if (!found) { - WD_ERR("invalid: dev_id %u is not registered!\n", key->dev_id); - return false; + skey->ctx_prop = def_prop; + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); + return (handle_t)(-WD_ENOMEM); } - return true; + return hskey; } -/* - * sched_dev_get_region - Get ctx region from ctx_map by the wd comp arg +/** + * skey_sched_init - Initialize Hungry scheduler session + */ +static handle_t skey_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + 0, UADK_ALG_TYPE_MAX - 1); + return hskey; +} + +/** + * skey_sched_pick_next_ctx - Pick context from hungry scheduler with load awareness + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) + * + * Returns: Context with minimum load, or expands if threshold exceeded + * Time complexity: O(1) for selection, O(n) if expansion needed */ -static struct sched_ctx_region *sched_dev_get_region(struct wd_sched_ctx *sched_ctx, - const struct sched_key *key) +static __u32 skey_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) { - struct wd_sched_info *sched_info; - int i, region_id; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_domain_idx_cache *idx_cache; + struct wd_sched_key_domain *domain; + __u32 min_ctx, min_load, ctx_idx; - for (i = 0; i < sched_ctx->dev_num; i++) { - if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { - region_id = sched_ctx->dev_id_map[i].region_id; - sched_info = &sched_ctx->sched_info[region_id]; - if (sched_info->ctx_region[key->mode][key->type].valid) - return &sched_info->ctx_region[key->mode][key->type]; + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; + } + + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; + + idx_cache = &domain->idx_cache; + if (sched_mode == SCHED_MODE_SYNC) { + /* Sync: send+recv atomic, spinlock serialized, use RR */ + if (!idx_cache->valid_count) + return INVALID_POS; + + ctx_idx = __atomic_fetch_add(&idx_cache->rr_ptr, 1, __ATOMIC_RELAXED) % + idx_cache->valid_count; + return idx_cache->idx_list[ctx_idx]; + } + + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) + return INVALID_POS; + + if (skey->ctxs) { + min_load = __atomic_load_n(&skey->ctxs[min_ctx].hw_load, __ATOMIC_RELAXED); + if (min_load >= HW_QUEUE_FULL_DEPTH) + return QUEUE_FULL_POS; + __atomic_fetch_add(&skey->ctxs[min_ctx].hw_load, 1, __ATOMIC_RELAXED); + wd_sched_skey_update_load(idx_cache, ctx_idx, 1); + } else { + min_load = __atomic_load_n(&idx_cache->load_values[ctx_idx], __ATOMIC_RELAXED); + if (min_load >= HW_QUEUE_FULL_DEPTH && idx_cache->load_decreased > 0) + return QUEUE_FULL_POS; + wd_sched_skey_update_load(idx_cache, ctx_idx, 1); + } + /* Check if we need to expand context pool */ + + return min_ctx; +} + +/** + * skey_sched_poll_policy - Poll policy for hungry scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) + * + * Returns: Status code + */ +static int skey_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_key *skey; + __u16 i, tpos, start_pos; + __u32 poll_num, skey_num; + int ret = -WD_EAGAIN; + __u32 sum_count = 0; + + if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { + WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); + return -WD_EINVAL; + } + + /* Randomize the initial query position. */ + skey_num = sched_ctx->skey_num; + start_pos = sched_get_poll_skey_idx(sched_ctx); + if (!skey_num || start_pos >= sched_ctx->skey_num) { + *count = 0; + return -WD_EAGAIN; + } + + /* Query the queues on each skey separately. */ + for (i = 0; i < skey_num; i++) { + tpos = (start_pos + i) % skey_num; + skey = sched_ctx->skey[tpos]; + + if (unlikely(!skey)) + continue; + + if (!sched_skey_get_ref(skey)) + continue; + + if (__atomic_test_and_set(&skey->poll_lock, __ATOMIC_ACQUIRE)) { + sched_skey_poll_release(skey); + continue; } + + ret = wd_sched_poll_skey(sched_ctx, skey, expect, &poll_num); + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + sched_skey_poll_release(skey); + /* + * This query returned 0, indicating the hardware + * likely hasn't finished processing yet. + * Implementing a delay and releasing the CPU is a predictive optimization + */ + if (!poll_num) + usleep(1); + + sum_count += poll_num; + if (ret == -WD_EAGAIN) + continue; + if (unlikely(ret) || sum_count >= expect) + break; } + *count = sum_count; - /* - * If the scheduling domain of dev_id does not exist, - * taskes operations cannot be executed using queues from other devices; - * otherwise, an SMMU error will occur. - */ - return NULL; + return ret; } -/* - * session_dev_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/** + * loop_sched_init - Initialize Loop scheduler session + */ +static handle_t loop_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + 0, UADK_ALG_TYPE_MAX - 1); + return hskey; +} + +/** + * loop_sched_pick_next_ctx - Pick context for loop scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) * - * The user must init the schedule info through wd_sched_rr_instance + * Returns: Context index with minimum load + * Time complexity: O(1) + */ +static __u32 loop_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + return round_robin_pick_next_ctx(h_sched_ctx, sched_key, sched_mode); +} + +static int loop_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + return round_robin_poll_policy(h_sched_ctx, expect, count); +} + +/** + * instr_sched_init - Initialize Instr scheduler session */ -static __u32 session_dev_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, - const int sched_mode) +static handle_t instr_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + UADK_ALG_CE_INSTR, + UADK_ALG_SVE_INSTR); + return hskey; +} + +static __u32 instr_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) { - struct sched_ctx_region *region = NULL; - bool ret; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_key_domain *domain; + __u32 min_ctx, ctx_idx; - key->mode = sched_mode; - ret = sched_dev_key_valid(sched_ctx, key); - if (!ret) + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); return INVALID_POS; + } + + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; - region = sched_dev_get_region(sched_ctx, key); - if (!region) + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) return INVALID_POS; - return sched_get_next_pos_rr(region, NULL); + return min_ctx; +} + +/** + * instr_sched_poll_policy - Poll policy for instruction scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) + * + * Returns: Status code + */ +static int instr_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + return round_robin_poll_policy(h_sched_ctx, expect, count); } static handle_t session_dev_sched_init(handle_t h_sched_ctx, void *sched_param) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; struct sched_params *param = (struct sched_params *)sched_param; - struct sched_key *skey; - unsigned int node; + struct wd_sched_key *skey; + handle_t hskey; + int ret = 0; - if (getcpu(NULL, &node)) { - WD_ERR("failed to get numa node, errno %d!\n", errno); - return (handle_t)(-errno); - } - if (node == (unsigned int)NUMA_NO_NODE) { - WD_ERR("invalid: failed to get numa node for dev sched init!\n"); + if (!param) { + WD_ERR("invalid: dev sched param is NULL!\n"); return (handle_t)(-WD_EINVAL); } - if (!sched_ctx) { - WD_ERR("invalid: sched ctx is NULL!\n"); - return (handle_t)(-WD_EINVAL); + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; } - if (!param) { - WD_DEBUG("no-sva session don't set scheduler parameters!\n"); + skey = (struct wd_sched_key *)hskey; + skey->type = param->type; + skey->dev_id = param->dev_id; + + /* DEV: pinned to a specific device, do not cross prop families */ + ret = session_sched_domain_init(sched_ctx, skey, false); + if (ret) { + WD_ERR("failed to initialize session domains!\n"); + free(skey); return (handle_t)(-WD_EINVAL); } - skey = malloc(sizeof(struct sched_key)); - if (!skey) { - WD_ERR("failed to alloc memory for session sched key!\n"); + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); return (handle_t)(-WD_ENOMEM); } - skey->type = param->type; - skey->dev_id = param->dev_id; + return (handle_t)skey; +} + +/** + * wd_sched_set_param - Set scheduler parameters + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_param: Scheduling parameters (cannot modify per API contract) + */ + +static void wd_sched_set_param(handle_t h_sched_ctx, + void *sched_key, void *sched_param) +{ + struct wd_sched_params *params = (struct wd_sched_params *)sched_param; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; - skey->sync_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); - skey->async_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); - if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { - WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); - goto out; + if (unlikely(!params || !skey)) { + WD_ERR("invalid: sched parmas or skey is NULL!\n"); + return; } - return (handle_t)skey; + skey->pkt_size = params->pkt_size; + skey->is_stream = params->data_mode; + skey->prio_mode = params->prio_mode; + + /* Store compat filtering parameters */ + skey->alg_name = params->alg_name; + skey->ctxs = params->ctxs; -out: - free(skey); - return (handle_t)(-WD_EINVAL); } static struct wd_sched sched_table[SCHED_POLICY_BUTT] = { { .name = "RR scheduler", .sched_policy = SCHED_POLICY_RR, - .sched_init = session_sched_init, - .pick_next_ctx = session_sched_pick_next_ctx, - .poll_policy = session_sched_poll_policy, + .sched_init = round_robin_sched_init, + .pick_next_ctx = round_robin_pick_next_ctx, + .poll_policy = round_robin_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "None scheduler", .sched_policy = SCHED_POLICY_NONE, .sched_init = sched_none_init, .pick_next_ctx = sched_none_pick_next_ctx, .poll_policy = sched_none_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "Single scheduler", .sched_policy = SCHED_POLICY_SINGLE, .sched_init = sched_single_init, .pick_next_ctx = sched_single_pick_next_ctx, .poll_policy = sched_single_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "Device RR scheduler", .sched_policy = SCHED_POLICY_DEV, .sched_init = session_dev_sched_init, - .pick_next_ctx = session_sched_pick_next_ctx, - .poll_policy = session_sched_poll_policy, - } + .pick_next_ctx = round_robin_pick_next_ctx, + .poll_policy = round_robin_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Loop scheduler", + .sched_policy = SCHED_POLICY_LOOP, + .sched_init = loop_sched_init, + .pick_next_ctx = loop_sched_pick_next_ctx, + .poll_policy = loop_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Hungry scheduler", + .sched_policy = SCHED_POLICY_HUNGRY, + .sched_init = skey_sched_init, + .pick_next_ctx = skey_sched_pick_next_ctx, + .poll_policy = skey_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Instr scheduler", + .sched_policy = SCHED_POLICY_INSTR, + .sched_init = instr_sched_init, + .pick_next_ctx = instr_sched_pick_next_ctx, + .poll_policy = instr_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, }; -static int wd_sched_get_nearby_numa_id(struct wd_sched_info *sched_info, int node, int numa_num) -{ - int dis = INT32_MAX; - int valid_id = -1; - int i, tmp; - - for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) { - tmp = numa_distance(node, i); - if (dis > tmp) { - valid_id = i; - dis = tmp; - } - } - } - - return valid_id; -} - -static void wd_sched_map_cpus_to_dev(struct wd_sched_ctx *sched_ctx) -{ - struct wd_sched_info *sched_info = sched_ctx->sched_info; - int i, numa_num = sched_ctx->numa_num; - int *numa_map = sched_ctx->numa_map; - - for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) - numa_map[i] = i; - else - numa_map[i] = wd_sched_get_nearby_numa_id(sched_info, i, numa_num); - } -} - -static int wd_instance_dev_region(struct wd_sched_ctx *sched_ctx, - struct sched_params *param) +static int numa_num_check(__u16 region_num) { - struct wd_sched_info *sched_info; - __u32 region_idx = INVALID_POS; - __u8 type, mode; - __u32 dev_id; - int i; - - dev_id = param->dev_id; - type = param->type; - mode = param->mode; - - /* Check whether dev_id has already been registered. */ - for (i = 0; i < sched_ctx->dev_num; i++) { - if (sched_ctx->dev_id_map[i].dev_id == dev_id) { - region_idx = sched_ctx->dev_id_map[i].region_id; - break; - } - } - - /* If not registered, allocate a new region. */ - if (region_idx == INVALID_POS) { - if (sched_ctx->dev_num >= DEVICE_REGION_MAX) { - WD_ERR("too many devices registered!\n"); - return -WD_EINVAL; - } - - region_idx = sched_ctx->dev_num; - sched_ctx->dev_id_map[region_idx].dev_id = dev_id; - sched_ctx->dev_id_map[region_idx].region_id = region_idx; - sched_ctx->dev_num++; + int max_node; - sched_info = &sched_ctx->sched_info[region_idx]; - } else { - sched_info = &sched_ctx->sched_info[region_idx]; + max_node = numa_max_node() + 1; + if (max_node <= 0) { + WD_ERR("invalid: numa max node is %d!\n", max_node); + return -WD_EINVAL; } - /* Check whether the mode and type have already been registered. */ - if (sched_info->ctx_region[mode][type].valid) { - WD_INFO("device %u mode %u type %u already registered\n", - dev_id, mode, type); - return WD_SUCCESS; + if (!region_num || region_num > max_node) { + WD_ERR("invalid: region number is %u!\n", region_num); + return -WD_EINVAL; } - /* Initialize the scheduling region for this mode and type */ - sched_info->ctx_region[mode][type].begin = param->begin; - sched_info->ctx_region[mode][type].end = param->end; - sched_info->ctx_region[mode][type].last = param->begin; - sched_info->ctx_region[mode][type].valid = true; - sched_info->valid = true; - - pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); - - return WD_SUCCESS; + return 0; } +/** + * wd_sched_rr_instance - External API for scheduling region instance + * @sched: Scheduler (cannot modify per API contract) + * @param: Scheduling parameters (cannot modify per API contract) + * + * Creates scheduling region for given parameters. + */ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *param) { - struct wd_sched_info *sched_info = NULL; struct wd_sched_ctx *sched_ctx = NULL; - __u8 type, mode; - int numa_id; + struct wd_sched_ctx_domain *domain; + int region_key; + __u8 mode; + int ret; if (!sched || !sched->h_sched_ctx || !param) { WD_ERR("invalid: sched or sched_params is NULL!\n"); @@ -721,57 +2007,68 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para return -WD_EINVAL; } - numa_id = param->numa_id; - type = param->type; mode = param->mode; sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; - if (sched_ctx->numa_num > 0 && (numa_id >= sched_ctx->numa_num || - numa_id < 0)) { - WD_ERR("invalid: sched_ctx's numa_id is %d, numa_num is %u!\n", - numa_id, sched_ctx->numa_num); - return -WD_EINVAL; + if (sched_ctx->policy == SCHED_POLICY_DEV) { + region_key = param->dev_id; + if (region_key < 0) { + WD_ERR("invalid: dev_id is %d!\n", region_key); + return -WD_EINVAL; + } + } else { + region_key = param->numa_id; + if (region_key >= sched_ctx->region_num || region_key < 0) { + WD_ERR("invalid: region_key is %d, region_num is %u!\n", + region_key, sched_ctx->region_num); + return -WD_EINVAL; + } } - if (type >= sched_ctx->type_num) { - WD_ERR("invalid: sched_ctx's type is %u, type_num is %u!\n", - type, sched_ctx->type_num); + if (param->type >= sched_ctx->type_num) { + WD_ERR("invalid: type is %u, type_num is %u!\n", + param->type, sched_ctx->type_num); return -WD_EINVAL; } if (mode >= SCHED_MODE_BUTT) { - WD_ERR("invalid: sched_ctx's mode is %u, mode_num is %d!\n", - mode, SCHED_MODE_BUTT); + WD_ERR("invalid: mode is %u, mode_num is %u!\n", + mode, sched_ctx->mode_num); return -WD_EINVAL; } - if (sched_ctx->policy == SCHED_POLICY_DEV) - return wd_instance_dev_region(sched_ctx, param); - - sched_info = &sched_ctx->sched_info[numa_id]; - if (!sched_info->ctx_region[mode]) { - WD_ERR("invalid: ctx_region is NULL, numa: %d, mode: %u!\n", - numa_id, mode); - return -WD_EINVAL; + if (param->ctx_prop < 0 || param->ctx_prop >= UADK_ALG_TYPE_MAX) { + WD_INFO("Info: ctx_prop %d exceeds max type!\n", param->ctx_prop); + param->ctx_prop = UADK_ALG_HW; } - sched_info->ctx_region[mode][type].begin = param->begin; - sched_info->ctx_region[mode][type].end = param->end; - sched_info->ctx_region[mode][type].last = param->begin; - sched_info->ctx_region[mode][type].valid = true; - sched_info->valid = true; + domain = wd_sched_hash_table_insert(sched_ctx->domain_hash_table, + region_key, mode, param->type, + param->ctx_prop); + if (!domain) + return -WD_ENOMEM; - wd_sched_map_cpus_to_dev(sched_ctx); - pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); + ret = wd_sched_domain_add_segment(domain, param->begin, param->end); + if (ret) { + WD_ERR("failed to add segment to domain!\n"); + return ret; + } + domain->valid = true; return WD_SUCCESS; } +/** + * wd_sched_rr_release - External API for scheduler release + * @sched: Scheduler to release (cannot modify per API contract) + * + * Releases all scheduler resources. + */ void wd_sched_rr_release(struct wd_sched *sched) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; - int i, j, region_num; + struct wd_sched_key *skey; + __u32 i; if (!sched) return; @@ -780,59 +2077,49 @@ void wd_sched_rr_release(struct wd_sched *sched) if (!sched_ctx) goto ctx_out; - /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ - if (sched_ctx->policy == SCHED_POLICY_DEV) - region_num = DEVICE_REGION_MAX; - else - region_num = sched_ctx->numa_num; + /* Release all session keys - iterate full array to catch residual entries */ + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + skey = sched_ctx->skey[i]; + if (!skey) + continue; - sched_info = sched_ctx->sched_info; - if (!sched_info) - goto info_out; + sched_ctx->skey[i] = NULL; + __atomic_store_n(&skey->refcount, 0, __ATOMIC_RELAXED); + session_sched_domain_destroy(skey); + free(skey); + } + sched_ctx->skey_num = 0; - for (i = 0; i < region_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - if (sched_info[i].ctx_region[j]) { - free(sched_info[i].ctx_region[j]); - sched_info[i].ctx_region[j] = NULL; - } - } + /* Release hash table */ + if (sched_ctx->domain_hash_table) { + wd_sched_hash_table_destroy(sched_ctx->domain_hash_table); + sched_ctx->domain_hash_table = NULL; } -info_out: + pthread_mutex_destroy(&sched_ctx->skey_lock); free(sched_ctx); + ctx_out: free(sched); - return; } -static int numa_num_check(__u16 numa_num) -{ - int max_node; - - max_node = numa_max_node() + 1; - if (max_node <= 0) { - WD_ERR("invalid: numa max node is %d!\n", max_node); - return -WD_EINVAL; - } - - if (!numa_num || numa_num > max_node) { - WD_ERR("invalid: numa number is %u!\n", numa_num); - return -WD_EINVAL; - } - - return 0; -} - +/** + * wd_sched_rr_alloc - External API for scheduler allocation + * @sched_type: Scheduling policy type (cannot modify per API contract) + * @type_num: Number of operation types (cannot modify per API contract) + * @region_num: Number of regions (cannot modify per API contract) + * @func: Poll function (cannot modify per API contract) + * + * Allocates and initializes scheduler with single global hash table. + */ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, - __u16 numa_num, user_poll_func func) + __u16 region_num, user_poll_func func) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; struct wd_sched *sched; - int region_num; - int i, j; + __u32 estimated_entries; + __u32 i; if (sched_type >= SCHED_POLICY_BUTT || !type_num) { WD_ERR("invalid: sched_type is %u or type_num is %u!\n", @@ -846,63 +2133,62 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, return NULL; } - if (sched_type == SCHED_POLICY_DEV) - region_num = DEVICE_REGION_MAX; - else - region_num = numa_num; - - sched_ctx = calloc(1, sizeof(struct wd_sched_ctx) + - sizeof(struct wd_sched_info) * region_num); + sched_ctx = calloc(1, sizeof(struct wd_sched_ctx)); if (!sched_ctx) { WD_ERR("failed to alloc memory for sched_ctx!\n"); goto err_out; } - /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ + /* Cache dimension parameters */ + sched_ctx->type_num = type_num; + sched_ctx->mode_num = SCHED_MODE_BUTT; + sched_ctx->region_num = region_num; + sched_ctx->policy = sched_type; + if (sched_type == SCHED_POLICY_DEV) { - sched_ctx->numa_num = 0; - sched_ctx->dev_num = 0; - for (i = 0; i < DEVICE_REGION_MAX; i++) { - sched_ctx->dev_id_map[i].dev_id = INVALID_POS; - sched_ctx->dev_id_map[i].region_id = INVALID_POS; - } + /* Device mode: region_num is actually device count */ + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } else { - sched_ctx->numa_num = numa_num; - sched_ctx->dev_num = 0; - if (numa_num_check(sched_ctx->numa_num)) + /* NUMA mode: validate region_num */ + if (numa_num_check(region_num)) goto err_out; + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } - sched->h_sched_ctx = (handle_t)sched_ctx; - if (sched_type == SCHED_POLICY_NONE || - sched_type == SCHED_POLICY_SINGLE) - goto simple_ok; - - sched_info = sched_ctx->sched_info; - for (i = 0; i < region_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - sched_info[i].ctx_region[j] = - calloc(1, sizeof(struct sched_ctx_region) * type_num); - if (!sched_info[i].ctx_region[j]) - goto err_out; - } + /* Create single global hash table */ + sched_ctx->domain_hash_table = wd_sched_hash_table_create(estimated_entries); + if (!sched_ctx->domain_hash_table) { + WD_ERR("failed to create hash table!\n"); + goto ctx_out; } -simple_ok: sched_ctx->poll_func = func; - sched_ctx->policy = sched_type; - sched_ctx->type_num = type_num; - memset(sched_ctx->numa_map, -1, sizeof(int) * NUMA_NUM_NODES); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) + sched_ctx->skey[i] = NULL; + + if (pthread_mutex_init(&sched_ctx->skey_lock, NULL)) { + WD_ERR("failed to init skey_lock!\n"); + goto err_destroy_hash; + } + sched_ctx->skey_num = 0; + + sched->h_sched_ctx = (handle_t)sched_ctx; sched->sched_init = sched_table[sched_type].sched_init; + sched->sched_uninit = sched_table[sched_type].sched_uninit; sched->pick_next_ctx = sched_table[sched_type].pick_next_ctx; sched->poll_policy = sched_table[sched_type].poll_policy; sched->sched_policy = sched_type; sched->name = sched_table[sched_type].name; + sched->set_param = sched_table[sched_type].set_param; return sched; +err_destroy_hash: + wd_sched_hash_table_destroy(sched_ctx->domain_hash_table); +ctx_out: + free(sched_ctx); err_out: - wd_sched_rr_release(sched); + free(sched); return NULL; } -- 2.43.0

1 15