August 2026 - Acc - mailweb.openeuler.org

[PATCH] uadk_tool: fix segfault in digest/aead init1 from uninitialized wd_sched
by Chenghai Huang 29 Aug '26

29 Aug '26

From: Weili Qian <qianweili(a)huawei.com> digest_init1() and aead_init1() in uadk_tool/test/test_sec.c declared "struct wd_sched sched;" on the stack without initialization, leaving sched_policy and callback fields as garbage. This bypassed the SCHED_POLICY_NONE/SINGLE rejection in wd_digest_init/wd_aead_init and led to a segfault on the first wd_do_digest_sync/async call. Zero-initialize the struct so sched_policy defaults to SCHED_POLICY_RR and all callbacks/handles are NULL. Signed-off-by: Weili Qian <qianweili(a)huawei.com> --- uadk_tool/test/test_sec.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uadk_tool/test/test_sec.c b/uadk_tool/test/test_sec.c index 462ba7a..6e1b990 100644 --- a/uadk_tool/test/test_sec.c +++ b/uadk_tool/test/test_sec.c @@ -1416,7 +1416,7 @@ static __u32 sched_digest_pick_next_ctx(handle_t h_sched_ctx, static int digest_init1(int type, int mode) { struct uacce_dev_list *list; - struct wd_sched sched; + struct wd_sched sched = {0}; int ret; if (g_use_env) @@ -2685,7 +2685,7 @@ static __u32 sched_aead_pick_next_ctx(handle_t h_sched_ctx, static int aead_init1(int type, int mode) { struct uacce_dev_list *list; - struct wd_sched sched; + struct wd_sched sched = {0}; int ret; if (g_use_env) -- 2.33.0

1 0

[PATCH 01/16] uadk: rewrite scheduler with scheduling domain solution
by Chenghai Huang 25 Aug '26

25 Aug '26

From: Longfang Liu <liulongfang(a)huawei.com> Conduct a comprehensive update of the scheduler's internal triple-array-based pooling scheme. Adopt new data organization methods such as hash buckets and segment linked lists to achieve efficient insertion and query operations. Implement pre-fetch queue processing for business thread sessions, creating thread-level sub-queue pools for fast packet reception queries. Provide new scheduling algorithms for the new UADK framework. Signed-off-by: Longfang Liu <liulongfang(a)huawei.com> Signed-off-by: Chenghai Huang <huangchenghai2(a)huawei.com> Signed-off-by: Wenkai Lin <linwenkai6(a)hisilicon.com> --- include/wd_alg.h | 11 +- include/wd_alg_common.h | 5 +- include/wd_internal.h | 18 +- include/wd_sched.h | 10 +- wd_sched.c | 2304 ++++++++++++++++++++++++++++++--------- 5 files changed, 1832 insertions(+), 516 deletions(-) diff --git a/include/wd_alg.h b/include/wd_alg.h index 18503ca..1ae1dae 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -64,10 +64,13 @@ extern "C" { #endif enum alg_dev_type { - UADK_ALG_SOFT = 0x0, - UADK_ALG_CE_INSTR = 0x1, - UADK_ALG_SVE_INSTR = 0x2, - UADK_ALG_HW = 0x3 + UADK_ALG_HW = 0x0, + UADK_ALG_CE_INSTR = 0x1, + UADK_ALG_SVE_INSTR = 0x2, + UADK_ALG_SOFT = 0x3, + UADK_ALG_NPU = 0x4, + UADK_ALG_GPU = 0x5, + UADK_ALG_TYPE_MAX, }; /* diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index a294877..ebba7f8 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -41,6 +41,7 @@ extern "C" { /* Key size of digest */ #define MAX_HMAC_KEY_SIZE 128U +#define STATUS_ENABLE (void *)0x1 enum alg_task_type { TASK_MIX = 0x0, @@ -140,7 +141,7 @@ struct wd_ctx_params { }; /* - * struct wd_comp_sched - Define a scheduler. + * struct wd_sched - Define a scheduler. * @name: Name of this scheduler. * @sched_policy: Method for scheduler to perform scheduling * @sched_init: inited the scheduler input parameters. @@ -157,10 +158,12 @@ struct wd_sched { const char *name; int sched_policy; handle_t (*sched_init)(handle_t h_sched_ctx, void *sched_param); + void (*sched_uninit)(handle_t h_sched_ctx, handle_t h_sched_key); __u32 (*pick_next_ctx)(handle_t h_sched_ctx, void *sched_key, const int sched_mode); int (*poll_policy)(handle_t h_sched_ctx, __u32 expect, __u32 *count); + void (*set_param)(handle_t h_sched_ctx, void *sched_key, void *sched_param); handle_t h_sched_ctx; }; diff --git a/include/wd_internal.h b/include/wd_internal.h index 4f19d3a..62cf1a9 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -7,8 +7,10 @@ #define WD_INTERNAL_H #include <pthread.h> +#include <stdatomic.h> #include <stdbool.h> #include "wd.h" +#include "wd_alg.h" #ifdef __cplusplus extern "C" { @@ -41,11 +43,14 @@ struct wd_ce_ctx { }; struct wd_ctx_internal { - handle_t ctx; __u8 op_type; __u8 ctx_mode; + __u8 ctx_type; + handle_t ctx; __u16 sqn; pthread_spinlock_t lock; + struct wd_alg_driver *drv; + __u32 hw_load; }; struct wd_ctx_config_internal { @@ -64,6 +69,17 @@ struct wd_datalist { struct wd_datalist *next; }; +struct wd_sched_params { + __u32 pkt_size; + /* block mode or stream mode */ + __u16 data_mode; + __u16 prio_mode; + + /* Compat filtering parameters for session-ctx matching */ + const char *alg_name; + struct wd_ctx_internal *ctxs; +}; + int memcmp_consttime(const void *s1, const void *s2, size_t n); #ifdef __cplusplus diff --git a/include/wd_sched.h b/include/wd_sched.h index 5baecd3..f81025f 100644 --- a/include/wd_sched.h +++ b/include/wd_sched.h @@ -13,7 +13,8 @@ extern "C" { #endif -#define INVALID_POS 0xFFFFFFFF +#define INVALID_POS 0xFFFF +#define QUEUE_FULL_POS 0x1FFF /* The global policy type */ enum sched_policy_type { @@ -25,6 +26,12 @@ enum sched_policy_type { SCHED_POLICY_SINGLE, /* requests will be sent to ctxs and dev_id */ SCHED_POLICY_DEV, + /* Hard calculation and soft calculation interval loop call */ + SCHED_POLICY_LOOP, + /* Perform heterogeneous calculations through ctx of session key */ + SCHED_POLICY_HUNGRY, + /* Instructions to accelerate heterogeneous computing */ + SCHED_POLICY_INSTR, SCHED_POLICY_BUTT, }; @@ -35,6 +42,7 @@ struct sched_params { __u32 begin; __u32 end; __u32 dev_id; + int ctx_prop; }; typedef int (*user_poll_func)(__u32 pos, __u32 expect, __u32 *count); diff --git a/wd_sched.c b/wd_sched.c index 19936fd..2efa2aa 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -1,374 +1,1442 @@ // SPDX-License-Identifier: Apache-2.0 /* - * Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved. * Copyright 2020-2021 Linaro ltd. + * + * Scheduler: Simplified Pure Hash Table with Dynamic Context Expansion + * + * Key improvements: + * - Single global hash table with (region_id, mode, op_type, prop) dimensions + * - Segment list for non-contiguous ctx ranges + * - Dual-domain queues for session key. + * - Dynamic ctx expansion in HUNGRY mode based on load threshold + * - Packet reception is handled through the active queues in the session key. + * - Simplified sched_init: only allocate one sync + one async ctx + * - Removed redundant wd_sched_info layer */ #define _GNU_SOURCE #include <stdlib.h> #include <stdbool.h> +#include <string.h> #include <sched.h> #include <numa.h> +#include <limits.h> +#include <pthread.h> #include "wd_sched.h" +#include "wd_alg.h" +#include "wd_internal.h" -#define MAX_POLL_TIMES 1000 +#define MAX_POLL_TIMES 1000 +#define HUNGRY_LOAD_THRESHOLD 256 +#define SKEY_CTX_MAX_NUM 16 +#define SKEY_MAX_THREAD_NUM 64 +#define SKEY_LOAD_UPDATE_INTERVAL 1 +#define HW_QUEUE_FULL_DEPTH 1024 +#define MAX_NUMA_NODES (NUMA_NUM_NODES >> 5) + +/* ============================================================================ + * Hash Table Configuration + * ============================================================================ + */ +#define WD_SCHED_MAX_BUCKETS 512 +#define WD_SCHED_MIN_BUCKETS 32 +#define WD_SCHED_LOAD_FACTOR 0.75f +#define HASH_PRIME1 73 +#define HASH_PRIME2 13 +#define HASH_PRIME3 7 +#define HASH_PRIME4 11 + +/* ============================================================================ + * Scheduling Region Mode + * ============================================================================ + */ enum sched_region_mode { SCHED_MODE_SYNC = 0, SCHED_MODE_ASYNC = 1, SCHED_MODE_BUTT }; -/* - * sched_key - The key if schedule region. - * @numa_id: The schedule numa region id. - * @mode: Sync mode:0, async_mode:1 - * @type: Service type , the value must smaller than type_num. - * @sync_ctxid: alloc ctx id for sync mode - * @async_ctxid: alloc ctx id for async mode - */ -struct sched_key { - int numa_id; - __u8 type; - __u8 mode; - __u32 sync_ctxid; - __u32 async_ctxid; - __u32 dev_id; -}; +/* ============================================================================ + * Segment List for Domain Index Organization + * ============================================================================ + */ -/* - * struct sched_ctx_range - define one ctx pos. - * @begin: the start pos in ctxs of config. - * @end: the end pos in ctxx of config. - * @last: the last one which be distributed. - * @valid: the region used flag. - * @lock: lock the currentscheduling region. - */ -struct sched_ctx_region { +/** + * wd_sched_ctx_segment - Contiguous segment of ctx indices in domain + * @begin: Start index of this segment + * @end: End index of this segment (inclusive) + * @next: Pointer to next segment in the linked list + * + * Supports non-contiguous ctx ranges via segment list. + */ +struct wd_sched_ctx_segment { __u32 begin; __u32 end; - __u32 last; + struct wd_sched_ctx_segment *next; +}; + +/* ============================================================================ + * Session key domain cache processing. + * ============================================================================ + */ + +/** + * wd_sched_domain_idx_cache - Simplified fixed array cache for skey domains + * + * Design principles: + * - Fixed array for cache-friendly memory layout + * - Atomic operations for lock-free load tracking + * - Simple RR and load balancing strategies + * - Maximum 16 queues per thread (typical usage) + */ +struct wd_sched_domain_idx_cache { + /* Queue index array */ + __u32 idx_list[SKEY_CTX_MAX_NUM]; /* Array of ctx indices */ + __u32 load_values[SKEY_CTX_MAX_NUM]; /* Atomic load counters */ + __u32 valid_count; /* Number of valid queues */ + + /* Scheduling state */ + __u32 rr_ptr; /* Round-robin pointer */ + __u32 min_load_idx; /* Cached min load index */ + __u32 op_counter; /* Operation counter for updates */ + __u8 load_decreased; /* Non-zero if poll ever decremented load */ + + /* Configuration */ + __u32 update_interval; /* Min load update interval */ + __u8 policy; /* Scheduling policy */ + + /* Synchronization */ + pthread_mutex_t cache_lock; /* Lock for structure modifications */ +}; + +/** + * wd_sched_ctx_domain - Scheduling domain with four dimensions + * @region_id: Region identifier (numa_id or device_id) + * @mode: Context mode (SYNC/ASYNC) + * @op_type: Operation type + * @prop: Property (e.g., device type: HW, CE, SOFT) + * @segments: Linked list of context ranges + * @segment_count: Number of segments + * @total_ctx_count: Total contexts across all segments + * @current_segment: Current segment pointer for round-robin + * @current_pos: Current position within segment + * @valid: Domain validity flag + * @lock: Synchronization spinlock + */ +struct wd_sched_ctx_domain { + int region_id; + __u8 mode; + __u32 op_type; + __u8 prop; + + struct wd_sched_ctx_segment *segments; + __u32 segment_count; + __u32 total_ctx_count; + + struct wd_sched_ctx_segment *current_segment; + __u32 current_pos; bool valid; + pthread_mutex_t lock; }; -/* - * wd_sched_info - define the context of the scheduler. - * @ctx_region: define the map for the comp ctxs, using for quickly search. - * the x range: two(sync and async), the y range: - * two(e.g. comp and uncomp) the map[x][y]'s value is the ctx - * begin and end pos. - * @valid: the region used flag. - */ -struct wd_sched_info { - struct sched_ctx_region *ctx_region[SCHED_MODE_BUTT]; - bool valid; +/** + * wd_sched_domain_hash_node - Hash table collision chain node + */ +struct wd_sched_domain_hash_node { + struct wd_sched_ctx_domain domain; + struct wd_sched_domain_hash_node *next; }; -struct dev_region_map { +/** + * wd_sched_domain_hash_table - Pure dynamic hash table for scheduling domains + * @buckets: Hash table bucket array + * @bucket_size: Number of buckets + * @lock: Read-write lock for concurrent access + */ +struct wd_sched_domain_hash_table { + struct wd_sched_domain_hash_node **buckets; + __u32 bucket_size; + pthread_mutex_t lock; +}; + +/* ============================================================================ + * Dual-Domain Structure for Session Key + * ============================================================================ + */ + +/** + * wd_sched_key_domain - Session domain with min-heap + * @idx_cache: Index cache with min-heap for load-based selection + * @lock: Synchronization spinlock + * @expanded_count: Track how many times ctx has been expanded + */ +struct wd_sched_key_domain { + struct wd_sched_domain_idx_cache idx_cache; + pthread_mutex_t lock; + __u32 expanded_count; +}; + +/** + * wd_sched_key - Session-level scheduling key + * @region_id: Region identifier + * @type: Operation type + * @mode: Current mode (SYNC/ASYNC) + * @dev_id: Device identifier (for SCHED_POLICY_DEV) + * @ctx_prop: Context property + * @is_stream: Stream mode flag + * @prio_mode: Priority mode + * @pkt_size: Current packet size + * @sync_domain: Min-heap domain for sync contexts + * @async_domain: Min-heap domain for async contexts + * @lock: Synchronization spinlock + */ +struct wd_sched_key { + int region_id; + __u8 type; + __u8 mode; __u32 dev_id; - __u32 region_id; + __u8 ctx_prop; + __u16 is_stream; + __u16 prio_mode; + __u32 pkt_size; + + struct wd_sched_key_domain sync_domain; + struct wd_sched_key_domain async_domain; + + pthread_mutex_t lock; + __u32 poll_lock; + __u32 refcount; + + /* Compat filtering parameters for session-ctx matching */ + const char *alg_name; + struct wd_ctx_internal *ctxs; }; -/* - * wd_sched_ctx - define the context of the scheduler. - * @policy: define the policy of the scheduler. - * @numa_num: the max numa numbers of the scheduler. - * @type_num: the max operation types of the scheduler. - * @poll_func: the task's poll operation function. - * @numa_map: a map of cpus to devices. - * @sched_info: the context of the scheduler. + +/** + * wd_sched_ctx - Main scheduler context + * @policy: Scheduling policy type + * @type_num: Number of operation types + * @mode_num: Number of modes (SYNC/ASYNC) + * @region_num: Number of regions (numa or devices) + * @poll_func: Poll function for receiving responses + * @domain_hash_table: Global hash table for all domains + * @skey_num: Number of active session keys + * @skey_lock: Lock for skey array + * @skey: Array of session keys */ struct wd_sched_ctx { __u32 policy; __u32 type_num; - __u16 numa_num; - __u16 dev_num; + __u32 mode_num; + __u16 region_num; + user_poll_func poll_func; - int numa_map[NUMA_NUM_NODES]; - struct dev_region_map dev_id_map[DEVICE_REGION_MAX]; - struct wd_sched_info sched_info[0]; + struct wd_sched_domain_hash_table *domain_hash_table; + + __u32 skey_num; + pthread_mutex_t skey_lock; + struct wd_sched_key *skey[SKEY_MAX_THREAD_NUM]; }; -static bool sched_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +/* ============================================================================ + * Hash Table Core Operations + * ============================================================================ + */ + +static bool wd_sched_is_prime(__u32 n) { - if (key->numa_id >= sched_ctx->numa_num || key->mode >= SCHED_MODE_BUTT || - key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's numa: %d, mode: %u, type: %u!\n", - key->numa_id, key->mode, key->type); + __u32 i; + + if (n <= 1) return false; + if (n <= 3) + return true; + if (n % 2 == 0 || n % 3 == 0) + return false; + + for (i = 5; i * i <= n; i += 6) { + if (n % i == 0 || n % (i + 2) == 0) + return false; } return true; } -/* - * sched_get_ctx_range - Get ctx range from ctx_map by the wd comp arg +static __u32 wd_sched_find_prime(__u32 n) +{ + while (!wd_sched_is_prime(n)) + n++; + return n; +} + +static __u32 wd_sched_compute_bucket_size(__u32 estimated_entries) +{ + __u32 target_size; + + target_size = (estimated_entries * 4) / 3; + + if (target_size < WD_SCHED_MIN_BUCKETS) + target_size = WD_SCHED_MIN_BUCKETS; + if (target_size > WD_SCHED_MAX_BUCKETS) + target_size = WD_SCHED_MAX_BUCKETS; + + return wd_sched_find_prime(target_size); +} + +/** + * wd_sched_hash_compute - Compute hash value for four-dimensional domain key + * @region_id: Region identifier + * @mode: Context mode + * @op_type: Operation type + * @prop: Property + * @bucket_size: Hash table bucket count + * + * Combines four dimensions using prime number multipliers. */ -static struct sched_ctx_region *sched_get_ctx_range(struct wd_sched_ctx *sched_ctx, - const struct sched_key *key) +static inline __u32 wd_sched_hash_compute(int region_id, __u8 mode, + __u32 op_type, __u8 prop, __u32 bucket_size) { - struct wd_sched_info *sched_info; - int numa_id; + __u32 hash; - sched_info = sched_ctx->sched_info; - if (key->numa_id >= 0 && - sched_info[key->numa_id].ctx_region[key->mode][key->type].valid) - return &sched_info[key->numa_id].ctx_region[key->mode][key->type]; + hash = (region_id * HASH_PRIME1) + (mode * HASH_PRIME2) + + (op_type * HASH_PRIME3) + (prop * HASH_PRIME4); + return hash % bucket_size; +} + +static inline bool wd_sched_domain_key_match( + int region_id1, __u8 mode1, __u32 op_type1, __u8 prop1, + int region_id2, __u8 mode2, __u32 op_type2, __u8 prop2) +{ + return (region_id1 == region_id2 && mode1 == mode2 && + op_type1 == op_type2 && prop1 == prop2); +} + +/** + * wd_sched_hash_table_create - Create hash table + * @estimated_entries: Estimated number of entries + * + * Returns: Initialized hash table or NULL on error + */ +static struct wd_sched_domain_hash_table * +wd_sched_hash_table_create(__u32 estimated_entries) +{ + struct wd_sched_domain_hash_table *table; + __u32 bucket_size; + int ret; - /* If the key->numa_id is not exist, we should scan for a region */ - for (numa_id = 0; numa_id < sched_ctx->numa_num; numa_id++) { - if (sched_info[numa_id].ctx_region[key->mode][key->type].valid) - return &sched_info[numa_id].ctx_region[key->mode][key->type]; + table = calloc(1, sizeof(*table)); + if (!table) + return NULL; + + bucket_size = wd_sched_compute_bucket_size(estimated_entries); + + table->buckets = calloc(bucket_size, sizeof(*table->buckets)); + if (!table->buckets) { + free(table); + return NULL; } - return NULL; + table->bucket_size = bucket_size; + ret = pthread_mutex_init(&table->lock, NULL); + if (ret) { + free(table->buckets); + free(table); + return NULL; + } + + return table; } -/* - * sched_get_next_pos_rr - Get next resource pos by RR schedule. - * The second para is reserved for future. +static void wd_sched_hash_table_destroy(struct wd_sched_domain_hash_table *table) +{ + struct wd_sched_domain_hash_node *node, *next; + struct wd_sched_ctx_segment *seg, *next_seg; + __u32 i; + + if (!table) + return; + + for (i = 0; i < table->bucket_size; i++) { + node = table->buckets[i]; + while (node) { + next = node->next; + + /* Release segment linked list */ + seg = node->domain.segments; + while (seg) { + next_seg = seg->next; + free(seg); + seg = next_seg; + } + + pthread_mutex_destroy(&node->domain.lock); + free(node); + node = next; + } + } + + pthread_mutex_destroy(&table->lock); + free(table->buckets); + free(table); +} + +static struct wd_sched_ctx_domain * +wd_sched_hash_table_lookup(struct wd_sched_domain_hash_table *table, + int region_id, __u8 mode, __u32 op_type, __u8 prop) +{ + struct wd_sched_domain_hash_node *node; + struct wd_sched_ctx_domain *domain = NULL; + __u32 hash_idx; + + if (!table) + return NULL; + + pthread_mutex_lock(&table->lock); + hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size); + node = table->buckets[hash_idx]; + while (node) { + if (wd_sched_domain_key_match( + node->domain.region_id, node->domain.mode, node->domain.op_type, + node->domain.prop, region_id, mode, op_type, prop)) { + domain = &node->domain; + break; + } + node = node->next; + } + pthread_mutex_unlock(&table->lock); + + return domain; +} + +static struct wd_sched_ctx_domain * +wd_sched_hash_table_insert(struct wd_sched_domain_hash_table *table, + int region_id, __u8 mode, __u32 op_type, __u8 prop) +{ + struct wd_sched_domain_hash_node *new_node; + struct wd_sched_ctx_domain *existing; + __u32 hash_idx; + int ret; + + if (!table) + return NULL; + + existing = wd_sched_hash_table_lookup(table, region_id, mode, op_type, prop); + if (existing) + return existing; + + pthread_mutex_lock(&table->lock); + hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size); + /* Alloc and initialize new domain */ + new_node = calloc(1, sizeof(*new_node)); + if (!new_node) { + pthread_mutex_unlock(&table->lock); + return NULL; + } + + /* Initialize new domain */ + new_node->domain.region_id = region_id; + new_node->domain.mode = mode; + new_node->domain.op_type = op_type; + new_node->domain.prop = prop; + new_node->domain.segments = NULL; + new_node->domain.segment_count = 0; + new_node->domain.total_ctx_count = 0; + new_node->domain.current_segment = NULL; + new_node->domain.current_pos = 0; + new_node->domain.valid = false; + + ret = pthread_mutex_init(&new_node->domain.lock, NULL); + if (ret) { + pthread_mutex_unlock(&table->lock); + free(new_node); + return NULL; + } + + new_node->next = table->buckets[hash_idx]; + table->buckets[hash_idx] = new_node; + pthread_mutex_unlock(&table->lock); + + return &new_node->domain; +} + +/* ============================================================================ + * Segment List Operations + * ============================================================================ + */ + +/** + * wd_sched_domain_add_segment - Add context range segment to domain + * @domain: Target domain + * @begin: Start context index + * @end: End context index (inclusive) + * + * Supports non-contiguous context ranges via segment list. */ -static __u32 sched_get_next_pos_rr(struct sched_ctx_region *region, void *para) +static int wd_sched_domain_add_segment(struct wd_sched_ctx_domain *domain, + __u32 begin, __u32 end) { + struct wd_sched_ctx_segment *seg, *new_seg; + + if (!domain || begin > end) + return -WD_EINVAL; + + new_seg = calloc(1, sizeof(*new_seg)); + if (!new_seg) + return -WD_ENOMEM; + + new_seg->begin = begin; + new_seg->end = end; + new_seg->next = NULL; + + pthread_mutex_lock(&domain->lock); + + /* Append to segment list tail */ + if (!domain->segments) { + domain->segments = new_seg; + } else { + seg = domain->segments; + while (seg->next) + seg = seg->next; + seg->next = new_seg; + } + + domain->segment_count++; + domain->total_ctx_count += (end - begin + 1); + + /* Initialize polling state */ + if (!domain->current_segment) + domain->current_segment = domain->segments; + + pthread_mutex_unlock(&domain->lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_domain_get_next_rr - Get next context via round-robin from domain + * @domain: Source domain + * + * Returns: Next global queue index in round-robin order + * Time complexity: O(1) + */ +static __u32 wd_sched_domain_get_next_rr(struct wd_sched_ctx_domain *domain) +{ + __u32 ctx_idx; __u32 pos; - pthread_mutex_lock(&region->lock); + if (!domain || !domain->segments || !domain->total_ctx_count) + return INVALID_POS; - pos = region->last; + pthread_mutex_lock(&domain->lock); + if (!domain->current_segment) + domain->current_segment = domain->segments; - if (pos < region->end) - region->last++; - else - region->last = region->begin; + pos = domain->current_pos; + + /* Calculate global queue number: segment.begin + relative position */ + ctx_idx = domain->current_segment->begin + pos; - pthread_mutex_unlock(&region->lock); + /* Move to next position */ + if (pos + 1 < domain->current_segment->end - domain->current_segment->begin + 1) { + /* Within same segment */ + domain->current_pos = pos + 1; + } else if (domain->current_segment->next) { + /* Move to next segment */ + domain->current_segment = domain->current_segment->next; + domain->current_pos = 0; + } else { + /* Loop back to beginning */ + domain->current_segment = domain->segments; + domain->current_pos = 0; + } + + pthread_mutex_unlock(&domain->lock); - return pos; + return ctx_idx; } -/* - * session_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/* ============================================================================ + * SKey Domain Cache Management Functions + * ============================================================================ + */ +/** + * wd_sched_skey_cache_init - Initialize skey domain cache + * @cache: Pointer to cache structure + * @policy: Scheduling policy * @sched_type: Scheduling policy type (cannot modify per + * API contract) + * + * Initialize fixed array cache with invalid positions and zero loads. + */ +static int wd_sched_skey_cache_init(struct wd_sched_domain_idx_cache *cache, + __u8 policy) +{ + int i; + + if (!cache) { + WD_ERR("invalid: cache pointer is NULL!\n"); + return -WD_EINVAL; + } + + /* Initialize array with invalid positions */ + for (i = 0; i < SKEY_CTX_MAX_NUM; i++) { + cache->idx_list[i] = INVALID_POS; + __atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED); + } + + /* Initialize atomic counters */ + __atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->op_counter, 0, __ATOMIC_RELAXED); + cache->load_decreased = 0; + + /* Set configuration */ + cache->valid_count = 0; + cache->update_interval = SKEY_LOAD_UPDATE_INTERVAL; + cache->policy = policy; + + /* Initialize structure lock */ + if (pthread_mutex_init(&cache->cache_lock, NULL)) { + WD_ERR("failed to init cache lock!\n"); + return -WD_EINVAL; + } + + return WD_SUCCESS; +} +/** + * wd_sched_skey_cache_uninit - Cleanup skey domain cache + * @cache: Pointer to cache structure + * + * Release resources and reset cache state. + */ +static void wd_sched_skey_cache_uninit(struct wd_sched_domain_idx_cache *cache) +{ + if (!cache) + return; + + pthread_mutex_destroy(&cache->cache_lock); + + /* Reset cache state */ + for (int i = 0; i < SKEY_CTX_MAX_NUM; i++) { + cache->idx_list[i] = INVALID_POS; + __atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED); + } + + cache->valid_count = 0; + cache->load_decreased = 0; +} +/** + * wd_sched_skey_add_ctx - Add ctx to skey domain cache + * @cache: Pointer to cache structure + * @ctx_id: Context ID to add + * + * Add ctx to next available position in fixed array. + * Returns 0 on success, negative error code on failure. + */ +static int wd_sched_skey_add_ctx(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_id) +{ + __u32 i; + + if (!cache || ctx_id == INVALID_POS) { + WD_ERR("invalid: parameters are NULL!\n"); + return -WD_EINVAL; + } + + pthread_mutex_lock(&cache->cache_lock); + /* Check if cache is full */ + if (cache->valid_count >= SKEY_CTX_MAX_NUM) { + pthread_mutex_unlock(&cache->cache_lock); + WD_ERR("invalid: skey cache full, cannot add more queues!\n"); + return -WD_EINVAL; + } + + /* Check for duplicate ctx_id */ + for (i = 0; i < cache->valid_count; i++) { + if (cache->idx_list[i] == ctx_id) { + WD_ERR("invalid: context %u already exists in skey cache at pos %u!\n", + ctx_id, i); + pthread_mutex_unlock(&cache->cache_lock); + return -WD_EEXIST; + } + } + + /* Update min load index if as the new ctx */ + __atomic_store_n(&cache->min_load_idx, cache->valid_count, __ATOMIC_RELAXED); + + /* Add to next available position */ + cache->idx_list[cache->valid_count] = ctx_id; + __atomic_store_n(&cache->load_values[cache->valid_count], 0, __ATOMIC_RELAXED); + cache->valid_count++; + pthread_mutex_unlock(&cache->cache_lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_skey_remove_ctx - Remove ctx from skey domain cache + * @cache: Pointer to cache structure + * @ctx_id: Context ID to remove + * + * Remove ctx by shifting array elements to maintain continuity. + * Returns 0 on success, negative error code if not found. + */ +static int wd_sched_skey_remove_ctx(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_id) +{ + __u32 i, current_min; + int found = 0; + + if (!cache) { + WD_ERR("invalid: cache pointer is NULL!\n"); + return -WD_EINVAL; + } + + pthread_mutex_lock(&cache->cache_lock); + /* Find and remove the ctx */ + for (i = 0; i < cache->valid_count; i++) { + if (cache->idx_list[i] == ctx_id) { + found = 1; + break; + } + } + + if (!found) { + WD_ERR("invalid: context %u not found in skey cache!\n", ctx_id); + pthread_mutex_unlock(&cache->cache_lock); + return -WD_ENODEV; + } + + /* Shift remaining elements to fill the gap */ + for (; i < cache->valid_count - 1; i++) { + cache->idx_list[i] = cache->idx_list[i + 1]; + __atomic_store_n(&cache->load_values[i], + __atomic_load_n(&cache->load_values[i + 1], __ATOMIC_RELAXED), + __ATOMIC_RELAXED); + } + + /* Clear last position */ + cache->idx_list[cache->valid_count - 1] = INVALID_POS; + __atomic_store_n(&cache->load_values[cache->valid_count - 1], 0, __ATOMIC_RELAXED); + cache->valid_count--; + + /* Reset pointers if cache becomes empty */ + if (!cache->valid_count) { + __atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + } else { + /* Adjust min load index if necessary */ + current_min = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED); + if (current_min >= cache->valid_count) + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + } + pthread_mutex_unlock(&cache->cache_lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_update_min_load - Update cached min load index + * @cache: Pointer to cache structure + * @ctxs: Context array for hw_load lookup, or NULL to use cache->load_values * - * The user must init the schedule info through wd_sched_rr_instance + * Scan valid queues to find the one with minimum load. + * When ctxs is provided, reads per-ctx hw_load for cross-session accuracy. */ -static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, - const int sched_mode) +static void wd_sched_update_min_load(struct wd_sched_domain_idx_cache *cache, + struct wd_ctx_internal *ctxs) { - struct sched_ctx_region *region = NULL; - bool ret; + __u32 min_load = UINT_MAX; + __u32 min_idx = 0; + __u32 i, load; - key->mode = sched_mode; - ret = sched_key_valid(sched_ctx, key); - if (!ret) + if (!cache->valid_count) + return; + + for (i = 0; i < cache->valid_count; i++) { + if (ctxs) + load = __atomic_load_n(&ctxs[cache->idx_list[i]].hw_load, + __ATOMIC_RELAXED); + else + load = __atomic_load_n(&cache->load_values[i], __ATOMIC_RELAXED); + if (load < min_load) { + min_load = load; + min_idx = i; + } + } + + __atomic_store_n(&cache->min_load_idx, min_idx, __ATOMIC_RELAXED); +} + +/** + * wd_sched_skey_pick_next - Pick next ctx from skey domain cache + * @cache: Pointer to cache structure + * @ctx_idx: Output index within cache array + * @ctxs: Context array for hw_load lookup, or NULL + * + * Select next ctx based on scheduling policy: + * - RR: Simple round-robin selection + * - HUNGRY: Choose ctx with minimum load (hw_load if ctxs available) + * + * Returns selected ctx index, or INVALID_POS if no valid ctx. + */ +static __u32 wd_sched_skey_pick_next(struct wd_sched_domain_idx_cache *cache, + __u32 *ctx_idx, struct wd_ctx_internal *ctxs) +{ + __u32 selected_idx; + __u32 op_count; + + if (!cache || !cache->valid_count) return INVALID_POS; - region = sched_get_ctx_range(sched_ctx, key); - if (!region) + switch (cache->policy) { + case SCHED_POLICY_RR: + case SCHED_POLICY_NONE: + case SCHED_POLICY_SINGLE: + case SCHED_POLICY_DEV: + case SCHED_POLICY_LOOP: + case SCHED_POLICY_INSTR: + /* Round-robin: atomic increment and module */ + selected_idx = __atomic_fetch_add(&cache->rr_ptr, 1, __ATOMIC_RELAXED) % + cache->valid_count; + break; + case SCHED_POLICY_HUNGRY: + /* Update min load periodically */ + op_count = __atomic_fetch_add(&cache->op_counter, 1, __ATOMIC_RELAXED); + if (op_count % cache->update_interval == 0) + wd_sched_update_min_load(cache, ctxs); + + /* Load balancing: use cached min load index */ + selected_idx = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED); + break; + default: + WD_ERR("invalid: unknown scheduling policy %d!\n", cache->policy); + selected_idx = INVALID_POS; + break; + } + + /* Ensure index is within valid range */ + if (selected_idx >= cache->valid_count) return INVALID_POS; - return sched_get_next_pos_rr(region, NULL); + *ctx_idx = selected_idx; + return cache->idx_list[selected_idx]; } -static handle_t session_sched_init(handle_t h_sched_ctx, void *sched_param) +/** + * wd_sched_skey_update_load - Update load for a specific ctx + * @cache: Pointer to cache structure + * @ctx_idx: Context index in list + * @delta: Load delta (positive for send, negative for receive) + * + * Atomically update load counter for the specified ctx. + * Returns 0 on success, negative error code if ctx not found. + */ +static int wd_sched_skey_update_load(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_idx, int delta) { - struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - struct sched_params *param = (struct sched_params *)sched_param; - struct sched_key *skey; + /* Atomic update without locking, ctx_idx's value is guaranteed by the caller. */ + if (delta > 0) + __atomic_fetch_add(&cache->load_values[ctx_idx], delta, __ATOMIC_RELAXED); + else { + __atomic_fetch_sub(&cache->load_values[ctx_idx], -delta, __ATOMIC_RELAXED); + if (unlikely(!cache->load_decreased)) + cache->load_decreased = 1; + } + return WD_SUCCESS; +} + +/* ============================================================================ + * Session Key Domain Initialization + * ============================================================================ + */ + +/** + * wd_sched_skey_domain_init - Initialize session domain with min-heap + * @key_domain: Target key domain + * @ctx_idx: context indices idx + * @policy: current session's policy + * + * Initializes dual-domain structure for session. + */ +static int wd_sched_skey_domain_init(struct wd_sched_key_domain *key_domain, + __u32 ctx_idx, __u8 policy) +{ + int ret; + + if (!key_domain) + return -WD_EINVAL; + + ret = wd_sched_skey_cache_init(&key_domain->idx_cache, policy); + if (ret) + return ret; + + ret = wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx); + if (ret) + goto init_err; + + ret = pthread_mutex_init(&key_domain->lock, NULL); + if (ret) + goto add_ctx_err; + + key_domain->expanded_count = 0; + + return WD_SUCCESS; + +add_ctx_err: + wd_sched_skey_remove_ctx(&key_domain->idx_cache, ctx_idx); +init_err: + wd_sched_skey_cache_uninit(&key_domain->idx_cache); + return ret; +} + +/** + * wd_sched_skey_domain_destroy - Release session domain resources + */ +static void wd_sched_skey_domain_destroy(struct wd_sched_key_domain *key_domain) +{ + if (!key_domain) + return; + + pthread_mutex_destroy(&key_domain->lock); + wd_sched_skey_cache_uninit(&key_domain->idx_cache); +} + + +/** + * wd_sched_poll_skey - Poll contexts for scheduler session + * @sched_ctx: Scheduler context + * @skey: Session key + * @expect: Expected number of responses + * @count: Actual response count (output) + * + * Polls all contexts in session domains and updates load values. + */ +static int wd_sched_poll_skey(struct wd_sched_ctx *sched_ctx, struct wd_sched_key *skey, + __u32 expect, __u32 *count) +{ + struct wd_sched_domain_idx_cache *cache; + __u32 ctx_list_num = 0; + __u32 sum_poll_num = 0; + bool hungry_policy; + __u32 poll_num; + __u32 idx, i; + int ret = 0; + + /* Get cache pointer and check if HUNGRY policy */ + cache = &skey->async_domain.idx_cache; + ctx_list_num = cache->valid_count; + hungry_policy = (cache->policy == SCHED_POLICY_HUNGRY); + + /* Poll async domain contexts */ + for (i = 0; i < ctx_list_num; i++) { + idx = cache->idx_list[i]; + if (idx == INVALID_POS) + continue; + poll_num = 0; + ret = sched_ctx->poll_func(idx, expect, &poll_num); + if (poll_num > 0) + sum_poll_num += poll_num; + + /* Update load value for this context */ + if (hungry_policy && poll_num > 0) { + if (skey->ctxs) + __atomic_fetch_sub(&skey->ctxs[idx].hw_load, + poll_num, __ATOMIC_RELAXED); + wd_sched_skey_update_load(cache, i, -poll_num); + } + + if (ret < 0 && ret != -WD_EAGAIN) + break; + } + *count = sum_poll_num; + + return ret; +} + +/* ============================================================================ + * Utility Functions + * ============================================================================ + */ +static inline bool sched_skey_get_ref(struct wd_sched_key *skey) +{ + if (!skey) + return false; + return __atomic_fetch_add(&skey->refcount, 1, __ATOMIC_ACQUIRE) > 0; +} + +static inline bool sched_skey_put_ref(struct wd_sched_key *skey) +{ + return __atomic_fetch_sub(&skey->refcount, 1, __ATOMIC_RELEASE) == 1; +} + +static int sched_skey_param_init(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey) +{ + __u32 i; + + pthread_mutex_lock(&sched_ctx->skey_lock); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + if (!sched_ctx->skey[i]) { + sched_ctx->skey[i] = skey; + if (sched_ctx->skey_num < SKEY_MAX_THREAD_NUM) + sched_ctx->skey_num++; + pthread_mutex_unlock(&sched_ctx->skey_lock); + return 0; + } + } + pthread_mutex_unlock(&sched_ctx->skey_lock); + WD_ERR("invalid: skey node number exceeds SKEY_MAX_THREAD_NUM(%d)!\n", + SKEY_MAX_THREAD_NUM); + return -WD_ENOMEM; +} + +static void sched_skey_param_uninit(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey) +{ + __u32 i; + + if (!sched_ctx || !skey) + return; + + pthread_mutex_lock(&sched_ctx->skey_lock); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + if (sched_ctx->skey[i] == skey) { + sched_ctx->skey[i] = NULL; + pthread_mutex_unlock(&sched_ctx->skey_lock); + return; + } + } + pthread_mutex_unlock(&sched_ctx->skey_lock); + WD_ERR("warning: skey %p not found in sched_ctx array\n", skey); +} + +static handle_t sched_session_common_init(struct wd_sched_ctx *sched_ctx, + struct sched_params *param) +{ + struct wd_sched_key *skey; unsigned int node; if (getcpu(NULL, &node)) { WD_ERR("failed to get node, errno %d!\n", errno); return (handle_t)(-errno); } - if (node == (unsigned int)NUMA_NO_NODE) { - WD_ERR("invalid: failed to get numa node!\n"); - return (handle_t)(-WD_EINVAL); - } if (!sched_ctx) { WD_ERR("invalid: sched ctx is NULL!\n"); return (handle_t)(-WD_EINVAL); } - skey = malloc(sizeof(struct sched_key)); + skey = malloc(sizeof(struct wd_sched_key)); if (!skey) { WD_ERR("failed to alloc memory for session sched key!\n"); return (handle_t)(-WD_ENOMEM); } + memset(skey, 0, sizeof(struct wd_sched_key)); if (!param) { - memset(skey, 0, sizeof(struct sched_key)); - skey->numa_id = sched_ctx->numa_map[node]; + skey->region_id = node; if (wd_need_debug()) WD_DEBUG("session don't set scheduler parameters!\n"); - } else if (param->numa_id < 0) { - skey->type = param->type; - skey->numa_id = sched_ctx->numa_map[node]; } else { + if (sched_ctx->policy == SCHED_POLICY_DEV) + skey->region_id = param->dev_id; + else if (param->numa_id >= 0) + skey->region_id = param->numa_id; + else + skey->region_id = node; skey->type = param->type; - skey->numa_id = param->numa_id; + skey->ctx_prop = param->ctx_prop; } + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + __atomic_store_n(&skey->refcount, 1, __ATOMIC_RELAXED); - if (skey->numa_id < 0) { - WD_ERR("failed to get valid sched numa region!\n"); - goto out; - } + return (handle_t)skey; +} - skey->sync_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); - skey->async_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); - if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { - WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); - goto out; +static __u16 sched_get_poll_skey_idx(struct wd_sched_ctx *sched_ctx) +{ + /* Thread-local sequence number with atomic global counter */ + static __thread __u32 thread_seq = UINT32_MAX; + static __u32 global_seq_counter; + __u32 skey_num = sched_ctx->skey_num; + __u16 start_pos; + + if (unlikely(!sched_ctx || !skey_num)) + return skey_num; + + /* Assign unique sequence number on first call */ + if (unlikely(thread_seq == UINT32_MAX)) { + thread_seq = __atomic_fetch_add(&global_seq_counter, 1, __ATOMIC_RELAXED); + + /* Basic overflow protection */ + if (thread_seq >= UINT32_MAX - 1) + WD_DEBUG("Thread sequence counter approaching limit: %u\n", thread_seq); } - return (handle_t)skey; + /* Calculate start_pos based on thread_seq */ + start_pos = thread_seq % skey_num; -out: - free(skey); - return (handle_t)(-WD_EINVAL); + return start_pos; } -/* - * session_pick_next_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/** + * session_sched_init_ctx - Pre-fetch single context from domain for session + * @sched_ctx: Scheduler context + * @skey: session scheduler param + * @sched_mode: Mode (SYNC/ASYNC) * - * The user must init the schedule info through session_sched_init + * Returns: Context index from domain (compatible with alg_name if skey provided) */ -static __u32 session_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, - const int sched_mode) +static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, int sched_mode) { - struct sched_key *key = (struct sched_key *)sched_key; + struct wd_sched_ctx_domain *domain = NULL; + int region_id = skey->region_id; + __u32 op_type = skey->type; + __u8 prop = skey->ctx_prop; + __u32 ctx_idx; + __u16 r; + + if (sched_mode >= SCHED_MODE_BUTT || + op_type >= sched_ctx->type_num || prop >= UADK_ALG_TYPE_MAX) { + WD_ERR("invalid: region: %d, mode: %d, type: %u!, prop: %u\n", + region_id, sched_mode, op_type, prop); + return INVALID_POS; + } - if (unlikely(!h_sched_ctx || !key)) { - WD_ERR("invalid: sched ctx or key is NULL!\n"); + if (!sched_ctx->domain_hash_table) + return INVALID_POS; + + /* Try current region first */ + if (region_id >= 0) { + if (sched_ctx->policy == SCHED_POLICY_DEV || + region_id < sched_ctx->region_num) { + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + region_id, sched_mode, op_type, prop); + if (domain && domain->valid) + return wd_sched_domain_get_next_rr(domain); + } + } + + /* DEV policy must not cross region */ + if (sched_ctx->policy == SCHED_POLICY_DEV) return INVALID_POS; + + /* Cross-region fallback for other policies */ + for (r = 0; r < sched_ctx->region_num; r++) { + if ((int)r == region_id) + continue; + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + r, sched_mode, op_type, prop); + if (domain && domain->valid) { + ctx_idx = wd_sched_domain_get_next_rr(domain); + if (ctx_idx != INVALID_POS) + return ctx_idx; + } } - /* return in do task */ - if (sched_mode == CTX_MODE_SYNC) - return key->sync_ctxid; - return key->async_ctxid; + return INVALID_POS; } -static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, - __u32 end, __u32 expect, __u32 *count) +/** + * session_sched_domain_destroy - Destroy session domains + * @skey: Session key to destroy domains for + * + * Releases all resources associated with session domains. + */ +static void session_sched_domain_destroy(struct wd_sched_key *skey) { - __u32 poll_num = 0; - __u32 i; - int ret; + if (!skey) + return; - /* i is the pos of sched_ctxs, the max is end */ - for (i = begin; i <= end; i++) { - /* - * RR schedule, one time poll one package, - * poll_num is always not more than one here. - */ - ret = sched_ctx->poll_func(i, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) - return ret; - else if (ret == -EAGAIN) + /* Destroy both sync and async domains */ + wd_sched_skey_domain_destroy(&skey->sync_domain); + wd_sched_skey_domain_destroy(&skey->async_domain); +} + +static inline void sched_skey_poll_release(struct wd_sched_key *skey) +{ + __u32 prev; + + if (!skey) + return; + + prev = __atomic_fetch_sub(&skey->refcount, 1, __ATOMIC_ACQ_REL); + if (prev == 1) { + session_sched_domain_destroy(skey); + free(skey); + } +} + +static void sched_session_uninit(handle_t h_sched_ctx, handle_t h_sched_key) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_key *skey = (struct wd_sched_key *)h_sched_key; + + if (!skey) + return; + + /* Remove from skey array first to prevent new poll discovery. + * Poll threads already holding a pointer are protected by refcount. + */ + if (sched_ctx) + sched_skey_param_uninit(sched_ctx, skey); + + /* Drop creator reference. If no poll holds a ref, free now. + * Otherwise the last poll thread will cleanup via poll_release. + */ + if (sched_skey_put_ref(skey)) { + session_sched_domain_destroy(skey); + free(skey); + } +} + +/** + * session_sched_init_ctx_with_fallback - Pre-fetch a ctx for one sched_mode, + * trying user_prop first then falling + * back through other props in enum order. + * @sched_ctx: Scheduler context + * @skey: Session key (skey->ctx_prop is the user-specified prop; restored on return) + * @sched_mode: SCHED_MODE_SYNC or SCHED_MODE_ASYNC + * + * Returns: ctx index, or INVALID_POS if no prop has a usable ctx. + * + * Falls back only when the user-specified prop's domain is unavailable + * (e.g. HW driver unloaded). Stops at the first prop that yields a ctx; + * does NOT accumulate ctxs across props. + */ +static __u32 session_sched_init_ctx_with_fallback(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + int sched_mode) +{ + __u8 user_prop = skey->ctx_prop; + __u32 ctx_idx; + __u8 p; + + /* Try user-specified prop first */ + ctx_idx = session_sched_init_ctx(sched_ctx, skey, sched_mode); + if (ctx_idx != INVALID_POS) + return ctx_idx; + + /* Fallback: target prop unavailable, try remaining props in enum order */ + for (p = 0; p < UADK_ALG_TYPE_MAX; p++) { + if (p == user_prop) continue; - *count += poll_num; - if (*count == expect) + skey->ctx_prop = p; + ctx_idx = session_sched_init_ctx(sched_ctx, skey, sched_mode); + if (ctx_idx != INVALID_POS) break; } - return 0; + /* Restore user_prop so skey state is not mutated */ + skey->ctx_prop = user_prop; + return ctx_idx; } -static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, - __u32 expect, __u32 *count) +/** + * session_sched_domain_init - Initialize session domains with sync/async + * @sched_ctx: Scheduler context + * @skey: Session key to initialize + * @allow_fallback: If true, fall back to other props when skey->ctx_prop has + * no usable domain; if false, only try skey->ctx_prop. + * + * Pre-fetches sync and async contexts and initializes corresponding domains. + * Returns: 0 on success, negative error code on failure. + */ +static int session_sched_domain_init(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + bool allow_fallback) { - struct sched_ctx_region **region = sched_ctx->sched_info[numa_id].ctx_region; - __u32 begin, end; - __u32 i; - int ret; + __u32 sync_ctx, async_ctx; - for (i = 0; i < sched_ctx->type_num; i++) { - if (!region[SCHED_MODE_ASYNC][i].valid) - continue; + if (!sched_ctx || !skey) { + WD_ERR("invalid: sched_ctx or skey is NULL!\n"); + return -WD_EINVAL; + } + + if (allow_fallback) { + /* Pre-fetch with per-mode prop fallback */ + sync_ctx = session_sched_init_ctx_with_fallback(sched_ctx, skey, + SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx_with_fallback(sched_ctx, skey, + SCHED_MODE_ASYNC); + } else { + sync_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_ASYNC); + } + + if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) { + WD_ERR("invalid: no valid sync_ctx or async_ctx domain!\n"); + return -WD_EINVAL; + } + + /* Initialize sync domain if context is valid */ + if (sync_ctx != INVALID_POS) { + if (wd_sched_skey_domain_init(&skey->sync_domain, sync_ctx, sched_ctx->policy)) { + WD_ERR("failed to init sync domain!\n"); + return -WD_EINVAL; + } + } + + /* Initialize async domain if context is valid */ + if (async_ctx != INVALID_POS) { + if (wd_sched_skey_domain_init(&skey->async_domain, async_ctx, sched_ctx->policy)) { + WD_ERR("failed to init async domain!\n"); + /* Cleanup sync domain if async domain init failed */ + if (sync_ctx != INVALID_POS) + wd_sched_skey_domain_destroy(&skey->sync_domain); + return -WD_EINVAL; + } + } + + return WD_SUCCESS; +} + +/* ============================================================================ + * Scheduler Policy Functions + * ============================================================================ + */ +/** + * round_robin_sched_init - Initialize session with single sync and async ctx + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_param: Scheduling parameters (cannot modify per API contract) + * + * Allocates session key and pre-fetches one sync and one async context. + */ +static handle_t round_robin_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + struct wd_sched_key *skey; + handle_t hskey; + int ret = 0; + + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; + } + + skey = (struct wd_sched_key *)hskey; + /* RR: allow prop fallback so session creation survives target prop unload */ + ret = session_sched_domain_init(sched_ctx, skey, true); + if (ret) { + WD_ERR("failed to initialize session domains!\n"); + free(skey); + return (handle_t)(-WD_EINVAL); + } + + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); + return (handle_t)(-WD_ENOMEM); + } + + return hskey; +} + +/** + * round_robin_pick_next_ctx - Pick context with load-based selection + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) + * + * Returns: Context index with minimum load + * Time complexity: O(1) + */ +static __u32 round_robin_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_key_domain *domain; + __u32 min_ctx, ctx_idx; - begin = region[SCHED_MODE_ASYNC][i].begin; - end = region[SCHED_MODE_ASYNC][i].end; - ret = session_poll_region(sched_ctx, begin, end, expect, count); - if (unlikely(ret)) - return ret; + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; } - return 0; + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; + + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) + return INVALID_POS; + + return min_ctx; } -/* - * session_poll_policy - The polling policy matches the pick next ctx. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @cfg: The global resoure info. - * @expect: User expect poll msg num. - * @count: The actually poll num. +/** + * round_robin_poll_policy - Poll policy for session scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) * - * The user must init the schedule info through wd_sched_rr_instance, the - * func interval will not check the valid, becouse it will affect performance. + * Returns: Status code */ -static int session_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +static int round_robin_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - struct wd_sched_info *sched_info; - __u32 loop_time = 0; - __u32 last_count = 0; - __u16 i, region_mum; - int ret; + struct wd_sched_key *skey; + __u16 i, tpos, start_pos; + __u32 poll_num, skey_num; + int ret = -WD_EAGAIN; + __u32 sum_count = 0; if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); return -WD_EINVAL; } - if (unlikely(sched_ctx->numa_num > NUMA_NUM_NODES)) { - WD_ERR("invalid: ctx's numa number is %u!\n", sched_ctx->numa_num); - return -WD_EINVAL; + /* Randomize the initial query position. */ + skey_num = sched_ctx->skey_num; + start_pos = sched_get_poll_skey_idx(sched_ctx); + if (!skey_num || start_pos >= sched_ctx->skey_num) { + *count = 0; + return -WD_EAGAIN; } - sched_info = sched_ctx->sched_info; - if (sched_ctx->policy == SCHED_POLICY_DEV) - region_mum = sched_ctx->dev_num; - else - region_mum = sched_ctx->numa_num; - - /* - * Try different region's ctx if we can't receive any - * package last time, it is more efficient. In most - * bad situation, poll ends after MAX_POLL_TIMES loop. - */ - while (++loop_time < MAX_POLL_TIMES) { - for (i = 0; i < region_mum;) { - /* If current numa is not valid, find next. */ - if (!sched_info[i].valid) { - i++; - continue; - } + /* Query the queues on each skey separately. */ + for (i = 0; i < skey_num; i++) { + tpos = (start_pos + i) % skey_num; + skey = sched_ctx->skey[tpos]; - last_count = *count; - ret = session_poll_policy_rr(sched_ctx, i, expect, count); - if (unlikely(ret)) - return ret; + if (unlikely(!skey)) + continue; - if (expect == *count) - return 0; + if (!sched_skey_get_ref(skey)) + continue; - /* - * If no package is received, find next numa, - * otherwise, keep receiving packets at this node. - */ - if (last_count == *count) - i++; + if (__atomic_test_and_set(&skey->poll_lock, __ATOMIC_ACQUIRE)) { + sched_skey_poll_release(skey); + continue; } + + ret = wd_sched_poll_skey(sched_ctx, skey, expect, &poll_num); + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + sched_skey_poll_release(skey); + + sum_count += poll_num; + if (unlikely(ret && ret != -WD_EAGAIN)) + goto poll_err; + + if (sum_count >= expect) + break; } - return 0; +poll_err: + *count = sum_count; + if (ret == -WD_EAGAIN) + return 0; + return ret; } static handle_t sched_none_init(handle_t h_sched_ctx, void *sched_param) @@ -396,12 +1464,11 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, } while (loop_times > 0) { - /* Default use ctx 0 */ loop_times--; ret = sched_ctx->poll_func(0, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) + if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; - else if (ret == -EAGAIN) + else if (ret == -WD_EAGAIN) continue; *count += poll_num; @@ -409,7 +1476,7 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, break; } - return 0; + return WD_SUCCESS; } static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) @@ -420,13 +1487,10 @@ static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) static __u32 sched_single_pick_next_ctx(handle_t sched_ctx, void *sched_key, const int sched_mode) { -#define CTX_ASYNC 1 -#define CTX_SYNC 0 - if (sched_mode) - return CTX_ASYNC; + return 1; else - return CTX_SYNC; + return 0; } static int sched_single_poll_policy(handle_t h_sched_ctx, @@ -443,12 +1507,11 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, } while (loop_times > 0) { - /* Default async mode use ctx 1 */ loop_times--; ret = sched_ctx->poll_func(1, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) + if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; - else if (ret == -EAGAIN) + else if (ret == -WD_EAGAIN) continue; *count += poll_num; @@ -456,260 +1519,483 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, break; } - return 0; + return WD_SUCCESS; } -static bool sched_dev_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +/** + * sched_skey_domain_fill - Initialize or append ctx to session domain + * @key_domain: Target domain (sync or async) + * @ctx_idx: Context index to add + * @policy: Scheduler policy + * @inited: Pointer to boolean tracking whether domain has been initialized + * + * First call: full domain init via wd_sched_skey_domain_init(). + * Subsequent calls: append via wd_sched_skey_add_ctx(). + */ +static void sched_skey_domain_fill(struct wd_sched_key_domain *key_domain, + __u32 ctx_idx, __u8 policy, bool *inited) { - bool found = false; - int i; + int ret; - if (key->mode >= SCHED_MODE_BUTT || key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's device id: %u, mode: %u, type: %u!\n", - key->dev_id, key->mode, key->type); - return false; + if (ctx_idx == INVALID_POS) + return; + + if (!*inited) { + ret = wd_sched_skey_domain_init(key_domain, ctx_idx, policy); + if (!ret) + *inited = true; + return; } - for (i = 0; i < sched_ctx->dev_num; i++) { - if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { - found = true; - break; + (void)wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx); +} + +/** + * sched_skey_common_init - Common scheduler init with init-once + add-rest pattern + * @h_sched_ctx: Scheduler handle + * @sched_param: Scheduling parameters + * @prop_begin: First prop type to iterate (inclusive) + * @prop_end: Last prop type to iterate (inclusive) + * + * Shared by Loop, Hungry, and Instr scheduler init functions. + * Iterates [prop_begin, prop_end], fetching sync/async ctxs per prop type. + * First valid ctx initializes the domain, subsequent ctxs are appended. + */ +static handle_t sched_skey_common_init(handle_t h_sched_ctx, void *sched_param, + __u8 prop_begin, __u8 prop_end) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + struct wd_sched_key *skey; + bool sync_inited = false; + bool async_inited = false; + __u32 sync_ctx, async_ctx; + __u32 req_ctx_num = 0; + handle_t hskey; + __u8 def_prop; + int ret, i; + + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; + } + + skey = (struct wd_sched_key *)hskey; + def_prop = skey->ctx_prop; + for (i = prop_begin; i <= prop_end; i++) { + skey->ctx_prop = i; + sync_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_ASYNC); + if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) + continue; + + if (sync_ctx != INVALID_POS) { + sched_skey_domain_fill(&skey->sync_domain, sync_ctx, + sched_ctx->policy, &sync_inited); } + if (async_ctx != INVALID_POS) + sched_skey_domain_fill(&skey->async_domain, async_ctx, + sched_ctx->policy, &async_inited); + + req_ctx_num += 2; + } + if (!req_ctx_num) { + free(skey); + return (handle_t)(-WD_EINVAL); } - if (!found) { - WD_ERR("invalid: dev_id %u is not registered!\n", key->dev_id); - return false; + skey->ctx_prop = def_prop; + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); + return (handle_t)(-WD_ENOMEM); } - return true; + return hskey; } -/* - * sched_dev_get_region - Get ctx region from ctx_map by the wd comp arg +/** + * skey_sched_init - Initialize Hungry scheduler session + */ +static handle_t skey_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + 0, UADK_ALG_TYPE_MAX - 1); + return hskey; +} + +/** + * skey_sched_pick_next_ctx - Pick context from hungry scheduler with load awareness + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) + * + * Returns: Context with minimum load, or expands if threshold exceeded + * Time complexity: O(1) for selection, O(n) if expansion needed */ -static struct sched_ctx_region *sched_dev_get_region(struct wd_sched_ctx *sched_ctx, - const struct sched_key *key) +static __u32 skey_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) { - struct wd_sched_info *sched_info; - int i, region_id; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_domain_idx_cache *idx_cache; + struct wd_sched_key_domain *domain; + __u32 min_ctx, min_load, ctx_idx; - for (i = 0; i < sched_ctx->dev_num; i++) { - if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { - region_id = sched_ctx->dev_id_map[i].region_id; - sched_info = &sched_ctx->sched_info[region_id]; - if (sched_info->ctx_region[key->mode][key->type].valid) - return &sched_info->ctx_region[key->mode][key->type]; + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; + } + + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; + + idx_cache = &domain->idx_cache; + if (sched_mode == SCHED_MODE_SYNC) { + /* Sync: send+recv atomic, spinlock serialized, use RR */ + if (!idx_cache->valid_count) + return INVALID_POS; + + ctx_idx = __atomic_fetch_add(&idx_cache->rr_ptr, 1, __ATOMIC_RELAXED) % + idx_cache->valid_count; + return idx_cache->idx_list[ctx_idx]; + } + + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) + return INVALID_POS; + + if (skey->ctxs) { + min_load = __atomic_load_n(&skey->ctxs[min_ctx].hw_load, __ATOMIC_RELAXED); + if (min_load >= HW_QUEUE_FULL_DEPTH) + return QUEUE_FULL_POS; + __atomic_fetch_add(&skey->ctxs[min_ctx].hw_load, 1, __ATOMIC_RELAXED); + wd_sched_skey_update_load(idx_cache, ctx_idx, 1); + } else { + min_load = __atomic_load_n(&idx_cache->load_values[ctx_idx], __ATOMIC_RELAXED); + if (min_load >= HW_QUEUE_FULL_DEPTH && idx_cache->load_decreased > 0) + return QUEUE_FULL_POS; + wd_sched_skey_update_load(idx_cache, ctx_idx, 1); + } + /* Check if we need to expand context pool */ + + return min_ctx; +} + +/** + * skey_sched_poll_policy - Poll policy for hungry scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) + * + * Returns: Status code + */ +static int skey_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_key *skey; + __u16 i, tpos, start_pos; + __u32 poll_num, skey_num; + int ret = -WD_EAGAIN; + __u32 sum_count = 0; + + if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { + WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); + return -WD_EINVAL; + } + + /* Randomize the initial query position. */ + skey_num = sched_ctx->skey_num; + start_pos = sched_get_poll_skey_idx(sched_ctx); + if (!skey_num || start_pos >= sched_ctx->skey_num) { + *count = 0; + return -WD_EAGAIN; + } + + /* Query the queues on each skey separately. */ + for (i = 0; i < skey_num; i++) { + tpos = (start_pos + i) % skey_num; + skey = sched_ctx->skey[tpos]; + + if (unlikely(!skey)) + continue; + + if (!sched_skey_get_ref(skey)) + continue; + + if (__atomic_test_and_set(&skey->poll_lock, __ATOMIC_ACQUIRE)) { + sched_skey_poll_release(skey); + continue; } + + ret = wd_sched_poll_skey(sched_ctx, skey, expect, &poll_num); + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + sched_skey_poll_release(skey); + /* + * This query returned 0, indicating the hardware + * likely hasn't finished processing yet. + * Implementing a delay and releasing the CPU is a predictive optimization + */ + if (!poll_num) + usleep(1); + + sum_count += poll_num; + if (ret == -WD_EAGAIN) + continue; + if (unlikely(ret) || sum_count >= expect) + break; } + *count = sum_count; - /* - * If the scheduling domain of dev_id does not exist, - * taskes operations cannot be executed using queues from other devices; - * otherwise, an SMMU error will occur. - */ - return NULL; + return ret; } -/* - * session_dev_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/** + * loop_sched_init - Initialize Loop scheduler session + */ +static handle_t loop_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + 0, UADK_ALG_TYPE_MAX - 1); + return hskey; +} + +/** + * loop_sched_pick_next_ctx - Pick context for loop scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) * - * The user must init the schedule info through wd_sched_rr_instance + * Returns: Context index with minimum load + * Time complexity: O(1) + */ +static __u32 loop_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + return round_robin_pick_next_ctx(h_sched_ctx, sched_key, sched_mode); +} + +static int loop_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + return round_robin_poll_policy(h_sched_ctx, expect, count); +} + +/** + * instr_sched_init - Initialize Instr scheduler session */ -static __u32 session_dev_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, - const int sched_mode) +static handle_t instr_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + UADK_ALG_CE_INSTR, + UADK_ALG_SVE_INSTR); + return hskey; +} + +static __u32 instr_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) { - struct sched_ctx_region *region = NULL; - bool ret; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_key_domain *domain; + __u32 min_ctx, ctx_idx; - key->mode = sched_mode; - ret = sched_dev_key_valid(sched_ctx, key); - if (!ret) + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); return INVALID_POS; + } + + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; - region = sched_dev_get_region(sched_ctx, key); - if (!region) + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) return INVALID_POS; - return sched_get_next_pos_rr(region, NULL); + return min_ctx; +} + +/** + * instr_sched_poll_policy - Poll policy for instruction scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) + * + * Returns: Status code + */ +static int instr_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + return round_robin_poll_policy(h_sched_ctx, expect, count); } static handle_t session_dev_sched_init(handle_t h_sched_ctx, void *sched_param) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; struct sched_params *param = (struct sched_params *)sched_param; - struct sched_key *skey; - unsigned int node; + struct wd_sched_key *skey; + handle_t hskey; + int ret = 0; - if (getcpu(NULL, &node)) { - WD_ERR("failed to get numa node, errno %d!\n", errno); - return (handle_t)(-errno); - } - if (node == (unsigned int)NUMA_NO_NODE) { - WD_ERR("invalid: failed to get numa node for dev sched init!\n"); + if (!param) { + WD_ERR("invalid: dev sched param is NULL!\n"); return (handle_t)(-WD_EINVAL); } - if (!sched_ctx) { - WD_ERR("invalid: sched ctx is NULL!\n"); - return (handle_t)(-WD_EINVAL); + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; } - if (!param) { - WD_DEBUG("no-sva session don't set scheduler parameters!\n"); + skey = (struct wd_sched_key *)hskey; + skey->type = param->type; + skey->dev_id = param->dev_id; + + /* DEV: pinned to a specific device, do not cross prop families */ + ret = session_sched_domain_init(sched_ctx, skey, false); + if (ret) { + WD_ERR("failed to initialize session domains!\n"); + free(skey); return (handle_t)(-WD_EINVAL); } - skey = malloc(sizeof(struct sched_key)); - if (!skey) { - WD_ERR("failed to alloc memory for session sched key!\n"); + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); return (handle_t)(-WD_ENOMEM); } - skey->type = param->type; - skey->dev_id = param->dev_id; + return (handle_t)skey; +} + +/** + * wd_sched_set_param - Set scheduler parameters + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_param: Scheduling parameters (cannot modify per API contract) + */ + +static void wd_sched_set_param(handle_t h_sched_ctx, + void *sched_key, void *sched_param) +{ + struct wd_sched_params *params = (struct wd_sched_params *)sched_param; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; - skey->sync_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); - skey->async_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); - if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { - WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); - goto out; + if (unlikely(!params || !skey)) { + WD_ERR("invalid: sched parmas or skey is NULL!\n"); + return; } - return (handle_t)skey; + skey->pkt_size = params->pkt_size; + skey->is_stream = params->data_mode; + skey->prio_mode = params->prio_mode; + + /* Store compat filtering parameters */ + skey->alg_name = params->alg_name; + skey->ctxs = params->ctxs; -out: - free(skey); - return (handle_t)(-WD_EINVAL); } static struct wd_sched sched_table[SCHED_POLICY_BUTT] = { { .name = "RR scheduler", .sched_policy = SCHED_POLICY_RR, - .sched_init = session_sched_init, - .pick_next_ctx = session_sched_pick_next_ctx, - .poll_policy = session_sched_poll_policy, + .sched_init = round_robin_sched_init, + .pick_next_ctx = round_robin_pick_next_ctx, + .poll_policy = round_robin_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "None scheduler", .sched_policy = SCHED_POLICY_NONE, .sched_init = sched_none_init, .pick_next_ctx = sched_none_pick_next_ctx, .poll_policy = sched_none_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "Single scheduler", .sched_policy = SCHED_POLICY_SINGLE, .sched_init = sched_single_init, .pick_next_ctx = sched_single_pick_next_ctx, .poll_policy = sched_single_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "Device RR scheduler", .sched_policy = SCHED_POLICY_DEV, .sched_init = session_dev_sched_init, - .pick_next_ctx = session_sched_pick_next_ctx, - .poll_policy = session_sched_poll_policy, - } + .pick_next_ctx = round_robin_pick_next_ctx, + .poll_policy = round_robin_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Loop scheduler", + .sched_policy = SCHED_POLICY_LOOP, + .sched_init = loop_sched_init, + .pick_next_ctx = loop_sched_pick_next_ctx, + .poll_policy = loop_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Hungry scheduler", + .sched_policy = SCHED_POLICY_HUNGRY, + .sched_init = skey_sched_init, + .pick_next_ctx = skey_sched_pick_next_ctx, + .poll_policy = skey_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Instr scheduler", + .sched_policy = SCHED_POLICY_INSTR, + .sched_init = instr_sched_init, + .pick_next_ctx = instr_sched_pick_next_ctx, + .poll_policy = instr_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, }; -static int wd_sched_get_nearby_numa_id(struct wd_sched_info *sched_info, int node, int numa_num) -{ - int dis = INT32_MAX; - int valid_id = -1; - int i, tmp; - - for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) { - tmp = numa_distance(node, i); - if (dis > tmp) { - valid_id = i; - dis = tmp; - } - } - } - - return valid_id; -} - -static void wd_sched_map_cpus_to_dev(struct wd_sched_ctx *sched_ctx) -{ - struct wd_sched_info *sched_info = sched_ctx->sched_info; - int i, numa_num = sched_ctx->numa_num; - int *numa_map = sched_ctx->numa_map; - - for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) - numa_map[i] = i; - else - numa_map[i] = wd_sched_get_nearby_numa_id(sched_info, i, numa_num); - } -} - -static int wd_instance_dev_region(struct wd_sched_ctx *sched_ctx, - struct sched_params *param) +static int numa_num_check(__u16 region_num) { - struct wd_sched_info *sched_info; - __u32 region_idx = INVALID_POS; - __u8 type, mode; - __u32 dev_id; - int i; - - dev_id = param->dev_id; - type = param->type; - mode = param->mode; - - /* Check whether dev_id has already been registered. */ - for (i = 0; i < sched_ctx->dev_num; i++) { - if (sched_ctx->dev_id_map[i].dev_id == dev_id) { - region_idx = sched_ctx->dev_id_map[i].region_id; - break; - } - } - - /* If not registered, allocate a new region. */ - if (region_idx == INVALID_POS) { - if (sched_ctx->dev_num >= DEVICE_REGION_MAX) { - WD_ERR("too many devices registered!\n"); - return -WD_EINVAL; - } - - region_idx = sched_ctx->dev_num; - sched_ctx->dev_id_map[region_idx].dev_id = dev_id; - sched_ctx->dev_id_map[region_idx].region_id = region_idx; - sched_ctx->dev_num++; + int max_node; - sched_info = &sched_ctx->sched_info[region_idx]; - } else { - sched_info = &sched_ctx->sched_info[region_idx]; + max_node = numa_max_node() + 1; + if (max_node <= 0) { + WD_ERR("invalid: numa max node is %d!\n", max_node); + return -WD_EINVAL; } - /* Check whether the mode and type have already been registered. */ - if (sched_info->ctx_region[mode][type].valid) { - WD_INFO("device %u mode %u type %u already registered\n", - dev_id, mode, type); - return WD_SUCCESS; + if (!region_num || region_num > max_node) { + WD_ERR("invalid: region number is %u!\n", region_num); + return -WD_EINVAL; } - /* Initialize the scheduling region for this mode and type */ - sched_info->ctx_region[mode][type].begin = param->begin; - sched_info->ctx_region[mode][type].end = param->end; - sched_info->ctx_region[mode][type].last = param->begin; - sched_info->ctx_region[mode][type].valid = true; - sched_info->valid = true; - - pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); - - return WD_SUCCESS; + return 0; } +/** + * wd_sched_rr_instance - External API for scheduling region instance + * @sched: Scheduler (cannot modify per API contract) + * @param: Scheduling parameters (cannot modify per API contract) + * + * Creates scheduling region for given parameters. + */ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *param) { - struct wd_sched_info *sched_info = NULL; struct wd_sched_ctx *sched_ctx = NULL; - __u8 type, mode; - int numa_id; + struct wd_sched_ctx_domain *domain; + int region_key; + __u8 mode; + int ret; if (!sched || !sched->h_sched_ctx || !param) { WD_ERR("invalid: sched or sched_params is NULL!\n"); @@ -721,57 +2007,68 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para return -WD_EINVAL; } - numa_id = param->numa_id; - type = param->type; mode = param->mode; sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; - if (sched_ctx->numa_num > 0 && (numa_id >= sched_ctx->numa_num || - numa_id < 0)) { - WD_ERR("invalid: sched_ctx's numa_id is %d, numa_num is %u!\n", - numa_id, sched_ctx->numa_num); - return -WD_EINVAL; + if (sched_ctx->policy == SCHED_POLICY_DEV) { + region_key = param->dev_id; + if (region_key < 0) { + WD_ERR("invalid: dev_id is %d!\n", region_key); + return -WD_EINVAL; + } + } else { + region_key = param->numa_id; + if (region_key >= sched_ctx->region_num || region_key < 0) { + WD_ERR("invalid: region_key is %d, region_num is %u!\n", + region_key, sched_ctx->region_num); + return -WD_EINVAL; + } } - if (type >= sched_ctx->type_num) { - WD_ERR("invalid: sched_ctx's type is %u, type_num is %u!\n", - type, sched_ctx->type_num); + if (param->type >= sched_ctx->type_num) { + WD_ERR("invalid: type is %u, type_num is %u!\n", + param->type, sched_ctx->type_num); return -WD_EINVAL; } if (mode >= SCHED_MODE_BUTT) { - WD_ERR("invalid: sched_ctx's mode is %u, mode_num is %d!\n", - mode, SCHED_MODE_BUTT); + WD_ERR("invalid: mode is %u, mode_num is %u!\n", + mode, sched_ctx->mode_num); return -WD_EINVAL; } - if (sched_ctx->policy == SCHED_POLICY_DEV) - return wd_instance_dev_region(sched_ctx, param); - - sched_info = &sched_ctx->sched_info[numa_id]; - if (!sched_info->ctx_region[mode]) { - WD_ERR("invalid: ctx_region is NULL, numa: %d, mode: %u!\n", - numa_id, mode); - return -WD_EINVAL; + if (param->ctx_prop < 0 || param->ctx_prop >= UADK_ALG_TYPE_MAX) { + WD_INFO("Info: ctx_prop %d exceeds max type!\n", param->ctx_prop); + param->ctx_prop = UADK_ALG_HW; } - sched_info->ctx_region[mode][type].begin = param->begin; - sched_info->ctx_region[mode][type].end = param->end; - sched_info->ctx_region[mode][type].last = param->begin; - sched_info->ctx_region[mode][type].valid = true; - sched_info->valid = true; + domain = wd_sched_hash_table_insert(sched_ctx->domain_hash_table, + region_key, mode, param->type, + param->ctx_prop); + if (!domain) + return -WD_ENOMEM; - wd_sched_map_cpus_to_dev(sched_ctx); - pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); + ret = wd_sched_domain_add_segment(domain, param->begin, param->end); + if (ret) { + WD_ERR("failed to add segment to domain!\n"); + return ret; + } + domain->valid = true; return WD_SUCCESS; } +/** + * wd_sched_rr_release - External API for scheduler release + * @sched: Scheduler to release (cannot modify per API contract) + * + * Releases all scheduler resources. + */ void wd_sched_rr_release(struct wd_sched *sched) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; - int i, j, region_num; + struct wd_sched_key *skey; + __u32 i; if (!sched) return; @@ -780,59 +2077,49 @@ void wd_sched_rr_release(struct wd_sched *sched) if (!sched_ctx) goto ctx_out; - /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ - if (sched_ctx->policy == SCHED_POLICY_DEV) - region_num = DEVICE_REGION_MAX; - else - region_num = sched_ctx->numa_num; + /* Release all session keys - iterate full array to catch residual entries */ + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + skey = sched_ctx->skey[i]; + if (!skey) + continue; - sched_info = sched_ctx->sched_info; - if (!sched_info) - goto info_out; + sched_ctx->skey[i] = NULL; + __atomic_store_n(&skey->refcount, 0, __ATOMIC_RELAXED); + session_sched_domain_destroy(skey); + free(skey); + } + sched_ctx->skey_num = 0; - for (i = 0; i < region_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - if (sched_info[i].ctx_region[j]) { - free(sched_info[i].ctx_region[j]); - sched_info[i].ctx_region[j] = NULL; - } - } + /* Release hash table */ + if (sched_ctx->domain_hash_table) { + wd_sched_hash_table_destroy(sched_ctx->domain_hash_table); + sched_ctx->domain_hash_table = NULL; } -info_out: + pthread_mutex_destroy(&sched_ctx->skey_lock); free(sched_ctx); + ctx_out: free(sched); - return; } -static int numa_num_check(__u16 numa_num) -{ - int max_node; - - max_node = numa_max_node() + 1; - if (max_node <= 0) { - WD_ERR("invalid: numa max node is %d!\n", max_node); - return -WD_EINVAL; - } - - if (!numa_num || numa_num > max_node) { - WD_ERR("invalid: numa number is %u!\n", numa_num); - return -WD_EINVAL; - } - - return 0; -} - +/** + * wd_sched_rr_alloc - External API for scheduler allocation + * @sched_type: Scheduling policy type (cannot modify per API contract) + * @type_num: Number of operation types (cannot modify per API contract) + * @region_num: Number of regions (cannot modify per API contract) + * @func: Poll function (cannot modify per API contract) + * + * Allocates and initializes scheduler with single global hash table. + */ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, - __u16 numa_num, user_poll_func func) + __u16 region_num, user_poll_func func) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; struct wd_sched *sched; - int region_num; - int i, j; + __u32 estimated_entries; + __u32 i; if (sched_type >= SCHED_POLICY_BUTT || !type_num) { WD_ERR("invalid: sched_type is %u or type_num is %u!\n", @@ -846,63 +2133,62 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, return NULL; } - if (sched_type == SCHED_POLICY_DEV) - region_num = DEVICE_REGION_MAX; - else - region_num = numa_num; - - sched_ctx = calloc(1, sizeof(struct wd_sched_ctx) + - sizeof(struct wd_sched_info) * region_num); + sched_ctx = calloc(1, sizeof(struct wd_sched_ctx)); if (!sched_ctx) { WD_ERR("failed to alloc memory for sched_ctx!\n"); goto err_out; } - /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ + /* Cache dimension parameters */ + sched_ctx->type_num = type_num; + sched_ctx->mode_num = SCHED_MODE_BUTT; + sched_ctx->region_num = region_num; + sched_ctx->policy = sched_type; + if (sched_type == SCHED_POLICY_DEV) { - sched_ctx->numa_num = 0; - sched_ctx->dev_num = 0; - for (i = 0; i < DEVICE_REGION_MAX; i++) { - sched_ctx->dev_id_map[i].dev_id = INVALID_POS; - sched_ctx->dev_id_map[i].region_id = INVALID_POS; - } + /* Device mode: region_num is actually device count */ + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } else { - sched_ctx->numa_num = numa_num; - sched_ctx->dev_num = 0; - if (numa_num_check(sched_ctx->numa_num)) + /* NUMA mode: validate region_num */ + if (numa_num_check(region_num)) goto err_out; + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } - sched->h_sched_ctx = (handle_t)sched_ctx; - if (sched_type == SCHED_POLICY_NONE || - sched_type == SCHED_POLICY_SINGLE) - goto simple_ok; - - sched_info = sched_ctx->sched_info; - for (i = 0; i < region_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - sched_info[i].ctx_region[j] = - calloc(1, sizeof(struct sched_ctx_region) * type_num); - if (!sched_info[i].ctx_region[j]) - goto err_out; - } + /* Create single global hash table */ + sched_ctx->domain_hash_table = wd_sched_hash_table_create(estimated_entries); + if (!sched_ctx->domain_hash_table) { + WD_ERR("failed to create hash table!\n"); + goto ctx_out; } -simple_ok: sched_ctx->poll_func = func; - sched_ctx->policy = sched_type; - sched_ctx->type_num = type_num; - memset(sched_ctx->numa_map, -1, sizeof(int) * NUMA_NUM_NODES); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) + sched_ctx->skey[i] = NULL; + + if (pthread_mutex_init(&sched_ctx->skey_lock, NULL)) { + WD_ERR("failed to init skey_lock!\n"); + goto err_destroy_hash; + } + sched_ctx->skey_num = 0; + + sched->h_sched_ctx = (handle_t)sched_ctx; sched->sched_init = sched_table[sched_type].sched_init; + sched->sched_uninit = sched_table[sched_type].sched_uninit; sched->pick_next_ctx = sched_table[sched_type].pick_next_ctx; sched->poll_policy = sched_table[sched_type].poll_policy; sched->sched_policy = sched_type; sched->name = sched_table[sched_type].name; + sched->set_param = sched_table[sched_type].set_param; return sched; +err_destroy_hash: + wd_sched_hash_table_destroy(sched_ctx->domain_hash_table); +ctx_out: + free(sched_ctx); err_out: - wd_sched_rr_release(sched); + free(sched); return NULL; } -- 2.43.0

1 15

[PATCH 01/16] uadk: rewrite scheduler with scheduling domain solution
by Chenghai Huang 25 Aug '26

25 Aug '26

From: Longfang Liu <liulongfang(a)huawei.com> Conduct a comprehensive update of the scheduler's internal triple-array-based pooling scheme. Adopt new data organization methods such as hash buckets and segment linked lists to achieve efficient insertion and query operations. Implement pre-fetch queue processing for business thread sessions, creating thread-level sub-queue pools for fast packet reception queries. Provide new scheduling algorithms for the new UADK framework. Signed-off-by: Longfang Liu <liulongfang(a)huawei.com> Signed-off-by: Chenghai Huang <huangchenghai2(a)huawei.com> Signed-off-by: Wenkai Lin <linwenkai6(a)hisilicon.com> --- include/wd_alg.h | 11 +- include/wd_alg_common.h | 5 +- include/wd_internal.h | 18 +- include/wd_sched.h | 10 +- wd_sched.c | 2304 ++++++++++++++++++++++++++++++--------- 5 files changed, 1832 insertions(+), 516 deletions(-) diff --git a/include/wd_alg.h b/include/wd_alg.h index 18503ca..1ae1dae 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -64,10 +64,13 @@ extern "C" { #endif enum alg_dev_type { - UADK_ALG_SOFT = 0x0, - UADK_ALG_CE_INSTR = 0x1, - UADK_ALG_SVE_INSTR = 0x2, - UADK_ALG_HW = 0x3 + UADK_ALG_HW = 0x0, + UADK_ALG_CE_INSTR = 0x1, + UADK_ALG_SVE_INSTR = 0x2, + UADK_ALG_SOFT = 0x3, + UADK_ALG_NPU = 0x4, + UADK_ALG_GPU = 0x5, + UADK_ALG_TYPE_MAX, }; /* diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index a294877..ebba7f8 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -41,6 +41,7 @@ extern "C" { /* Key size of digest */ #define MAX_HMAC_KEY_SIZE 128U +#define STATUS_ENABLE (void *)0x1 enum alg_task_type { TASK_MIX = 0x0, @@ -140,7 +141,7 @@ struct wd_ctx_params { }; /* - * struct wd_comp_sched - Define a scheduler. + * struct wd_sched - Define a scheduler. * @name: Name of this scheduler. * @sched_policy: Method for scheduler to perform scheduling * @sched_init: inited the scheduler input parameters. @@ -157,10 +158,12 @@ struct wd_sched { const char *name; int sched_policy; handle_t (*sched_init)(handle_t h_sched_ctx, void *sched_param); + void (*sched_uninit)(handle_t h_sched_ctx, handle_t h_sched_key); __u32 (*pick_next_ctx)(handle_t h_sched_ctx, void *sched_key, const int sched_mode); int (*poll_policy)(handle_t h_sched_ctx, __u32 expect, __u32 *count); + void (*set_param)(handle_t h_sched_ctx, void *sched_key, void *sched_param); handle_t h_sched_ctx; }; diff --git a/include/wd_internal.h b/include/wd_internal.h index 4f19d3a..62cf1a9 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -7,8 +7,10 @@ #define WD_INTERNAL_H #include <pthread.h> +#include <stdatomic.h> #include <stdbool.h> #include "wd.h" +#include "wd_alg.h" #ifdef __cplusplus extern "C" { @@ -41,11 +43,14 @@ struct wd_ce_ctx { }; struct wd_ctx_internal { - handle_t ctx; __u8 op_type; __u8 ctx_mode; + __u8 ctx_type; + handle_t ctx; __u16 sqn; pthread_spinlock_t lock; + struct wd_alg_driver *drv; + __u32 hw_load; }; struct wd_ctx_config_internal { @@ -64,6 +69,17 @@ struct wd_datalist { struct wd_datalist *next; }; +struct wd_sched_params { + __u32 pkt_size; + /* block mode or stream mode */ + __u16 data_mode; + __u16 prio_mode; + + /* Compat filtering parameters for session-ctx matching */ + const char *alg_name; + struct wd_ctx_internal *ctxs; +}; + int memcmp_consttime(const void *s1, const void *s2, size_t n); #ifdef __cplusplus diff --git a/include/wd_sched.h b/include/wd_sched.h index 5baecd3..f81025f 100644 --- a/include/wd_sched.h +++ b/include/wd_sched.h @@ -13,7 +13,8 @@ extern "C" { #endif -#define INVALID_POS 0xFFFFFFFF +#define INVALID_POS 0xFFFF +#define QUEUE_FULL_POS 0x1FFF /* The global policy type */ enum sched_policy_type { @@ -25,6 +26,12 @@ enum sched_policy_type { SCHED_POLICY_SINGLE, /* requests will be sent to ctxs and dev_id */ SCHED_POLICY_DEV, + /* Hard calculation and soft calculation interval loop call */ + SCHED_POLICY_LOOP, + /* Perform heterogeneous calculations through ctx of session key */ + SCHED_POLICY_HUNGRY, + /* Instructions to accelerate heterogeneous computing */ + SCHED_POLICY_INSTR, SCHED_POLICY_BUTT, }; @@ -35,6 +42,7 @@ struct sched_params { __u32 begin; __u32 end; __u32 dev_id; + int ctx_prop; }; typedef int (*user_poll_func)(__u32 pos, __u32 expect, __u32 *count); diff --git a/wd_sched.c b/wd_sched.c index 19936fd..2efa2aa 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -1,374 +1,1442 @@ // SPDX-License-Identifier: Apache-2.0 /* - * Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved. * Copyright 2020-2021 Linaro ltd. + * + * Scheduler: Simplified Pure Hash Table with Dynamic Context Expansion + * + * Key improvements: + * - Single global hash table with (region_id, mode, op_type, prop) dimensions + * - Segment list for non-contiguous ctx ranges + * - Dual-domain queues for session key. + * - Dynamic ctx expansion in HUNGRY mode based on load threshold + * - Packet reception is handled through the active queues in the session key. + * - Simplified sched_init: only allocate one sync + one async ctx + * - Removed redundant wd_sched_info layer */ #define _GNU_SOURCE #include <stdlib.h> #include <stdbool.h> +#include <string.h> #include <sched.h> #include <numa.h> +#include <limits.h> +#include <pthread.h> #include "wd_sched.h" +#include "wd_alg.h" +#include "wd_internal.h" -#define MAX_POLL_TIMES 1000 +#define MAX_POLL_TIMES 1000 +#define HUNGRY_LOAD_THRESHOLD 256 +#define SKEY_CTX_MAX_NUM 16 +#define SKEY_MAX_THREAD_NUM 64 +#define SKEY_LOAD_UPDATE_INTERVAL 1 +#define HW_QUEUE_FULL_DEPTH 1024 +#define MAX_NUMA_NODES (NUMA_NUM_NODES >> 5) + +/* ============================================================================ + * Hash Table Configuration + * ============================================================================ + */ +#define WD_SCHED_MAX_BUCKETS 512 +#define WD_SCHED_MIN_BUCKETS 32 +#define WD_SCHED_LOAD_FACTOR 0.75f +#define HASH_PRIME1 73 +#define HASH_PRIME2 13 +#define HASH_PRIME3 7 +#define HASH_PRIME4 11 + +/* ============================================================================ + * Scheduling Region Mode + * ============================================================================ + */ enum sched_region_mode { SCHED_MODE_SYNC = 0, SCHED_MODE_ASYNC = 1, SCHED_MODE_BUTT }; -/* - * sched_key - The key if schedule region. - * @numa_id: The schedule numa region id. - * @mode: Sync mode:0, async_mode:1 - * @type: Service type , the value must smaller than type_num. - * @sync_ctxid: alloc ctx id for sync mode - * @async_ctxid: alloc ctx id for async mode - */ -struct sched_key { - int numa_id; - __u8 type; - __u8 mode; - __u32 sync_ctxid; - __u32 async_ctxid; - __u32 dev_id; -}; +/* ============================================================================ + * Segment List for Domain Index Organization + * ============================================================================ + */ -/* - * struct sched_ctx_range - define one ctx pos. - * @begin: the start pos in ctxs of config. - * @end: the end pos in ctxx of config. - * @last: the last one which be distributed. - * @valid: the region used flag. - * @lock: lock the currentscheduling region. - */ -struct sched_ctx_region { +/** + * wd_sched_ctx_segment - Contiguous segment of ctx indices in domain + * @begin: Start index of this segment + * @end: End index of this segment (inclusive) + * @next: Pointer to next segment in the linked list + * + * Supports non-contiguous ctx ranges via segment list. + */ +struct wd_sched_ctx_segment { __u32 begin; __u32 end; - __u32 last; + struct wd_sched_ctx_segment *next; +}; + +/* ============================================================================ + * Session key domain cache processing. + * ============================================================================ + */ + +/** + * wd_sched_domain_idx_cache - Simplified fixed array cache for skey domains + * + * Design principles: + * - Fixed array for cache-friendly memory layout + * - Atomic operations for lock-free load tracking + * - Simple RR and load balancing strategies + * - Maximum 16 queues per thread (typical usage) + */ +struct wd_sched_domain_idx_cache { + /* Queue index array */ + __u32 idx_list[SKEY_CTX_MAX_NUM]; /* Array of ctx indices */ + __u32 load_values[SKEY_CTX_MAX_NUM]; /* Atomic load counters */ + __u32 valid_count; /* Number of valid queues */ + + /* Scheduling state */ + __u32 rr_ptr; /* Round-robin pointer */ + __u32 min_load_idx; /* Cached min load index */ + __u32 op_counter; /* Operation counter for updates */ + __u8 load_decreased; /* Non-zero if poll ever decremented load */ + + /* Configuration */ + __u32 update_interval; /* Min load update interval */ + __u8 policy; /* Scheduling policy */ + + /* Synchronization */ + pthread_mutex_t cache_lock; /* Lock for structure modifications */ +}; + +/** + * wd_sched_ctx_domain - Scheduling domain with four dimensions + * @region_id: Region identifier (numa_id or device_id) + * @mode: Context mode (SYNC/ASYNC) + * @op_type: Operation type + * @prop: Property (e.g., device type: HW, CE, SOFT) + * @segments: Linked list of context ranges + * @segment_count: Number of segments + * @total_ctx_count: Total contexts across all segments + * @current_segment: Current segment pointer for round-robin + * @current_pos: Current position within segment + * @valid: Domain validity flag + * @lock: Synchronization spinlock + */ +struct wd_sched_ctx_domain { + int region_id; + __u8 mode; + __u32 op_type; + __u8 prop; + + struct wd_sched_ctx_segment *segments; + __u32 segment_count; + __u32 total_ctx_count; + + struct wd_sched_ctx_segment *current_segment; + __u32 current_pos; bool valid; + pthread_mutex_t lock; }; -/* - * wd_sched_info - define the context of the scheduler. - * @ctx_region: define the map for the comp ctxs, using for quickly search. - * the x range: two(sync and async), the y range: - * two(e.g. comp and uncomp) the map[x][y]'s value is the ctx - * begin and end pos. - * @valid: the region used flag. - */ -struct wd_sched_info { - struct sched_ctx_region *ctx_region[SCHED_MODE_BUTT]; - bool valid; +/** + * wd_sched_domain_hash_node - Hash table collision chain node + */ +struct wd_sched_domain_hash_node { + struct wd_sched_ctx_domain domain; + struct wd_sched_domain_hash_node *next; }; -struct dev_region_map { +/** + * wd_sched_domain_hash_table - Pure dynamic hash table for scheduling domains + * @buckets: Hash table bucket array + * @bucket_size: Number of buckets + * @lock: Read-write lock for concurrent access + */ +struct wd_sched_domain_hash_table { + struct wd_sched_domain_hash_node **buckets; + __u32 bucket_size; + pthread_mutex_t lock; +}; + +/* ============================================================================ + * Dual-Domain Structure for Session Key + * ============================================================================ + */ + +/** + * wd_sched_key_domain - Session domain with min-heap + * @idx_cache: Index cache with min-heap for load-based selection + * @lock: Synchronization spinlock + * @expanded_count: Track how many times ctx has been expanded + */ +struct wd_sched_key_domain { + struct wd_sched_domain_idx_cache idx_cache; + pthread_mutex_t lock; + __u32 expanded_count; +}; + +/** + * wd_sched_key - Session-level scheduling key + * @region_id: Region identifier + * @type: Operation type + * @mode: Current mode (SYNC/ASYNC) + * @dev_id: Device identifier (for SCHED_POLICY_DEV) + * @ctx_prop: Context property + * @is_stream: Stream mode flag + * @prio_mode: Priority mode + * @pkt_size: Current packet size + * @sync_domain: Min-heap domain for sync contexts + * @async_domain: Min-heap domain for async contexts + * @lock: Synchronization spinlock + */ +struct wd_sched_key { + int region_id; + __u8 type; + __u8 mode; __u32 dev_id; - __u32 region_id; + __u8 ctx_prop; + __u16 is_stream; + __u16 prio_mode; + __u32 pkt_size; + + struct wd_sched_key_domain sync_domain; + struct wd_sched_key_domain async_domain; + + pthread_mutex_t lock; + __u32 poll_lock; + __u32 refcount; + + /* Compat filtering parameters for session-ctx matching */ + const char *alg_name; + struct wd_ctx_internal *ctxs; }; -/* - * wd_sched_ctx - define the context of the scheduler. - * @policy: define the policy of the scheduler. - * @numa_num: the max numa numbers of the scheduler. - * @type_num: the max operation types of the scheduler. - * @poll_func: the task's poll operation function. - * @numa_map: a map of cpus to devices. - * @sched_info: the context of the scheduler. + +/** + * wd_sched_ctx - Main scheduler context + * @policy: Scheduling policy type + * @type_num: Number of operation types + * @mode_num: Number of modes (SYNC/ASYNC) + * @region_num: Number of regions (numa or devices) + * @poll_func: Poll function for receiving responses + * @domain_hash_table: Global hash table for all domains + * @skey_num: Number of active session keys + * @skey_lock: Lock for skey array + * @skey: Array of session keys */ struct wd_sched_ctx { __u32 policy; __u32 type_num; - __u16 numa_num; - __u16 dev_num; + __u32 mode_num; + __u16 region_num; + user_poll_func poll_func; - int numa_map[NUMA_NUM_NODES]; - struct dev_region_map dev_id_map[DEVICE_REGION_MAX]; - struct wd_sched_info sched_info[0]; + struct wd_sched_domain_hash_table *domain_hash_table; + + __u32 skey_num; + pthread_mutex_t skey_lock; + struct wd_sched_key *skey[SKEY_MAX_THREAD_NUM]; }; -static bool sched_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +/* ============================================================================ + * Hash Table Core Operations + * ============================================================================ + */ + +static bool wd_sched_is_prime(__u32 n) { - if (key->numa_id >= sched_ctx->numa_num || key->mode >= SCHED_MODE_BUTT || - key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's numa: %d, mode: %u, type: %u!\n", - key->numa_id, key->mode, key->type); + __u32 i; + + if (n <= 1) return false; + if (n <= 3) + return true; + if (n % 2 == 0 || n % 3 == 0) + return false; + + for (i = 5; i * i <= n; i += 6) { + if (n % i == 0 || n % (i + 2) == 0) + return false; } return true; } -/* - * sched_get_ctx_range - Get ctx range from ctx_map by the wd comp arg +static __u32 wd_sched_find_prime(__u32 n) +{ + while (!wd_sched_is_prime(n)) + n++; + return n; +} + +static __u32 wd_sched_compute_bucket_size(__u32 estimated_entries) +{ + __u32 target_size; + + target_size = (estimated_entries * 4) / 3; + + if (target_size < WD_SCHED_MIN_BUCKETS) + target_size = WD_SCHED_MIN_BUCKETS; + if (target_size > WD_SCHED_MAX_BUCKETS) + target_size = WD_SCHED_MAX_BUCKETS; + + return wd_sched_find_prime(target_size); +} + +/** + * wd_sched_hash_compute - Compute hash value for four-dimensional domain key + * @region_id: Region identifier + * @mode: Context mode + * @op_type: Operation type + * @prop: Property + * @bucket_size: Hash table bucket count + * + * Combines four dimensions using prime number multipliers. */ -static struct sched_ctx_region *sched_get_ctx_range(struct wd_sched_ctx *sched_ctx, - const struct sched_key *key) +static inline __u32 wd_sched_hash_compute(int region_id, __u8 mode, + __u32 op_type, __u8 prop, __u32 bucket_size) { - struct wd_sched_info *sched_info; - int numa_id; + __u32 hash; - sched_info = sched_ctx->sched_info; - if (key->numa_id >= 0 && - sched_info[key->numa_id].ctx_region[key->mode][key->type].valid) - return &sched_info[key->numa_id].ctx_region[key->mode][key->type]; + hash = (region_id * HASH_PRIME1) + (mode * HASH_PRIME2) + + (op_type * HASH_PRIME3) + (prop * HASH_PRIME4); + return hash % bucket_size; +} + +static inline bool wd_sched_domain_key_match( + int region_id1, __u8 mode1, __u32 op_type1, __u8 prop1, + int region_id2, __u8 mode2, __u32 op_type2, __u8 prop2) +{ + return (region_id1 == region_id2 && mode1 == mode2 && + op_type1 == op_type2 && prop1 == prop2); +} + +/** + * wd_sched_hash_table_create - Create hash table + * @estimated_entries: Estimated number of entries + * + * Returns: Initialized hash table or NULL on error + */ +static struct wd_sched_domain_hash_table * +wd_sched_hash_table_create(__u32 estimated_entries) +{ + struct wd_sched_domain_hash_table *table; + __u32 bucket_size; + int ret; - /* If the key->numa_id is not exist, we should scan for a region */ - for (numa_id = 0; numa_id < sched_ctx->numa_num; numa_id++) { - if (sched_info[numa_id].ctx_region[key->mode][key->type].valid) - return &sched_info[numa_id].ctx_region[key->mode][key->type]; + table = calloc(1, sizeof(*table)); + if (!table) + return NULL; + + bucket_size = wd_sched_compute_bucket_size(estimated_entries); + + table->buckets = calloc(bucket_size, sizeof(*table->buckets)); + if (!table->buckets) { + free(table); + return NULL; } - return NULL; + table->bucket_size = bucket_size; + ret = pthread_mutex_init(&table->lock, NULL); + if (ret) { + free(table->buckets); + free(table); + return NULL; + } + + return table; } -/* - * sched_get_next_pos_rr - Get next resource pos by RR schedule. - * The second para is reserved for future. +static void wd_sched_hash_table_destroy(struct wd_sched_domain_hash_table *table) +{ + struct wd_sched_domain_hash_node *node, *next; + struct wd_sched_ctx_segment *seg, *next_seg; + __u32 i; + + if (!table) + return; + + for (i = 0; i < table->bucket_size; i++) { + node = table->buckets[i]; + while (node) { + next = node->next; + + /* Release segment linked list */ + seg = node->domain.segments; + while (seg) { + next_seg = seg->next; + free(seg); + seg = next_seg; + } + + pthread_mutex_destroy(&node->domain.lock); + free(node); + node = next; + } + } + + pthread_mutex_destroy(&table->lock); + free(table->buckets); + free(table); +} + +static struct wd_sched_ctx_domain * +wd_sched_hash_table_lookup(struct wd_sched_domain_hash_table *table, + int region_id, __u8 mode, __u32 op_type, __u8 prop) +{ + struct wd_sched_domain_hash_node *node; + struct wd_sched_ctx_domain *domain = NULL; + __u32 hash_idx; + + if (!table) + return NULL; + + pthread_mutex_lock(&table->lock); + hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size); + node = table->buckets[hash_idx]; + while (node) { + if (wd_sched_domain_key_match( + node->domain.region_id, node->domain.mode, node->domain.op_type, + node->domain.prop, region_id, mode, op_type, prop)) { + domain = &node->domain; + break; + } + node = node->next; + } + pthread_mutex_unlock(&table->lock); + + return domain; +} + +static struct wd_sched_ctx_domain * +wd_sched_hash_table_insert(struct wd_sched_domain_hash_table *table, + int region_id, __u8 mode, __u32 op_type, __u8 prop) +{ + struct wd_sched_domain_hash_node *new_node; + struct wd_sched_ctx_domain *existing; + __u32 hash_idx; + int ret; + + if (!table) + return NULL; + + existing = wd_sched_hash_table_lookup(table, region_id, mode, op_type, prop); + if (existing) + return existing; + + pthread_mutex_lock(&table->lock); + hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size); + /* Alloc and initialize new domain */ + new_node = calloc(1, sizeof(*new_node)); + if (!new_node) { + pthread_mutex_unlock(&table->lock); + return NULL; + } + + /* Initialize new domain */ + new_node->domain.region_id = region_id; + new_node->domain.mode = mode; + new_node->domain.op_type = op_type; + new_node->domain.prop = prop; + new_node->domain.segments = NULL; + new_node->domain.segment_count = 0; + new_node->domain.total_ctx_count = 0; + new_node->domain.current_segment = NULL; + new_node->domain.current_pos = 0; + new_node->domain.valid = false; + + ret = pthread_mutex_init(&new_node->domain.lock, NULL); + if (ret) { + pthread_mutex_unlock(&table->lock); + free(new_node); + return NULL; + } + + new_node->next = table->buckets[hash_idx]; + table->buckets[hash_idx] = new_node; + pthread_mutex_unlock(&table->lock); + + return &new_node->domain; +} + +/* ============================================================================ + * Segment List Operations + * ============================================================================ + */ + +/** + * wd_sched_domain_add_segment - Add context range segment to domain + * @domain: Target domain + * @begin: Start context index + * @end: End context index (inclusive) + * + * Supports non-contiguous context ranges via segment list. */ -static __u32 sched_get_next_pos_rr(struct sched_ctx_region *region, void *para) +static int wd_sched_domain_add_segment(struct wd_sched_ctx_domain *domain, + __u32 begin, __u32 end) { + struct wd_sched_ctx_segment *seg, *new_seg; + + if (!domain || begin > end) + return -WD_EINVAL; + + new_seg = calloc(1, sizeof(*new_seg)); + if (!new_seg) + return -WD_ENOMEM; + + new_seg->begin = begin; + new_seg->end = end; + new_seg->next = NULL; + + pthread_mutex_lock(&domain->lock); + + /* Append to segment list tail */ + if (!domain->segments) { + domain->segments = new_seg; + } else { + seg = domain->segments; + while (seg->next) + seg = seg->next; + seg->next = new_seg; + } + + domain->segment_count++; + domain->total_ctx_count += (end - begin + 1); + + /* Initialize polling state */ + if (!domain->current_segment) + domain->current_segment = domain->segments; + + pthread_mutex_unlock(&domain->lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_domain_get_next_rr - Get next context via round-robin from domain + * @domain: Source domain + * + * Returns: Next global queue index in round-robin order + * Time complexity: O(1) + */ +static __u32 wd_sched_domain_get_next_rr(struct wd_sched_ctx_domain *domain) +{ + __u32 ctx_idx; __u32 pos; - pthread_mutex_lock(&region->lock); + if (!domain || !domain->segments || !domain->total_ctx_count) + return INVALID_POS; - pos = region->last; + pthread_mutex_lock(&domain->lock); + if (!domain->current_segment) + domain->current_segment = domain->segments; - if (pos < region->end) - region->last++; - else - region->last = region->begin; + pos = domain->current_pos; + + /* Calculate global queue number: segment.begin + relative position */ + ctx_idx = domain->current_segment->begin + pos; - pthread_mutex_unlock(&region->lock); + /* Move to next position */ + if (pos + 1 < domain->current_segment->end - domain->current_segment->begin + 1) { + /* Within same segment */ + domain->current_pos = pos + 1; + } else if (domain->current_segment->next) { + /* Move to next segment */ + domain->current_segment = domain->current_segment->next; + domain->current_pos = 0; + } else { + /* Loop back to beginning */ + domain->current_segment = domain->segments; + domain->current_pos = 0; + } + + pthread_mutex_unlock(&domain->lock); - return pos; + return ctx_idx; } -/* - * session_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/* ============================================================================ + * SKey Domain Cache Management Functions + * ============================================================================ + */ +/** + * wd_sched_skey_cache_init - Initialize skey domain cache + * @cache: Pointer to cache structure + * @policy: Scheduling policy * @sched_type: Scheduling policy type (cannot modify per + * API contract) + * + * Initialize fixed array cache with invalid positions and zero loads. + */ +static int wd_sched_skey_cache_init(struct wd_sched_domain_idx_cache *cache, + __u8 policy) +{ + int i; + + if (!cache) { + WD_ERR("invalid: cache pointer is NULL!\n"); + return -WD_EINVAL; + } + + /* Initialize array with invalid positions */ + for (i = 0; i < SKEY_CTX_MAX_NUM; i++) { + cache->idx_list[i] = INVALID_POS; + __atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED); + } + + /* Initialize atomic counters */ + __atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->op_counter, 0, __ATOMIC_RELAXED); + cache->load_decreased = 0; + + /* Set configuration */ + cache->valid_count = 0; + cache->update_interval = SKEY_LOAD_UPDATE_INTERVAL; + cache->policy = policy; + + /* Initialize structure lock */ + if (pthread_mutex_init(&cache->cache_lock, NULL)) { + WD_ERR("failed to init cache lock!\n"); + return -WD_EINVAL; + } + + return WD_SUCCESS; +} +/** + * wd_sched_skey_cache_uninit - Cleanup skey domain cache + * @cache: Pointer to cache structure + * + * Release resources and reset cache state. + */ +static void wd_sched_skey_cache_uninit(struct wd_sched_domain_idx_cache *cache) +{ + if (!cache) + return; + + pthread_mutex_destroy(&cache->cache_lock); + + /* Reset cache state */ + for (int i = 0; i < SKEY_CTX_MAX_NUM; i++) { + cache->idx_list[i] = INVALID_POS; + __atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED); + } + + cache->valid_count = 0; + cache->load_decreased = 0; +} +/** + * wd_sched_skey_add_ctx - Add ctx to skey domain cache + * @cache: Pointer to cache structure + * @ctx_id: Context ID to add + * + * Add ctx to next available position in fixed array. + * Returns 0 on success, negative error code on failure. + */ +static int wd_sched_skey_add_ctx(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_id) +{ + __u32 i; + + if (!cache || ctx_id == INVALID_POS) { + WD_ERR("invalid: parameters are NULL!\n"); + return -WD_EINVAL; + } + + pthread_mutex_lock(&cache->cache_lock); + /* Check if cache is full */ + if (cache->valid_count >= SKEY_CTX_MAX_NUM) { + pthread_mutex_unlock(&cache->cache_lock); + WD_ERR("invalid: skey cache full, cannot add more queues!\n"); + return -WD_EINVAL; + } + + /* Check for duplicate ctx_id */ + for (i = 0; i < cache->valid_count; i++) { + if (cache->idx_list[i] == ctx_id) { + WD_ERR("invalid: context %u already exists in skey cache at pos %u!\n", + ctx_id, i); + pthread_mutex_unlock(&cache->cache_lock); + return -WD_EEXIST; + } + } + + /* Update min load index if as the new ctx */ + __atomic_store_n(&cache->min_load_idx, cache->valid_count, __ATOMIC_RELAXED); + + /* Add to next available position */ + cache->idx_list[cache->valid_count] = ctx_id; + __atomic_store_n(&cache->load_values[cache->valid_count], 0, __ATOMIC_RELAXED); + cache->valid_count++; + pthread_mutex_unlock(&cache->cache_lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_skey_remove_ctx - Remove ctx from skey domain cache + * @cache: Pointer to cache structure + * @ctx_id: Context ID to remove + * + * Remove ctx by shifting array elements to maintain continuity. + * Returns 0 on success, negative error code if not found. + */ +static int wd_sched_skey_remove_ctx(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_id) +{ + __u32 i, current_min; + int found = 0; + + if (!cache) { + WD_ERR("invalid: cache pointer is NULL!\n"); + return -WD_EINVAL; + } + + pthread_mutex_lock(&cache->cache_lock); + /* Find and remove the ctx */ + for (i = 0; i < cache->valid_count; i++) { + if (cache->idx_list[i] == ctx_id) { + found = 1; + break; + } + } + + if (!found) { + WD_ERR("invalid: context %u not found in skey cache!\n", ctx_id); + pthread_mutex_unlock(&cache->cache_lock); + return -WD_ENODEV; + } + + /* Shift remaining elements to fill the gap */ + for (; i < cache->valid_count - 1; i++) { + cache->idx_list[i] = cache->idx_list[i + 1]; + __atomic_store_n(&cache->load_values[i], + __atomic_load_n(&cache->load_values[i + 1], __ATOMIC_RELAXED), + __ATOMIC_RELAXED); + } + + /* Clear last position */ + cache->idx_list[cache->valid_count - 1] = INVALID_POS; + __atomic_store_n(&cache->load_values[cache->valid_count - 1], 0, __ATOMIC_RELAXED); + cache->valid_count--; + + /* Reset pointers if cache becomes empty */ + if (!cache->valid_count) { + __atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + } else { + /* Adjust min load index if necessary */ + current_min = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED); + if (current_min >= cache->valid_count) + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + } + pthread_mutex_unlock(&cache->cache_lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_update_min_load - Update cached min load index + * @cache: Pointer to cache structure + * @ctxs: Context array for hw_load lookup, or NULL to use cache->load_values * - * The user must init the schedule info through wd_sched_rr_instance + * Scan valid queues to find the one with minimum load. + * When ctxs is provided, reads per-ctx hw_load for cross-session accuracy. */ -static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, - const int sched_mode) +static void wd_sched_update_min_load(struct wd_sched_domain_idx_cache *cache, + struct wd_ctx_internal *ctxs) { - struct sched_ctx_region *region = NULL; - bool ret; + __u32 min_load = UINT_MAX; + __u32 min_idx = 0; + __u32 i, load; - key->mode = sched_mode; - ret = sched_key_valid(sched_ctx, key); - if (!ret) + if (!cache->valid_count) + return; + + for (i = 0; i < cache->valid_count; i++) { + if (ctxs) + load = __atomic_load_n(&ctxs[cache->idx_list[i]].hw_load, + __ATOMIC_RELAXED); + else + load = __atomic_load_n(&cache->load_values[i], __ATOMIC_RELAXED); + if (load < min_load) { + min_load = load; + min_idx = i; + } + } + + __atomic_store_n(&cache->min_load_idx, min_idx, __ATOMIC_RELAXED); +} + +/** + * wd_sched_skey_pick_next - Pick next ctx from skey domain cache + * @cache: Pointer to cache structure + * @ctx_idx: Output index within cache array + * @ctxs: Context array for hw_load lookup, or NULL + * + * Select next ctx based on scheduling policy: + * - RR: Simple round-robin selection + * - HUNGRY: Choose ctx with minimum load (hw_load if ctxs available) + * + * Returns selected ctx index, or INVALID_POS if no valid ctx. + */ +static __u32 wd_sched_skey_pick_next(struct wd_sched_domain_idx_cache *cache, + __u32 *ctx_idx, struct wd_ctx_internal *ctxs) +{ + __u32 selected_idx; + __u32 op_count; + + if (!cache || !cache->valid_count) return INVALID_POS; - region = sched_get_ctx_range(sched_ctx, key); - if (!region) + switch (cache->policy) { + case SCHED_POLICY_RR: + case SCHED_POLICY_NONE: + case SCHED_POLICY_SINGLE: + case SCHED_POLICY_DEV: + case SCHED_POLICY_LOOP: + case SCHED_POLICY_INSTR: + /* Round-robin: atomic increment and module */ + selected_idx = __atomic_fetch_add(&cache->rr_ptr, 1, __ATOMIC_RELAXED) % + cache->valid_count; + break; + case SCHED_POLICY_HUNGRY: + /* Update min load periodically */ + op_count = __atomic_fetch_add(&cache->op_counter, 1, __ATOMIC_RELAXED); + if (op_count % cache->update_interval == 0) + wd_sched_update_min_load(cache, ctxs); + + /* Load balancing: use cached min load index */ + selected_idx = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED); + break; + default: + WD_ERR("invalid: unknown scheduling policy %d!\n", cache->policy); + selected_idx = INVALID_POS; + break; + } + + /* Ensure index is within valid range */ + if (selected_idx >= cache->valid_count) return INVALID_POS; - return sched_get_next_pos_rr(region, NULL); + *ctx_idx = selected_idx; + return cache->idx_list[selected_idx]; } -static handle_t session_sched_init(handle_t h_sched_ctx, void *sched_param) +/** + * wd_sched_skey_update_load - Update load for a specific ctx + * @cache: Pointer to cache structure + * @ctx_idx: Context index in list + * @delta: Load delta (positive for send, negative for receive) + * + * Atomically update load counter for the specified ctx. + * Returns 0 on success, negative error code if ctx not found. + */ +static int wd_sched_skey_update_load(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_idx, int delta) { - struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - struct sched_params *param = (struct sched_params *)sched_param; - struct sched_key *skey; + /* Atomic update without locking, ctx_idx's value is guaranteed by the caller. */ + if (delta > 0) + __atomic_fetch_add(&cache->load_values[ctx_idx], delta, __ATOMIC_RELAXED); + else { + __atomic_fetch_sub(&cache->load_values[ctx_idx], -delta, __ATOMIC_RELAXED); + if (unlikely(!cache->load_decreased)) + cache->load_decreased = 1; + } + return WD_SUCCESS; +} + +/* ============================================================================ + * Session Key Domain Initialization + * ============================================================================ + */ + +/** + * wd_sched_skey_domain_init - Initialize session domain with min-heap + * @key_domain: Target key domain + * @ctx_idx: context indices idx + * @policy: current session's policy + * + * Initializes dual-domain structure for session. + */ +static int wd_sched_skey_domain_init(struct wd_sched_key_domain *key_domain, + __u32 ctx_idx, __u8 policy) +{ + int ret; + + if (!key_domain) + return -WD_EINVAL; + + ret = wd_sched_skey_cache_init(&key_domain->idx_cache, policy); + if (ret) + return ret; + + ret = wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx); + if (ret) + goto init_err; + + ret = pthread_mutex_init(&key_domain->lock, NULL); + if (ret) + goto add_ctx_err; + + key_domain->expanded_count = 0; + + return WD_SUCCESS; + +add_ctx_err: + wd_sched_skey_remove_ctx(&key_domain->idx_cache, ctx_idx); +init_err: + wd_sched_skey_cache_uninit(&key_domain->idx_cache); + return ret; +} + +/** + * wd_sched_skey_domain_destroy - Release session domain resources + */ +static void wd_sched_skey_domain_destroy(struct wd_sched_key_domain *key_domain) +{ + if (!key_domain) + return; + + pthread_mutex_destroy(&key_domain->lock); + wd_sched_skey_cache_uninit(&key_domain->idx_cache); +} + + +/** + * wd_sched_poll_skey - Poll contexts for scheduler session + * @sched_ctx: Scheduler context + * @skey: Session key + * @expect: Expected number of responses + * @count: Actual response count (output) + * + * Polls all contexts in session domains and updates load values. + */ +static int wd_sched_poll_skey(struct wd_sched_ctx *sched_ctx, struct wd_sched_key *skey, + __u32 expect, __u32 *count) +{ + struct wd_sched_domain_idx_cache *cache; + __u32 ctx_list_num = 0; + __u32 sum_poll_num = 0; + bool hungry_policy; + __u32 poll_num; + __u32 idx, i; + int ret = 0; + + /* Get cache pointer and check if HUNGRY policy */ + cache = &skey->async_domain.idx_cache; + ctx_list_num = cache->valid_count; + hungry_policy = (cache->policy == SCHED_POLICY_HUNGRY); + + /* Poll async domain contexts */ + for (i = 0; i < ctx_list_num; i++) { + idx = cache->idx_list[i]; + if (idx == INVALID_POS) + continue; + poll_num = 0; + ret = sched_ctx->poll_func(idx, expect, &poll_num); + if (poll_num > 0) + sum_poll_num += poll_num; + + /* Update load value for this context */ + if (hungry_policy && poll_num > 0) { + if (skey->ctxs) + __atomic_fetch_sub(&skey->ctxs[idx].hw_load, + poll_num, __ATOMIC_RELAXED); + wd_sched_skey_update_load(cache, i, -poll_num); + } + + if (ret < 0 && ret != -WD_EAGAIN) + break; + } + *count = sum_poll_num; + + return ret; +} + +/* ============================================================================ + * Utility Functions + * ============================================================================ + */ +static inline bool sched_skey_get_ref(struct wd_sched_key *skey) +{ + if (!skey) + return false; + return __atomic_fetch_add(&skey->refcount, 1, __ATOMIC_ACQUIRE) > 0; +} + +static inline bool sched_skey_put_ref(struct wd_sched_key *skey) +{ + return __atomic_fetch_sub(&skey->refcount, 1, __ATOMIC_RELEASE) == 1; +} + +static int sched_skey_param_init(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey) +{ + __u32 i; + + pthread_mutex_lock(&sched_ctx->skey_lock); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + if (!sched_ctx->skey[i]) { + sched_ctx->skey[i] = skey; + if (sched_ctx->skey_num < SKEY_MAX_THREAD_NUM) + sched_ctx->skey_num++; + pthread_mutex_unlock(&sched_ctx->skey_lock); + return 0; + } + } + pthread_mutex_unlock(&sched_ctx->skey_lock); + WD_ERR("invalid: skey node number exceeds SKEY_MAX_THREAD_NUM(%d)!\n", + SKEY_MAX_THREAD_NUM); + return -WD_ENOMEM; +} + +static void sched_skey_param_uninit(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey) +{ + __u32 i; + + if (!sched_ctx || !skey) + return; + + pthread_mutex_lock(&sched_ctx->skey_lock); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + if (sched_ctx->skey[i] == skey) { + sched_ctx->skey[i] = NULL; + pthread_mutex_unlock(&sched_ctx->skey_lock); + return; + } + } + pthread_mutex_unlock(&sched_ctx->skey_lock); + WD_ERR("warning: skey %p not found in sched_ctx array\n", skey); +} + +static handle_t sched_session_common_init(struct wd_sched_ctx *sched_ctx, + struct sched_params *param) +{ + struct wd_sched_key *skey; unsigned int node; if (getcpu(NULL, &node)) { WD_ERR("failed to get node, errno %d!\n", errno); return (handle_t)(-errno); } - if (node == (unsigned int)NUMA_NO_NODE) { - WD_ERR("invalid: failed to get numa node!\n"); - return (handle_t)(-WD_EINVAL); - } if (!sched_ctx) { WD_ERR("invalid: sched ctx is NULL!\n"); return (handle_t)(-WD_EINVAL); } - skey = malloc(sizeof(struct sched_key)); + skey = malloc(sizeof(struct wd_sched_key)); if (!skey) { WD_ERR("failed to alloc memory for session sched key!\n"); return (handle_t)(-WD_ENOMEM); } + memset(skey, 0, sizeof(struct wd_sched_key)); if (!param) { - memset(skey, 0, sizeof(struct sched_key)); - skey->numa_id = sched_ctx->numa_map[node]; + skey->region_id = node; if (wd_need_debug()) WD_DEBUG("session don't set scheduler parameters!\n"); - } else if (param->numa_id < 0) { - skey->type = param->type; - skey->numa_id = sched_ctx->numa_map[node]; } else { + if (sched_ctx->policy == SCHED_POLICY_DEV) + skey->region_id = param->dev_id; + else if (param->numa_id >= 0) + skey->region_id = param->numa_id; + else + skey->region_id = node; skey->type = param->type; - skey->numa_id = param->numa_id; + skey->ctx_prop = param->ctx_prop; } + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + __atomic_store_n(&skey->refcount, 1, __ATOMIC_RELAXED); - if (skey->numa_id < 0) { - WD_ERR("failed to get valid sched numa region!\n"); - goto out; - } + return (handle_t)skey; +} - skey->sync_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); - skey->async_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); - if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { - WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); - goto out; +static __u16 sched_get_poll_skey_idx(struct wd_sched_ctx *sched_ctx) +{ + /* Thread-local sequence number with atomic global counter */ + static __thread __u32 thread_seq = UINT32_MAX; + static __u32 global_seq_counter; + __u32 skey_num = sched_ctx->skey_num; + __u16 start_pos; + + if (unlikely(!sched_ctx || !skey_num)) + return skey_num; + + /* Assign unique sequence number on first call */ + if (unlikely(thread_seq == UINT32_MAX)) { + thread_seq = __atomic_fetch_add(&global_seq_counter, 1, __ATOMIC_RELAXED); + + /* Basic overflow protection */ + if (thread_seq >= UINT32_MAX - 1) + WD_DEBUG("Thread sequence counter approaching limit: %u\n", thread_seq); } - return (handle_t)skey; + /* Calculate start_pos based on thread_seq */ + start_pos = thread_seq % skey_num; -out: - free(skey); - return (handle_t)(-WD_EINVAL); + return start_pos; } -/* - * session_pick_next_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/** + * session_sched_init_ctx - Pre-fetch single context from domain for session + * @sched_ctx: Scheduler context + * @skey: session scheduler param + * @sched_mode: Mode (SYNC/ASYNC) * - * The user must init the schedule info through session_sched_init + * Returns: Context index from domain (compatible with alg_name if skey provided) */ -static __u32 session_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, - const int sched_mode) +static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, int sched_mode) { - struct sched_key *key = (struct sched_key *)sched_key; + struct wd_sched_ctx_domain *domain = NULL; + int region_id = skey->region_id; + __u32 op_type = skey->type; + __u8 prop = skey->ctx_prop; + __u32 ctx_idx; + __u16 r; + + if (sched_mode >= SCHED_MODE_BUTT || + op_type >= sched_ctx->type_num || prop >= UADK_ALG_TYPE_MAX) { + WD_ERR("invalid: region: %d, mode: %d, type: %u!, prop: %u\n", + region_id, sched_mode, op_type, prop); + return INVALID_POS; + } - if (unlikely(!h_sched_ctx || !key)) { - WD_ERR("invalid: sched ctx or key is NULL!\n"); + if (!sched_ctx->domain_hash_table) + return INVALID_POS; + + /* Try current region first */ + if (region_id >= 0) { + if (sched_ctx->policy == SCHED_POLICY_DEV || + region_id < sched_ctx->region_num) { + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + region_id, sched_mode, op_type, prop); + if (domain && domain->valid) + return wd_sched_domain_get_next_rr(domain); + } + } + + /* DEV policy must not cross region */ + if (sched_ctx->policy == SCHED_POLICY_DEV) return INVALID_POS; + + /* Cross-region fallback for other policies */ + for (r = 0; r < sched_ctx->region_num; r++) { + if ((int)r == region_id) + continue; + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + r, sched_mode, op_type, prop); + if (domain && domain->valid) { + ctx_idx = wd_sched_domain_get_next_rr(domain); + if (ctx_idx != INVALID_POS) + return ctx_idx; + } } - /* return in do task */ - if (sched_mode == CTX_MODE_SYNC) - return key->sync_ctxid; - return key->async_ctxid; + return INVALID_POS; } -static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, - __u32 end, __u32 expect, __u32 *count) +/** + * session_sched_domain_destroy - Destroy session domains + * @skey: Session key to destroy domains for + * + * Releases all resources associated with session domains. + */ +static void session_sched_domain_destroy(struct wd_sched_key *skey) { - __u32 poll_num = 0; - __u32 i; - int ret; + if (!skey) + return; - /* i is the pos of sched_ctxs, the max is end */ - for (i = begin; i <= end; i++) { - /* - * RR schedule, one time poll one package, - * poll_num is always not more than one here. - */ - ret = sched_ctx->poll_func(i, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) - return ret; - else if (ret == -EAGAIN) + /* Destroy both sync and async domains */ + wd_sched_skey_domain_destroy(&skey->sync_domain); + wd_sched_skey_domain_destroy(&skey->async_domain); +} + +static inline void sched_skey_poll_release(struct wd_sched_key *skey) +{ + __u32 prev; + + if (!skey) + return; + + prev = __atomic_fetch_sub(&skey->refcount, 1, __ATOMIC_ACQ_REL); + if (prev == 1) { + session_sched_domain_destroy(skey); + free(skey); + } +} + +static void sched_session_uninit(handle_t h_sched_ctx, handle_t h_sched_key) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_key *skey = (struct wd_sched_key *)h_sched_key; + + if (!skey) + return; + + /* Remove from skey array first to prevent new poll discovery. + * Poll threads already holding a pointer are protected by refcount. + */ + if (sched_ctx) + sched_skey_param_uninit(sched_ctx, skey); + + /* Drop creator reference. If no poll holds a ref, free now. + * Otherwise the last poll thread will cleanup via poll_release. + */ + if (sched_skey_put_ref(skey)) { + session_sched_domain_destroy(skey); + free(skey); + } +} + +/** + * session_sched_init_ctx_with_fallback - Pre-fetch a ctx for one sched_mode, + * trying user_prop first then falling + * back through other props in enum order. + * @sched_ctx: Scheduler context + * @skey: Session key (skey->ctx_prop is the user-specified prop; restored on return) + * @sched_mode: SCHED_MODE_SYNC or SCHED_MODE_ASYNC + * + * Returns: ctx index, or INVALID_POS if no prop has a usable ctx. + * + * Falls back only when the user-specified prop's domain is unavailable + * (e.g. HW driver unloaded). Stops at the first prop that yields a ctx; + * does NOT accumulate ctxs across props. + */ +static __u32 session_sched_init_ctx_with_fallback(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + int sched_mode) +{ + __u8 user_prop = skey->ctx_prop; + __u32 ctx_idx; + __u8 p; + + /* Try user-specified prop first */ + ctx_idx = session_sched_init_ctx(sched_ctx, skey, sched_mode); + if (ctx_idx != INVALID_POS) + return ctx_idx; + + /* Fallback: target prop unavailable, try remaining props in enum order */ + for (p = 0; p < UADK_ALG_TYPE_MAX; p++) { + if (p == user_prop) continue; - *count += poll_num; - if (*count == expect) + skey->ctx_prop = p; + ctx_idx = session_sched_init_ctx(sched_ctx, skey, sched_mode); + if (ctx_idx != INVALID_POS) break; } - return 0; + /* Restore user_prop so skey state is not mutated */ + skey->ctx_prop = user_prop; + return ctx_idx; } -static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, - __u32 expect, __u32 *count) +/** + * session_sched_domain_init - Initialize session domains with sync/async + * @sched_ctx: Scheduler context + * @skey: Session key to initialize + * @allow_fallback: If true, fall back to other props when skey->ctx_prop has + * no usable domain; if false, only try skey->ctx_prop. + * + * Pre-fetches sync and async contexts and initializes corresponding domains. + * Returns: 0 on success, negative error code on failure. + */ +static int session_sched_domain_init(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + bool allow_fallback) { - struct sched_ctx_region **region = sched_ctx->sched_info[numa_id].ctx_region; - __u32 begin, end; - __u32 i; - int ret; + __u32 sync_ctx, async_ctx; - for (i = 0; i < sched_ctx->type_num; i++) { - if (!region[SCHED_MODE_ASYNC][i].valid) - continue; + if (!sched_ctx || !skey) { + WD_ERR("invalid: sched_ctx or skey is NULL!\n"); + return -WD_EINVAL; + } + + if (allow_fallback) { + /* Pre-fetch with per-mode prop fallback */ + sync_ctx = session_sched_init_ctx_with_fallback(sched_ctx, skey, + SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx_with_fallback(sched_ctx, skey, + SCHED_MODE_ASYNC); + } else { + sync_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_ASYNC); + } + + if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) { + WD_ERR("invalid: no valid sync_ctx or async_ctx domain!\n"); + return -WD_EINVAL; + } + + /* Initialize sync domain if context is valid */ + if (sync_ctx != INVALID_POS) { + if (wd_sched_skey_domain_init(&skey->sync_domain, sync_ctx, sched_ctx->policy)) { + WD_ERR("failed to init sync domain!\n"); + return -WD_EINVAL; + } + } + + /* Initialize async domain if context is valid */ + if (async_ctx != INVALID_POS) { + if (wd_sched_skey_domain_init(&skey->async_domain, async_ctx, sched_ctx->policy)) { + WD_ERR("failed to init async domain!\n"); + /* Cleanup sync domain if async domain init failed */ + if (sync_ctx != INVALID_POS) + wd_sched_skey_domain_destroy(&skey->sync_domain); + return -WD_EINVAL; + } + } + + return WD_SUCCESS; +} + +/* ============================================================================ + * Scheduler Policy Functions + * ============================================================================ + */ +/** + * round_robin_sched_init - Initialize session with single sync and async ctx + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_param: Scheduling parameters (cannot modify per API contract) + * + * Allocates session key and pre-fetches one sync and one async context. + */ +static handle_t round_robin_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + struct wd_sched_key *skey; + handle_t hskey; + int ret = 0; + + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; + } + + skey = (struct wd_sched_key *)hskey; + /* RR: allow prop fallback so session creation survives target prop unload */ + ret = session_sched_domain_init(sched_ctx, skey, true); + if (ret) { + WD_ERR("failed to initialize session domains!\n"); + free(skey); + return (handle_t)(-WD_EINVAL); + } + + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); + return (handle_t)(-WD_ENOMEM); + } + + return hskey; +} + +/** + * round_robin_pick_next_ctx - Pick context with load-based selection + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) + * + * Returns: Context index with minimum load + * Time complexity: O(1) + */ +static __u32 round_robin_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_key_domain *domain; + __u32 min_ctx, ctx_idx; - begin = region[SCHED_MODE_ASYNC][i].begin; - end = region[SCHED_MODE_ASYNC][i].end; - ret = session_poll_region(sched_ctx, begin, end, expect, count); - if (unlikely(ret)) - return ret; + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; } - return 0; + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; + + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) + return INVALID_POS; + + return min_ctx; } -/* - * session_poll_policy - The polling policy matches the pick next ctx. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @cfg: The global resoure info. - * @expect: User expect poll msg num. - * @count: The actually poll num. +/** + * round_robin_poll_policy - Poll policy for session scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) * - * The user must init the schedule info through wd_sched_rr_instance, the - * func interval will not check the valid, becouse it will affect performance. + * Returns: Status code */ -static int session_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +static int round_robin_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - struct wd_sched_info *sched_info; - __u32 loop_time = 0; - __u32 last_count = 0; - __u16 i, region_mum; - int ret; + struct wd_sched_key *skey; + __u16 i, tpos, start_pos; + __u32 poll_num, skey_num; + int ret = -WD_EAGAIN; + __u32 sum_count = 0; if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); return -WD_EINVAL; } - if (unlikely(sched_ctx->numa_num > NUMA_NUM_NODES)) { - WD_ERR("invalid: ctx's numa number is %u!\n", sched_ctx->numa_num); - return -WD_EINVAL; + /* Randomize the initial query position. */ + skey_num = sched_ctx->skey_num; + start_pos = sched_get_poll_skey_idx(sched_ctx); + if (!skey_num || start_pos >= sched_ctx->skey_num) { + *count = 0; + return -WD_EAGAIN; } - sched_info = sched_ctx->sched_info; - if (sched_ctx->policy == SCHED_POLICY_DEV) - region_mum = sched_ctx->dev_num; - else - region_mum = sched_ctx->numa_num; - - /* - * Try different region's ctx if we can't receive any - * package last time, it is more efficient. In most - * bad situation, poll ends after MAX_POLL_TIMES loop. - */ - while (++loop_time < MAX_POLL_TIMES) { - for (i = 0; i < region_mum;) { - /* If current numa is not valid, find next. */ - if (!sched_info[i].valid) { - i++; - continue; - } + /* Query the queues on each skey separately. */ + for (i = 0; i < skey_num; i++) { + tpos = (start_pos + i) % skey_num; + skey = sched_ctx->skey[tpos]; - last_count = *count; - ret = session_poll_policy_rr(sched_ctx, i, expect, count); - if (unlikely(ret)) - return ret; + if (unlikely(!skey)) + continue; - if (expect == *count) - return 0; + if (!sched_skey_get_ref(skey)) + continue; - /* - * If no package is received, find next numa, - * otherwise, keep receiving packets at this node. - */ - if (last_count == *count) - i++; + if (__atomic_test_and_set(&skey->poll_lock, __ATOMIC_ACQUIRE)) { + sched_skey_poll_release(skey); + continue; } + + ret = wd_sched_poll_skey(sched_ctx, skey, expect, &poll_num); + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + sched_skey_poll_release(skey); + + sum_count += poll_num; + if (unlikely(ret && ret != -WD_EAGAIN)) + goto poll_err; + + if (sum_count >= expect) + break; } - return 0; +poll_err: + *count = sum_count; + if (ret == -WD_EAGAIN) + return 0; + return ret; } static handle_t sched_none_init(handle_t h_sched_ctx, void *sched_param) @@ -396,12 +1464,11 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, } while (loop_times > 0) { - /* Default use ctx 0 */ loop_times--; ret = sched_ctx->poll_func(0, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) + if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; - else if (ret == -EAGAIN) + else if (ret == -WD_EAGAIN) continue; *count += poll_num; @@ -409,7 +1476,7 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, break; } - return 0; + return WD_SUCCESS; } static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) @@ -420,13 +1487,10 @@ static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) static __u32 sched_single_pick_next_ctx(handle_t sched_ctx, void *sched_key, const int sched_mode) { -#define CTX_ASYNC 1 -#define CTX_SYNC 0 - if (sched_mode) - return CTX_ASYNC; + return 1; else - return CTX_SYNC; + return 0; } static int sched_single_poll_policy(handle_t h_sched_ctx, @@ -443,12 +1507,11 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, } while (loop_times > 0) { - /* Default async mode use ctx 1 */ loop_times--; ret = sched_ctx->poll_func(1, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) + if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; - else if (ret == -EAGAIN) + else if (ret == -WD_EAGAIN) continue; *count += poll_num; @@ -456,260 +1519,483 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, break; } - return 0; + return WD_SUCCESS; } -static bool sched_dev_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +/** + * sched_skey_domain_fill - Initialize or append ctx to session domain + * @key_domain: Target domain (sync or async) + * @ctx_idx: Context index to add + * @policy: Scheduler policy + * @inited: Pointer to boolean tracking whether domain has been initialized + * + * First call: full domain init via wd_sched_skey_domain_init(). + * Subsequent calls: append via wd_sched_skey_add_ctx(). + */ +static void sched_skey_domain_fill(struct wd_sched_key_domain *key_domain, + __u32 ctx_idx, __u8 policy, bool *inited) { - bool found = false; - int i; + int ret; - if (key->mode >= SCHED_MODE_BUTT || key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's device id: %u, mode: %u, type: %u!\n", - key->dev_id, key->mode, key->type); - return false; + if (ctx_idx == INVALID_POS) + return; + + if (!*inited) { + ret = wd_sched_skey_domain_init(key_domain, ctx_idx, policy); + if (!ret) + *inited = true; + return; } - for (i = 0; i < sched_ctx->dev_num; i++) { - if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { - found = true; - break; + (void)wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx); +} + +/** + * sched_skey_common_init - Common scheduler init with init-once + add-rest pattern + * @h_sched_ctx: Scheduler handle + * @sched_param: Scheduling parameters + * @prop_begin: First prop type to iterate (inclusive) + * @prop_end: Last prop type to iterate (inclusive) + * + * Shared by Loop, Hungry, and Instr scheduler init functions. + * Iterates [prop_begin, prop_end], fetching sync/async ctxs per prop type. + * First valid ctx initializes the domain, subsequent ctxs are appended. + */ +static handle_t sched_skey_common_init(handle_t h_sched_ctx, void *sched_param, + __u8 prop_begin, __u8 prop_end) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + struct wd_sched_key *skey; + bool sync_inited = false; + bool async_inited = false; + __u32 sync_ctx, async_ctx; + __u32 req_ctx_num = 0; + handle_t hskey; + __u8 def_prop; + int ret, i; + + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; + } + + skey = (struct wd_sched_key *)hskey; + def_prop = skey->ctx_prop; + for (i = prop_begin; i <= prop_end; i++) { + skey->ctx_prop = i; + sync_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_ASYNC); + if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) + continue; + + if (sync_ctx != INVALID_POS) { + sched_skey_domain_fill(&skey->sync_domain, sync_ctx, + sched_ctx->policy, &sync_inited); } + if (async_ctx != INVALID_POS) + sched_skey_domain_fill(&skey->async_domain, async_ctx, + sched_ctx->policy, &async_inited); + + req_ctx_num += 2; + } + if (!req_ctx_num) { + free(skey); + return (handle_t)(-WD_EINVAL); } - if (!found) { - WD_ERR("invalid: dev_id %u is not registered!\n", key->dev_id); - return false; + skey->ctx_prop = def_prop; + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); + return (handle_t)(-WD_ENOMEM); } - return true; + return hskey; } -/* - * sched_dev_get_region - Get ctx region from ctx_map by the wd comp arg +/** + * skey_sched_init - Initialize Hungry scheduler session + */ +static handle_t skey_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + 0, UADK_ALG_TYPE_MAX - 1); + return hskey; +} + +/** + * skey_sched_pick_next_ctx - Pick context from hungry scheduler with load awareness + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) + * + * Returns: Context with minimum load, or expands if threshold exceeded + * Time complexity: O(1) for selection, O(n) if expansion needed */ -static struct sched_ctx_region *sched_dev_get_region(struct wd_sched_ctx *sched_ctx, - const struct sched_key *key) +static __u32 skey_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) { - struct wd_sched_info *sched_info; - int i, region_id; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_domain_idx_cache *idx_cache; + struct wd_sched_key_domain *domain; + __u32 min_ctx, min_load, ctx_idx; - for (i = 0; i < sched_ctx->dev_num; i++) { - if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { - region_id = sched_ctx->dev_id_map[i].region_id; - sched_info = &sched_ctx->sched_info[region_id]; - if (sched_info->ctx_region[key->mode][key->type].valid) - return &sched_info->ctx_region[key->mode][key->type]; + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; + } + + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; + + idx_cache = &domain->idx_cache; + if (sched_mode == SCHED_MODE_SYNC) { + /* Sync: send+recv atomic, spinlock serialized, use RR */ + if (!idx_cache->valid_count) + return INVALID_POS; + + ctx_idx = __atomic_fetch_add(&idx_cache->rr_ptr, 1, __ATOMIC_RELAXED) % + idx_cache->valid_count; + return idx_cache->idx_list[ctx_idx]; + } + + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) + return INVALID_POS; + + if (skey->ctxs) { + min_load = __atomic_load_n(&skey->ctxs[min_ctx].hw_load, __ATOMIC_RELAXED); + if (min_load >= HW_QUEUE_FULL_DEPTH) + return QUEUE_FULL_POS; + __atomic_fetch_add(&skey->ctxs[min_ctx].hw_load, 1, __ATOMIC_RELAXED); + wd_sched_skey_update_load(idx_cache, ctx_idx, 1); + } else { + min_load = __atomic_load_n(&idx_cache->load_values[ctx_idx], __ATOMIC_RELAXED); + if (min_load >= HW_QUEUE_FULL_DEPTH && idx_cache->load_decreased > 0) + return QUEUE_FULL_POS; + wd_sched_skey_update_load(idx_cache, ctx_idx, 1); + } + /* Check if we need to expand context pool */ + + return min_ctx; +} + +/** + * skey_sched_poll_policy - Poll policy for hungry scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) + * + * Returns: Status code + */ +static int skey_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_key *skey; + __u16 i, tpos, start_pos; + __u32 poll_num, skey_num; + int ret = -WD_EAGAIN; + __u32 sum_count = 0; + + if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { + WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); + return -WD_EINVAL; + } + + /* Randomize the initial query position. */ + skey_num = sched_ctx->skey_num; + start_pos = sched_get_poll_skey_idx(sched_ctx); + if (!skey_num || start_pos >= sched_ctx->skey_num) { + *count = 0; + return -WD_EAGAIN; + } + + /* Query the queues on each skey separately. */ + for (i = 0; i < skey_num; i++) { + tpos = (start_pos + i) % skey_num; + skey = sched_ctx->skey[tpos]; + + if (unlikely(!skey)) + continue; + + if (!sched_skey_get_ref(skey)) + continue; + + if (__atomic_test_and_set(&skey->poll_lock, __ATOMIC_ACQUIRE)) { + sched_skey_poll_release(skey); + continue; } + + ret = wd_sched_poll_skey(sched_ctx, skey, expect, &poll_num); + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + sched_skey_poll_release(skey); + /* + * This query returned 0, indicating the hardware + * likely hasn't finished processing yet. + * Implementing a delay and releasing the CPU is a predictive optimization + */ + if (!poll_num) + usleep(1); + + sum_count += poll_num; + if (ret == -WD_EAGAIN) + continue; + if (unlikely(ret) || sum_count >= expect) + break; } + *count = sum_count; - /* - * If the scheduling domain of dev_id does not exist, - * taskes operations cannot be executed using queues from other devices; - * otherwise, an SMMU error will occur. - */ - return NULL; + return ret; } -/* - * session_dev_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/** + * loop_sched_init - Initialize Loop scheduler session + */ +static handle_t loop_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + 0, UADK_ALG_TYPE_MAX - 1); + return hskey; +} + +/** + * loop_sched_pick_next_ctx - Pick context for loop scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) * - * The user must init the schedule info through wd_sched_rr_instance + * Returns: Context index with minimum load + * Time complexity: O(1) + */ +static __u32 loop_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + return round_robin_pick_next_ctx(h_sched_ctx, sched_key, sched_mode); +} + +static int loop_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + return round_robin_poll_policy(h_sched_ctx, expect, count); +} + +/** + * instr_sched_init - Initialize Instr scheduler session */ -static __u32 session_dev_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, - const int sched_mode) +static handle_t instr_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + UADK_ALG_CE_INSTR, + UADK_ALG_SVE_INSTR); + return hskey; +} + +static __u32 instr_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) { - struct sched_ctx_region *region = NULL; - bool ret; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_key_domain *domain; + __u32 min_ctx, ctx_idx; - key->mode = sched_mode; - ret = sched_dev_key_valid(sched_ctx, key); - if (!ret) + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); return INVALID_POS; + } + + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; - region = sched_dev_get_region(sched_ctx, key); - if (!region) + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) return INVALID_POS; - return sched_get_next_pos_rr(region, NULL); + return min_ctx; +} + +/** + * instr_sched_poll_policy - Poll policy for instruction scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) + * + * Returns: Status code + */ +static int instr_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + return round_robin_poll_policy(h_sched_ctx, expect, count); } static handle_t session_dev_sched_init(handle_t h_sched_ctx, void *sched_param) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; struct sched_params *param = (struct sched_params *)sched_param; - struct sched_key *skey; - unsigned int node; + struct wd_sched_key *skey; + handle_t hskey; + int ret = 0; - if (getcpu(NULL, &node)) { - WD_ERR("failed to get numa node, errno %d!\n", errno); - return (handle_t)(-errno); - } - if (node == (unsigned int)NUMA_NO_NODE) { - WD_ERR("invalid: failed to get numa node for dev sched init!\n"); + if (!param) { + WD_ERR("invalid: dev sched param is NULL!\n"); return (handle_t)(-WD_EINVAL); } - if (!sched_ctx) { - WD_ERR("invalid: sched ctx is NULL!\n"); - return (handle_t)(-WD_EINVAL); + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; } - if (!param) { - WD_DEBUG("no-sva session don't set scheduler parameters!\n"); + skey = (struct wd_sched_key *)hskey; + skey->type = param->type; + skey->dev_id = param->dev_id; + + /* DEV: pinned to a specific device, do not cross prop families */ + ret = session_sched_domain_init(sched_ctx, skey, false); + if (ret) { + WD_ERR("failed to initialize session domains!\n"); + free(skey); return (handle_t)(-WD_EINVAL); } - skey = malloc(sizeof(struct sched_key)); - if (!skey) { - WD_ERR("failed to alloc memory for session sched key!\n"); + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); return (handle_t)(-WD_ENOMEM); } - skey->type = param->type; - skey->dev_id = param->dev_id; + return (handle_t)skey; +} + +/** + * wd_sched_set_param - Set scheduler parameters + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_param: Scheduling parameters (cannot modify per API contract) + */ + +static void wd_sched_set_param(handle_t h_sched_ctx, + void *sched_key, void *sched_param) +{ + struct wd_sched_params *params = (struct wd_sched_params *)sched_param; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; - skey->sync_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); - skey->async_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); - if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { - WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); - goto out; + if (unlikely(!params || !skey)) { + WD_ERR("invalid: sched parmas or skey is NULL!\n"); + return; } - return (handle_t)skey; + skey->pkt_size = params->pkt_size; + skey->is_stream = params->data_mode; + skey->prio_mode = params->prio_mode; + + /* Store compat filtering parameters */ + skey->alg_name = params->alg_name; + skey->ctxs = params->ctxs; -out: - free(skey); - return (handle_t)(-WD_EINVAL); } static struct wd_sched sched_table[SCHED_POLICY_BUTT] = { { .name = "RR scheduler", .sched_policy = SCHED_POLICY_RR, - .sched_init = session_sched_init, - .pick_next_ctx = session_sched_pick_next_ctx, - .poll_policy = session_sched_poll_policy, + .sched_init = round_robin_sched_init, + .pick_next_ctx = round_robin_pick_next_ctx, + .poll_policy = round_robin_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "None scheduler", .sched_policy = SCHED_POLICY_NONE, .sched_init = sched_none_init, .pick_next_ctx = sched_none_pick_next_ctx, .poll_policy = sched_none_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "Single scheduler", .sched_policy = SCHED_POLICY_SINGLE, .sched_init = sched_single_init, .pick_next_ctx = sched_single_pick_next_ctx, .poll_policy = sched_single_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "Device RR scheduler", .sched_policy = SCHED_POLICY_DEV, .sched_init = session_dev_sched_init, - .pick_next_ctx = session_sched_pick_next_ctx, - .poll_policy = session_sched_poll_policy, - } + .pick_next_ctx = round_robin_pick_next_ctx, + .poll_policy = round_robin_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Loop scheduler", + .sched_policy = SCHED_POLICY_LOOP, + .sched_init = loop_sched_init, + .pick_next_ctx = loop_sched_pick_next_ctx, + .poll_policy = loop_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Hungry scheduler", + .sched_policy = SCHED_POLICY_HUNGRY, + .sched_init = skey_sched_init, + .pick_next_ctx = skey_sched_pick_next_ctx, + .poll_policy = skey_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Instr scheduler", + .sched_policy = SCHED_POLICY_INSTR, + .sched_init = instr_sched_init, + .pick_next_ctx = instr_sched_pick_next_ctx, + .poll_policy = instr_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, }; -static int wd_sched_get_nearby_numa_id(struct wd_sched_info *sched_info, int node, int numa_num) -{ - int dis = INT32_MAX; - int valid_id = -1; - int i, tmp; - - for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) { - tmp = numa_distance(node, i); - if (dis > tmp) { - valid_id = i; - dis = tmp; - } - } - } - - return valid_id; -} - -static void wd_sched_map_cpus_to_dev(struct wd_sched_ctx *sched_ctx) -{ - struct wd_sched_info *sched_info = sched_ctx->sched_info; - int i, numa_num = sched_ctx->numa_num; - int *numa_map = sched_ctx->numa_map; - - for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) - numa_map[i] = i; - else - numa_map[i] = wd_sched_get_nearby_numa_id(sched_info, i, numa_num); - } -} - -static int wd_instance_dev_region(struct wd_sched_ctx *sched_ctx, - struct sched_params *param) +static int numa_num_check(__u16 region_num) { - struct wd_sched_info *sched_info; - __u32 region_idx = INVALID_POS; - __u8 type, mode; - __u32 dev_id; - int i; - - dev_id = param->dev_id; - type = param->type; - mode = param->mode; - - /* Check whether dev_id has already been registered. */ - for (i = 0; i < sched_ctx->dev_num; i++) { - if (sched_ctx->dev_id_map[i].dev_id == dev_id) { - region_idx = sched_ctx->dev_id_map[i].region_id; - break; - } - } - - /* If not registered, allocate a new region. */ - if (region_idx == INVALID_POS) { - if (sched_ctx->dev_num >= DEVICE_REGION_MAX) { - WD_ERR("too many devices registered!\n"); - return -WD_EINVAL; - } - - region_idx = sched_ctx->dev_num; - sched_ctx->dev_id_map[region_idx].dev_id = dev_id; - sched_ctx->dev_id_map[region_idx].region_id = region_idx; - sched_ctx->dev_num++; + int max_node; - sched_info = &sched_ctx->sched_info[region_idx]; - } else { - sched_info = &sched_ctx->sched_info[region_idx]; + max_node = numa_max_node() + 1; + if (max_node <= 0) { + WD_ERR("invalid: numa max node is %d!\n", max_node); + return -WD_EINVAL; } - /* Check whether the mode and type have already been registered. */ - if (sched_info->ctx_region[mode][type].valid) { - WD_INFO("device %u mode %u type %u already registered\n", - dev_id, mode, type); - return WD_SUCCESS; + if (!region_num || region_num > max_node) { + WD_ERR("invalid: region number is %u!\n", region_num); + return -WD_EINVAL; } - /* Initialize the scheduling region for this mode and type */ - sched_info->ctx_region[mode][type].begin = param->begin; - sched_info->ctx_region[mode][type].end = param->end; - sched_info->ctx_region[mode][type].last = param->begin; - sched_info->ctx_region[mode][type].valid = true; - sched_info->valid = true; - - pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); - - return WD_SUCCESS; + return 0; } +/** + * wd_sched_rr_instance - External API for scheduling region instance + * @sched: Scheduler (cannot modify per API contract) + * @param: Scheduling parameters (cannot modify per API contract) + * + * Creates scheduling region for given parameters. + */ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *param) { - struct wd_sched_info *sched_info = NULL; struct wd_sched_ctx *sched_ctx = NULL; - __u8 type, mode; - int numa_id; + struct wd_sched_ctx_domain *domain; + int region_key; + __u8 mode; + int ret; if (!sched || !sched->h_sched_ctx || !param) { WD_ERR("invalid: sched or sched_params is NULL!\n"); @@ -721,57 +2007,68 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para return -WD_EINVAL; } - numa_id = param->numa_id; - type = param->type; mode = param->mode; sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; - if (sched_ctx->numa_num > 0 && (numa_id >= sched_ctx->numa_num || - numa_id < 0)) { - WD_ERR("invalid: sched_ctx's numa_id is %d, numa_num is %u!\n", - numa_id, sched_ctx->numa_num); - return -WD_EINVAL; + if (sched_ctx->policy == SCHED_POLICY_DEV) { + region_key = param->dev_id; + if (region_key < 0) { + WD_ERR("invalid: dev_id is %d!\n", region_key); + return -WD_EINVAL; + } + } else { + region_key = param->numa_id; + if (region_key >= sched_ctx->region_num || region_key < 0) { + WD_ERR("invalid: region_key is %d, region_num is %u!\n", + region_key, sched_ctx->region_num); + return -WD_EINVAL; + } } - if (type >= sched_ctx->type_num) { - WD_ERR("invalid: sched_ctx's type is %u, type_num is %u!\n", - type, sched_ctx->type_num); + if (param->type >= sched_ctx->type_num) { + WD_ERR("invalid: type is %u, type_num is %u!\n", + param->type, sched_ctx->type_num); return -WD_EINVAL; } if (mode >= SCHED_MODE_BUTT) { - WD_ERR("invalid: sched_ctx's mode is %u, mode_num is %d!\n", - mode, SCHED_MODE_BUTT); + WD_ERR("invalid: mode is %u, mode_num is %u!\n", + mode, sched_ctx->mode_num); return -WD_EINVAL; } - if (sched_ctx->policy == SCHED_POLICY_DEV) - return wd_instance_dev_region(sched_ctx, param); - - sched_info = &sched_ctx->sched_info[numa_id]; - if (!sched_info->ctx_region[mode]) { - WD_ERR("invalid: ctx_region is NULL, numa: %d, mode: %u!\n", - numa_id, mode); - return -WD_EINVAL; + if (param->ctx_prop < 0 || param->ctx_prop >= UADK_ALG_TYPE_MAX) { + WD_INFO("Info: ctx_prop %d exceeds max type!\n", param->ctx_prop); + param->ctx_prop = UADK_ALG_HW; } - sched_info->ctx_region[mode][type].begin = param->begin; - sched_info->ctx_region[mode][type].end = param->end; - sched_info->ctx_region[mode][type].last = param->begin; - sched_info->ctx_region[mode][type].valid = true; - sched_info->valid = true; + domain = wd_sched_hash_table_insert(sched_ctx->domain_hash_table, + region_key, mode, param->type, + param->ctx_prop); + if (!domain) + return -WD_ENOMEM; - wd_sched_map_cpus_to_dev(sched_ctx); - pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); + ret = wd_sched_domain_add_segment(domain, param->begin, param->end); + if (ret) { + WD_ERR("failed to add segment to domain!\n"); + return ret; + } + domain->valid = true; return WD_SUCCESS; } +/** + * wd_sched_rr_release - External API for scheduler release + * @sched: Scheduler to release (cannot modify per API contract) + * + * Releases all scheduler resources. + */ void wd_sched_rr_release(struct wd_sched *sched) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; - int i, j, region_num; + struct wd_sched_key *skey; + __u32 i; if (!sched) return; @@ -780,59 +2077,49 @@ void wd_sched_rr_release(struct wd_sched *sched) if (!sched_ctx) goto ctx_out; - /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ - if (sched_ctx->policy == SCHED_POLICY_DEV) - region_num = DEVICE_REGION_MAX; - else - region_num = sched_ctx->numa_num; + /* Release all session keys - iterate full array to catch residual entries */ + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + skey = sched_ctx->skey[i]; + if (!skey) + continue; - sched_info = sched_ctx->sched_info; - if (!sched_info) - goto info_out; + sched_ctx->skey[i] = NULL; + __atomic_store_n(&skey->refcount, 0, __ATOMIC_RELAXED); + session_sched_domain_destroy(skey); + free(skey); + } + sched_ctx->skey_num = 0; - for (i = 0; i < region_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - if (sched_info[i].ctx_region[j]) { - free(sched_info[i].ctx_region[j]); - sched_info[i].ctx_region[j] = NULL; - } - } + /* Release hash table */ + if (sched_ctx->domain_hash_table) { + wd_sched_hash_table_destroy(sched_ctx->domain_hash_table); + sched_ctx->domain_hash_table = NULL; } -info_out: + pthread_mutex_destroy(&sched_ctx->skey_lock); free(sched_ctx); + ctx_out: free(sched); - return; } -static int numa_num_check(__u16 numa_num) -{ - int max_node; - - max_node = numa_max_node() + 1; - if (max_node <= 0) { - WD_ERR("invalid: numa max node is %d!\n", max_node); - return -WD_EINVAL; - } - - if (!numa_num || numa_num > max_node) { - WD_ERR("invalid: numa number is %u!\n", numa_num); - return -WD_EINVAL; - } - - return 0; -} - +/** + * wd_sched_rr_alloc - External API for scheduler allocation + * @sched_type: Scheduling policy type (cannot modify per API contract) + * @type_num: Number of operation types (cannot modify per API contract) + * @region_num: Number of regions (cannot modify per API contract) + * @func: Poll function (cannot modify per API contract) + * + * Allocates and initializes scheduler with single global hash table. + */ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, - __u16 numa_num, user_poll_func func) + __u16 region_num, user_poll_func func) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; struct wd_sched *sched; - int region_num; - int i, j; + __u32 estimated_entries; + __u32 i; if (sched_type >= SCHED_POLICY_BUTT || !type_num) { WD_ERR("invalid: sched_type is %u or type_num is %u!\n", @@ -846,63 +2133,62 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, return NULL; } - if (sched_type == SCHED_POLICY_DEV) - region_num = DEVICE_REGION_MAX; - else - region_num = numa_num; - - sched_ctx = calloc(1, sizeof(struct wd_sched_ctx) + - sizeof(struct wd_sched_info) * region_num); + sched_ctx = calloc(1, sizeof(struct wd_sched_ctx)); if (!sched_ctx) { WD_ERR("failed to alloc memory for sched_ctx!\n"); goto err_out; } - /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ + /* Cache dimension parameters */ + sched_ctx->type_num = type_num; + sched_ctx->mode_num = SCHED_MODE_BUTT; + sched_ctx->region_num = region_num; + sched_ctx->policy = sched_type; + if (sched_type == SCHED_POLICY_DEV) { - sched_ctx->numa_num = 0; - sched_ctx->dev_num = 0; - for (i = 0; i < DEVICE_REGION_MAX; i++) { - sched_ctx->dev_id_map[i].dev_id = INVALID_POS; - sched_ctx->dev_id_map[i].region_id = INVALID_POS; - } + /* Device mode: region_num is actually device count */ + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } else { - sched_ctx->numa_num = numa_num; - sched_ctx->dev_num = 0; - if (numa_num_check(sched_ctx->numa_num)) + /* NUMA mode: validate region_num */ + if (numa_num_check(region_num)) goto err_out; + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } - sched->h_sched_ctx = (handle_t)sched_ctx; - if (sched_type == SCHED_POLICY_NONE || - sched_type == SCHED_POLICY_SINGLE) - goto simple_ok; - - sched_info = sched_ctx->sched_info; - for (i = 0; i < region_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - sched_info[i].ctx_region[j] = - calloc(1, sizeof(struct sched_ctx_region) * type_num); - if (!sched_info[i].ctx_region[j]) - goto err_out; - } + /* Create single global hash table */ + sched_ctx->domain_hash_table = wd_sched_hash_table_create(estimated_entries); + if (!sched_ctx->domain_hash_table) { + WD_ERR("failed to create hash table!\n"); + goto ctx_out; } -simple_ok: sched_ctx->poll_func = func; - sched_ctx->policy = sched_type; - sched_ctx->type_num = type_num; - memset(sched_ctx->numa_map, -1, sizeof(int) * NUMA_NUM_NODES); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) + sched_ctx->skey[i] = NULL; + + if (pthread_mutex_init(&sched_ctx->skey_lock, NULL)) { + WD_ERR("failed to init skey_lock!\n"); + goto err_destroy_hash; + } + sched_ctx->skey_num = 0; + + sched->h_sched_ctx = (handle_t)sched_ctx; sched->sched_init = sched_table[sched_type].sched_init; + sched->sched_uninit = sched_table[sched_type].sched_uninit; sched->pick_next_ctx = sched_table[sched_type].pick_next_ctx; sched->poll_policy = sched_table[sched_type].poll_policy; sched->sched_policy = sched_type; sched->name = sched_table[sched_type].name; + sched->set_param = sched_table[sched_type].set_param; return sched; +err_destroy_hash: + wd_sched_hash_table_destroy(sched_ctx->domain_hash_table); +ctx_out: + free(sched_ctx); err_out: - wd_sched_rr_release(sched); + free(sched); return NULL; } -- 2.43.0

1 15

[PATCH] uadk: fix CCM algorithm long packet failure
by Chenghai Huang 13 Aug '26

13 Aug '26

From: Zhushuai Yin <yinzhushuai(a)huawei.com> When the packet length of the CCM algorithm exceeds 16 bits, the software and hardware comparison fails because the driver does not adapt to the scenario where the packet length exceeds 16 bits. Therefore, the algorithm needs to be modified. Signed-off-by: Zhushuai Yin <yinzhushuai(a)huawei.com> --- drv/hisi_sec.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index aaaaa1d..7d472b8 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -2751,7 +2751,7 @@ static void set_aead_auth_iv(struct wd_aead_msg *msg) __u32 data_size = msg->in_bytes; __u8 flags = 0x00; - __u8 cl, cm; + __u8 cl, cm, i; /* CCM need to cal a_iv, GCM same as c_iv */ memcpy(msg->aiv, msg->iv, msg->iv_bytes); @@ -2771,15 +2771,15 @@ static void set_aead_auth_iv(struct wd_aead_msg *msg) msg->aiv[0] = flags; /* - * the last 32bit is counter's initial number, - * but the nonce uses the first 16bit - * the tail 16bit fill with the cipher length - */ - msg->aiv[msg->iv_bytes - IV_LAST_BYTE1] = - data_size & IV_LAST_BYTE_MASK; - data_size >>= IV_BYTE_OFFSET; - msg->aiv[msg->iv_bytes - IV_LAST_BYTE2] = - data_size & IV_LAST_BYTE_MASK; + * the last 32bit is counter's initial number, + * but the nonce uses the first 16bit + * the tail 16bit fill with the cipher length + * When CL is 3, the tail 24bit fill with the cipher length. + */ + for (i = 1; i <= cl + 1; i++) { + msg->aiv[msg->iv_bytes - i] = data_size & IV_LAST_BYTE_MASK; + data_size >>= IV_BYTE_OFFSET; + } } } -- 2.28.0.windows.1

1 1