[Acc] [PATCH 01/16] uadk: rewrite scheduler with scheduling domain solution

From: Longfang Liu <liulongfang@huawei.com> Conduct a comprehensive update of the scheduler's internal triple-array-based pooling scheme. Adopt new data organization methods such as hash buckets and segment linked lists to achieve efficient insertion and query operations. Implement pre-fetch queue processing for business thread sessions, creating thread-level sub-queue pools for fast packet reception queries. Provide new scheduling algorithms for the new UADK framework. Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- include/wd_alg.h | 11 +- include/wd_alg_common.h | 5 +- include/wd_internal.h | 18 +- include/wd_sched.h | 10 +- wd_sched.c | 2304 ++++++++++++++++++++++++++++++--------- 5 files changed, 1832 insertions(+), 516 deletions(-) diff --git a/include/wd_alg.h b/include/wd_alg.h index 18503ca..1ae1dae 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -64,10 +64,13 @@ extern "C" { #endif enum alg_dev_type { - UADK_ALG_SOFT = 0x0, - UADK_ALG_CE_INSTR = 0x1, - UADK_ALG_SVE_INSTR = 0x2, - UADK_ALG_HW = 0x3 + UADK_ALG_HW = 0x0, + UADK_ALG_CE_INSTR = 0x1, + UADK_ALG_SVE_INSTR = 0x2, + UADK_ALG_SOFT = 0x3, + UADK_ALG_NPU = 0x4, + UADK_ALG_GPU = 0x5, + UADK_ALG_TYPE_MAX, }; /* diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index a294877..ebba7f8 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -41,6 +41,7 @@ extern "C" { /* Key size of digest */ #define MAX_HMAC_KEY_SIZE 128U +#define STATUS_ENABLE (void *)0x1 enum alg_task_type { TASK_MIX = 0x0, @@ -140,7 +141,7 @@ struct wd_ctx_params { }; /* - * struct wd_comp_sched - Define a scheduler. + * struct wd_sched - Define a scheduler. * @name: Name of this scheduler. * @sched_policy: Method for scheduler to perform scheduling * @sched_init: inited the scheduler input parameters. @@ -157,10 +158,12 @@ struct wd_sched { const char *name; int sched_policy; handle_t (*sched_init)(handle_t h_sched_ctx, void *sched_param); + void (*sched_uninit)(handle_t h_sched_ctx, handle_t h_sched_key); __u32 (*pick_next_ctx)(handle_t h_sched_ctx, void *sched_key, const int sched_mode); int (*poll_policy)(handle_t h_sched_ctx, __u32 expect, __u32 *count); + void (*set_param)(handle_t h_sched_ctx, void *sched_key, void *sched_param); handle_t h_sched_ctx; }; diff --git a/include/wd_internal.h b/include/wd_internal.h index 4f19d3a..62cf1a9 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -7,8 +7,10 @@ #define WD_INTERNAL_H #include <pthread.h> +#include <stdatomic.h> #include <stdbool.h> #include "wd.h" +#include "wd_alg.h" #ifdef __cplusplus extern "C" { @@ -41,11 +43,14 @@ struct wd_ce_ctx { }; struct wd_ctx_internal { - handle_t ctx; __u8 op_type; __u8 ctx_mode; + __u8 ctx_type; + handle_t ctx; __u16 sqn; pthread_spinlock_t lock; + struct wd_alg_driver *drv; + __u32 hw_load; }; struct wd_ctx_config_internal { @@ -64,6 +69,17 @@ struct wd_datalist { struct wd_datalist *next; }; +struct wd_sched_params { + __u32 pkt_size; + /* block mode or stream mode */ + __u16 data_mode; + __u16 prio_mode; + + /* Compat filtering parameters for session-ctx matching */ + const char *alg_name; + struct wd_ctx_internal *ctxs; +}; + int memcmp_consttime(const void *s1, const void *s2, size_t n); #ifdef __cplusplus diff --git a/include/wd_sched.h b/include/wd_sched.h index 5baecd3..f81025f 100644 --- a/include/wd_sched.h +++ b/include/wd_sched.h @@ -13,7 +13,8 @@ extern "C" { #endif -#define INVALID_POS 0xFFFFFFFF +#define INVALID_POS 0xFFFF +#define QUEUE_FULL_POS 0x1FFF /* The global policy type */ enum sched_policy_type { @@ -25,6 +26,12 @@ enum sched_policy_type { SCHED_POLICY_SINGLE, /* requests will be sent to ctxs and dev_id */ SCHED_POLICY_DEV, + /* Hard calculation and soft calculation interval loop call */ + SCHED_POLICY_LOOP, + /* Perform heterogeneous calculations through ctx of session key */ + SCHED_POLICY_HUNGRY, + /* Instructions to accelerate heterogeneous computing */ + SCHED_POLICY_INSTR, SCHED_POLICY_BUTT, }; @@ -35,6 +42,7 @@ struct sched_params { __u32 begin; __u32 end; __u32 dev_id; + int ctx_prop; }; typedef int (*user_poll_func)(__u32 pos, __u32 expect, __u32 *count); diff --git a/wd_sched.c b/wd_sched.c index 19936fd..2efa2aa 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -1,374 +1,1442 @@ // SPDX-License-Identifier: Apache-2.0 /* - * Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved. * Copyright 2020-2021 Linaro ltd. + * + * Scheduler: Simplified Pure Hash Table with Dynamic Context Expansion + * + * Key improvements: + * - Single global hash table with (region_id, mode, op_type, prop) dimensions + * - Segment list for non-contiguous ctx ranges + * - Dual-domain queues for session key. + * - Dynamic ctx expansion in HUNGRY mode based on load threshold + * - Packet reception is handled through the active queues in the session key. + * - Simplified sched_init: only allocate one sync + one async ctx + * - Removed redundant wd_sched_info layer */ #define _GNU_SOURCE #include <stdlib.h> #include <stdbool.h> +#include <string.h> #include <sched.h> #include <numa.h> +#include <limits.h> +#include <pthread.h> #include "wd_sched.h" +#include "wd_alg.h" +#include "wd_internal.h" -#define MAX_POLL_TIMES 1000 +#define MAX_POLL_TIMES 1000 +#define HUNGRY_LOAD_THRESHOLD 256 +#define SKEY_CTX_MAX_NUM 16 +#define SKEY_MAX_THREAD_NUM 64 +#define SKEY_LOAD_UPDATE_INTERVAL 1 +#define HW_QUEUE_FULL_DEPTH 1024 +#define MAX_NUMA_NODES (NUMA_NUM_NODES >> 5) + +/* ============================================================================ + * Hash Table Configuration + * ============================================================================ + */ +#define WD_SCHED_MAX_BUCKETS 512 +#define WD_SCHED_MIN_BUCKETS 32 +#define WD_SCHED_LOAD_FACTOR 0.75f +#define HASH_PRIME1 73 +#define HASH_PRIME2 13 +#define HASH_PRIME3 7 +#define HASH_PRIME4 11 + +/* ============================================================================ + * Scheduling Region Mode + * ============================================================================ + */ enum sched_region_mode { SCHED_MODE_SYNC = 0, SCHED_MODE_ASYNC = 1, SCHED_MODE_BUTT }; -/* - * sched_key - The key if schedule region. - * @numa_id: The schedule numa region id. - * @mode: Sync mode:0, async_mode:1 - * @type: Service type , the value must smaller than type_num. - * @sync_ctxid: alloc ctx id for sync mode - * @async_ctxid: alloc ctx id for async mode - */ -struct sched_key { - int numa_id; - __u8 type; - __u8 mode; - __u32 sync_ctxid; - __u32 async_ctxid; - __u32 dev_id; -}; +/* ============================================================================ + * Segment List for Domain Index Organization + * ============================================================================ + */ -/* - * struct sched_ctx_range - define one ctx pos. - * @begin: the start pos in ctxs of config. - * @end: the end pos in ctxx of config. - * @last: the last one which be distributed. - * @valid: the region used flag. - * @lock: lock the currentscheduling region. - */ -struct sched_ctx_region { +/** + * wd_sched_ctx_segment - Contiguous segment of ctx indices in domain + * @begin: Start index of this segment + * @end: End index of this segment (inclusive) + * @next: Pointer to next segment in the linked list + * + * Supports non-contiguous ctx ranges via segment list. + */ +struct wd_sched_ctx_segment { __u32 begin; __u32 end; - __u32 last; + struct wd_sched_ctx_segment *next; +}; + +/* ============================================================================ + * Session key domain cache processing. + * ============================================================================ + */ + +/** + * wd_sched_domain_idx_cache - Simplified fixed array cache for skey domains + * + * Design principles: + * - Fixed array for cache-friendly memory layout + * - Atomic operations for lock-free load tracking + * - Simple RR and load balancing strategies + * - Maximum 16 queues per thread (typical usage) + */ +struct wd_sched_domain_idx_cache { + /* Queue index array */ + __u32 idx_list[SKEY_CTX_MAX_NUM]; /* Array of ctx indices */ + __u32 load_values[SKEY_CTX_MAX_NUM]; /* Atomic load counters */ + __u32 valid_count; /* Number of valid queues */ + + /* Scheduling state */ + __u32 rr_ptr; /* Round-robin pointer */ + __u32 min_load_idx; /* Cached min load index */ + __u32 op_counter; /* Operation counter for updates */ + __u8 load_decreased; /* Non-zero if poll ever decremented load */ + + /* Configuration */ + __u32 update_interval; /* Min load update interval */ + __u8 policy; /* Scheduling policy */ + + /* Synchronization */ + pthread_mutex_t cache_lock; /* Lock for structure modifications */ +}; + +/** + * wd_sched_ctx_domain - Scheduling domain with four dimensions + * @region_id: Region identifier (numa_id or device_id) + * @mode: Context mode (SYNC/ASYNC) + * @op_type: Operation type + * @prop: Property (e.g., device type: HW, CE, SOFT) + * @segments: Linked list of context ranges + * @segment_count: Number of segments + * @total_ctx_count: Total contexts across all segments + * @current_segment: Current segment pointer for round-robin + * @current_pos: Current position within segment + * @valid: Domain validity flag + * @lock: Synchronization spinlock + */ +struct wd_sched_ctx_domain { + int region_id; + __u8 mode; + __u32 op_type; + __u8 prop; + + struct wd_sched_ctx_segment *segments; + __u32 segment_count; + __u32 total_ctx_count; + + struct wd_sched_ctx_segment *current_segment; + __u32 current_pos; bool valid; + pthread_mutex_t lock; }; -/* - * wd_sched_info - define the context of the scheduler. - * @ctx_region: define the map for the comp ctxs, using for quickly search. - * the x range: two(sync and async), the y range: - * two(e.g. comp and uncomp) the map[x][y]'s value is the ctx - * begin and end pos. - * @valid: the region used flag. - */ -struct wd_sched_info { - struct sched_ctx_region *ctx_region[SCHED_MODE_BUTT]; - bool valid; +/** + * wd_sched_domain_hash_node - Hash table collision chain node + */ +struct wd_sched_domain_hash_node { + struct wd_sched_ctx_domain domain; + struct wd_sched_domain_hash_node *next; }; -struct dev_region_map { +/** + * wd_sched_domain_hash_table - Pure dynamic hash table for scheduling domains + * @buckets: Hash table bucket array + * @bucket_size: Number of buckets + * @lock: Read-write lock for concurrent access + */ +struct wd_sched_domain_hash_table { + struct wd_sched_domain_hash_node **buckets; + __u32 bucket_size; + pthread_mutex_t lock; +}; + +/* ============================================================================ + * Dual-Domain Structure for Session Key + * ============================================================================ + */ + +/** + * wd_sched_key_domain - Session domain with min-heap + * @idx_cache: Index cache with min-heap for load-based selection + * @lock: Synchronization spinlock + * @expanded_count: Track how many times ctx has been expanded + */ +struct wd_sched_key_domain { + struct wd_sched_domain_idx_cache idx_cache; + pthread_mutex_t lock; + __u32 expanded_count; +}; + +/** + * wd_sched_key - Session-level scheduling key + * @region_id: Region identifier + * @type: Operation type + * @mode: Current mode (SYNC/ASYNC) + * @dev_id: Device identifier (for SCHED_POLICY_DEV) + * @ctx_prop: Context property + * @is_stream: Stream mode flag + * @prio_mode: Priority mode + * @pkt_size: Current packet size + * @sync_domain: Min-heap domain for sync contexts + * @async_domain: Min-heap domain for async contexts + * @lock: Synchronization spinlock + */ +struct wd_sched_key { + int region_id; + __u8 type; + __u8 mode; __u32 dev_id; - __u32 region_id; + __u8 ctx_prop; + __u16 is_stream; + __u16 prio_mode; + __u32 pkt_size; + + struct wd_sched_key_domain sync_domain; + struct wd_sched_key_domain async_domain; + + pthread_mutex_t lock; + __u32 poll_lock; + __u32 refcount; + + /* Compat filtering parameters for session-ctx matching */ + const char *alg_name; + struct wd_ctx_internal *ctxs; }; -/* - * wd_sched_ctx - define the context of the scheduler. - * @policy: define the policy of the scheduler. - * @numa_num: the max numa numbers of the scheduler. - * @type_num: the max operation types of the scheduler. - * @poll_func: the task's poll operation function. - * @numa_map: a map of cpus to devices. - * @sched_info: the context of the scheduler. + +/** + * wd_sched_ctx - Main scheduler context + * @policy: Scheduling policy type + * @type_num: Number of operation types + * @mode_num: Number of modes (SYNC/ASYNC) + * @region_num: Number of regions (numa or devices) + * @poll_func: Poll function for receiving responses + * @domain_hash_table: Global hash table for all domains + * @skey_num: Number of active session keys + * @skey_lock: Lock for skey array + * @skey: Array of session keys */ struct wd_sched_ctx { __u32 policy; __u32 type_num; - __u16 numa_num; - __u16 dev_num; + __u32 mode_num; + __u16 region_num; + user_poll_func poll_func; - int numa_map[NUMA_NUM_NODES]; - struct dev_region_map dev_id_map[DEVICE_REGION_MAX]; - struct wd_sched_info sched_info[0]; + struct wd_sched_domain_hash_table *domain_hash_table; + + __u32 skey_num; + pthread_mutex_t skey_lock; + struct wd_sched_key *skey[SKEY_MAX_THREAD_NUM]; }; -static bool sched_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +/* ============================================================================ + * Hash Table Core Operations + * ============================================================================ + */ + +static bool wd_sched_is_prime(__u32 n) { - if (key->numa_id >= sched_ctx->numa_num || key->mode >= SCHED_MODE_BUTT || - key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's numa: %d, mode: %u, type: %u!\n", - key->numa_id, key->mode, key->type); + __u32 i; + + if (n <= 1) return false; + if (n <= 3) + return true; + if (n % 2 == 0 || n % 3 == 0) + return false; + + for (i = 5; i * i <= n; i += 6) { + if (n % i == 0 || n % (i + 2) == 0) + return false; } return true; } -/* - * sched_get_ctx_range - Get ctx range from ctx_map by the wd comp arg +static __u32 wd_sched_find_prime(__u32 n) +{ + while (!wd_sched_is_prime(n)) + n++; + return n; +} + +static __u32 wd_sched_compute_bucket_size(__u32 estimated_entries) +{ + __u32 target_size; + + target_size = (estimated_entries * 4) / 3; + + if (target_size < WD_SCHED_MIN_BUCKETS) + target_size = WD_SCHED_MIN_BUCKETS; + if (target_size > WD_SCHED_MAX_BUCKETS) + target_size = WD_SCHED_MAX_BUCKETS; + + return wd_sched_find_prime(target_size); +} + +/** + * wd_sched_hash_compute - Compute hash value for four-dimensional domain key + * @region_id: Region identifier + * @mode: Context mode + * @op_type: Operation type + * @prop: Property + * @bucket_size: Hash table bucket count + * + * Combines four dimensions using prime number multipliers. */ -static struct sched_ctx_region *sched_get_ctx_range(struct wd_sched_ctx *sched_ctx, - const struct sched_key *key) +static inline __u32 wd_sched_hash_compute(int region_id, __u8 mode, + __u32 op_type, __u8 prop, __u32 bucket_size) { - struct wd_sched_info *sched_info; - int numa_id; + __u32 hash; - sched_info = sched_ctx->sched_info; - if (key->numa_id >= 0 && - sched_info[key->numa_id].ctx_region[key->mode][key->type].valid) - return &sched_info[key->numa_id].ctx_region[key->mode][key->type]; + hash = (region_id * HASH_PRIME1) + (mode * HASH_PRIME2) + + (op_type * HASH_PRIME3) + (prop * HASH_PRIME4); + return hash % bucket_size; +} + +static inline bool wd_sched_domain_key_match( + int region_id1, __u8 mode1, __u32 op_type1, __u8 prop1, + int region_id2, __u8 mode2, __u32 op_type2, __u8 prop2) +{ + return (region_id1 == region_id2 && mode1 == mode2 && + op_type1 == op_type2 && prop1 == prop2); +} + +/** + * wd_sched_hash_table_create - Create hash table + * @estimated_entries: Estimated number of entries + * + * Returns: Initialized hash table or NULL on error + */ +static struct wd_sched_domain_hash_table * +wd_sched_hash_table_create(__u32 estimated_entries) +{ + struct wd_sched_domain_hash_table *table; + __u32 bucket_size; + int ret; - /* If the key->numa_id is not exist, we should scan for a region */ - for (numa_id = 0; numa_id < sched_ctx->numa_num; numa_id++) { - if (sched_info[numa_id].ctx_region[key->mode][key->type].valid) - return &sched_info[numa_id].ctx_region[key->mode][key->type]; + table = calloc(1, sizeof(*table)); + if (!table) + return NULL; + + bucket_size = wd_sched_compute_bucket_size(estimated_entries); + + table->buckets = calloc(bucket_size, sizeof(*table->buckets)); + if (!table->buckets) { + free(table); + return NULL; } - return NULL; + table->bucket_size = bucket_size; + ret = pthread_mutex_init(&table->lock, NULL); + if (ret) { + free(table->buckets); + free(table); + return NULL; + } + + return table; } -/* - * sched_get_next_pos_rr - Get next resource pos by RR schedule. - * The second para is reserved for future. +static void wd_sched_hash_table_destroy(struct wd_sched_domain_hash_table *table) +{ + struct wd_sched_domain_hash_node *node, *next; + struct wd_sched_ctx_segment *seg, *next_seg; + __u32 i; + + if (!table) + return; + + for (i = 0; i < table->bucket_size; i++) { + node = table->buckets[i]; + while (node) { + next = node->next; + + /* Release segment linked list */ + seg = node->domain.segments; + while (seg) { + next_seg = seg->next; + free(seg); + seg = next_seg; + } + + pthread_mutex_destroy(&node->domain.lock); + free(node); + node = next; + } + } + + pthread_mutex_destroy(&table->lock); + free(table->buckets); + free(table); +} + +static struct wd_sched_ctx_domain * +wd_sched_hash_table_lookup(struct wd_sched_domain_hash_table *table, + int region_id, __u8 mode, __u32 op_type, __u8 prop) +{ + struct wd_sched_domain_hash_node *node; + struct wd_sched_ctx_domain *domain = NULL; + __u32 hash_idx; + + if (!table) + return NULL; + + pthread_mutex_lock(&table->lock); + hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size); + node = table->buckets[hash_idx]; + while (node) { + if (wd_sched_domain_key_match( + node->domain.region_id, node->domain.mode, node->domain.op_type, + node->domain.prop, region_id, mode, op_type, prop)) { + domain = &node->domain; + break; + } + node = node->next; + } + pthread_mutex_unlock(&table->lock); + + return domain; +} + +static struct wd_sched_ctx_domain * +wd_sched_hash_table_insert(struct wd_sched_domain_hash_table *table, + int region_id, __u8 mode, __u32 op_type, __u8 prop) +{ + struct wd_sched_domain_hash_node *new_node; + struct wd_sched_ctx_domain *existing; + __u32 hash_idx; + int ret; + + if (!table) + return NULL; + + existing = wd_sched_hash_table_lookup(table, region_id, mode, op_type, prop); + if (existing) + return existing; + + pthread_mutex_lock(&table->lock); + hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size); + /* Alloc and initialize new domain */ + new_node = calloc(1, sizeof(*new_node)); + if (!new_node) { + pthread_mutex_unlock(&table->lock); + return NULL; + } + + /* Initialize new domain */ + new_node->domain.region_id = region_id; + new_node->domain.mode = mode; + new_node->domain.op_type = op_type; + new_node->domain.prop = prop; + new_node->domain.segments = NULL; + new_node->domain.segment_count = 0; + new_node->domain.total_ctx_count = 0; + new_node->domain.current_segment = NULL; + new_node->domain.current_pos = 0; + new_node->domain.valid = false; + + ret = pthread_mutex_init(&new_node->domain.lock, NULL); + if (ret) { + pthread_mutex_unlock(&table->lock); + free(new_node); + return NULL; + } + + new_node->next = table->buckets[hash_idx]; + table->buckets[hash_idx] = new_node; + pthread_mutex_unlock(&table->lock); + + return &new_node->domain; +} + +/* ============================================================================ + * Segment List Operations + * ============================================================================ + */ + +/** + * wd_sched_domain_add_segment - Add context range segment to domain + * @domain: Target domain + * @begin: Start context index + * @end: End context index (inclusive) + * + * Supports non-contiguous context ranges via segment list. */ -static __u32 sched_get_next_pos_rr(struct sched_ctx_region *region, void *para) +static int wd_sched_domain_add_segment(struct wd_sched_ctx_domain *domain, + __u32 begin, __u32 end) { + struct wd_sched_ctx_segment *seg, *new_seg; + + if (!domain || begin > end) + return -WD_EINVAL; + + new_seg = calloc(1, sizeof(*new_seg)); + if (!new_seg) + return -WD_ENOMEM; + + new_seg->begin = begin; + new_seg->end = end; + new_seg->next = NULL; + + pthread_mutex_lock(&domain->lock); + + /* Append to segment list tail */ + if (!domain->segments) { + domain->segments = new_seg; + } else { + seg = domain->segments; + while (seg->next) + seg = seg->next; + seg->next = new_seg; + } + + domain->segment_count++; + domain->total_ctx_count += (end - begin + 1); + + /* Initialize polling state */ + if (!domain->current_segment) + domain->current_segment = domain->segments; + + pthread_mutex_unlock(&domain->lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_domain_get_next_rr - Get next context via round-robin from domain + * @domain: Source domain + * + * Returns: Next global queue index in round-robin order + * Time complexity: O(1) + */ +static __u32 wd_sched_domain_get_next_rr(struct wd_sched_ctx_domain *domain) +{ + __u32 ctx_idx; __u32 pos; - pthread_mutex_lock(®ion->lock); + if (!domain || !domain->segments || !domain->total_ctx_count) + return INVALID_POS; - pos = region->last; + pthread_mutex_lock(&domain->lock); + if (!domain->current_segment) + domain->current_segment = domain->segments; - if (pos < region->end) - region->last++; - else - region->last = region->begin; + pos = domain->current_pos; + + /* Calculate global queue number: segment.begin + relative position */ + ctx_idx = domain->current_segment->begin + pos; - pthread_mutex_unlock(®ion->lock); + /* Move to next position */ + if (pos + 1 < domain->current_segment->end - domain->current_segment->begin + 1) { + /* Within same segment */ + domain->current_pos = pos + 1; + } else if (domain->current_segment->next) { + /* Move to next segment */ + domain->current_segment = domain->current_segment->next; + domain->current_pos = 0; + } else { + /* Loop back to beginning */ + domain->current_segment = domain->segments; + domain->current_pos = 0; + } + + pthread_mutex_unlock(&domain->lock); - return pos; + return ctx_idx; } -/* - * session_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/* ============================================================================ + * SKey Domain Cache Management Functions + * ============================================================================ + */ +/** + * wd_sched_skey_cache_init - Initialize skey domain cache + * @cache: Pointer to cache structure + * @policy: Scheduling policy * @sched_type: Scheduling policy type (cannot modify per + * API contract) + * + * Initialize fixed array cache with invalid positions and zero loads. + */ +static int wd_sched_skey_cache_init(struct wd_sched_domain_idx_cache *cache, + __u8 policy) +{ + int i; + + if (!cache) { + WD_ERR("invalid: cache pointer is NULL!\n"); + return -WD_EINVAL; + } + + /* Initialize array with invalid positions */ + for (i = 0; i < SKEY_CTX_MAX_NUM; i++) { + cache->idx_list[i] = INVALID_POS; + __atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED); + } + + /* Initialize atomic counters */ + __atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->op_counter, 0, __ATOMIC_RELAXED); + cache->load_decreased = 0; + + /* Set configuration */ + cache->valid_count = 0; + cache->update_interval = SKEY_LOAD_UPDATE_INTERVAL; + cache->policy = policy; + + /* Initialize structure lock */ + if (pthread_mutex_init(&cache->cache_lock, NULL)) { + WD_ERR("failed to init cache lock!\n"); + return -WD_EINVAL; + } + + return WD_SUCCESS; +} +/** + * wd_sched_skey_cache_uninit - Cleanup skey domain cache + * @cache: Pointer to cache structure + * + * Release resources and reset cache state. + */ +static void wd_sched_skey_cache_uninit(struct wd_sched_domain_idx_cache *cache) +{ + if (!cache) + return; + + pthread_mutex_destroy(&cache->cache_lock); + + /* Reset cache state */ + for (int i = 0; i < SKEY_CTX_MAX_NUM; i++) { + cache->idx_list[i] = INVALID_POS; + __atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED); + } + + cache->valid_count = 0; + cache->load_decreased = 0; +} +/** + * wd_sched_skey_add_ctx - Add ctx to skey domain cache + * @cache: Pointer to cache structure + * @ctx_id: Context ID to add + * + * Add ctx to next available position in fixed array. + * Returns 0 on success, negative error code on failure. + */ +static int wd_sched_skey_add_ctx(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_id) +{ + __u32 i; + + if (!cache || ctx_id == INVALID_POS) { + WD_ERR("invalid: parameters are NULL!\n"); + return -WD_EINVAL; + } + + pthread_mutex_lock(&cache->cache_lock); + /* Check if cache is full */ + if (cache->valid_count >= SKEY_CTX_MAX_NUM) { + pthread_mutex_unlock(&cache->cache_lock); + WD_ERR("invalid: skey cache full, cannot add more queues!\n"); + return -WD_EINVAL; + } + + /* Check for duplicate ctx_id */ + for (i = 0; i < cache->valid_count; i++) { + if (cache->idx_list[i] == ctx_id) { + WD_ERR("invalid: context %u already exists in skey cache at pos %u!\n", + ctx_id, i); + pthread_mutex_unlock(&cache->cache_lock); + return -WD_EEXIST; + } + } + + /* Update min load index if as the new ctx */ + __atomic_store_n(&cache->min_load_idx, cache->valid_count, __ATOMIC_RELAXED); + + /* Add to next available position */ + cache->idx_list[cache->valid_count] = ctx_id; + __atomic_store_n(&cache->load_values[cache->valid_count], 0, __ATOMIC_RELAXED); + cache->valid_count++; + pthread_mutex_unlock(&cache->cache_lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_skey_remove_ctx - Remove ctx from skey domain cache + * @cache: Pointer to cache structure + * @ctx_id: Context ID to remove + * + * Remove ctx by shifting array elements to maintain continuity. + * Returns 0 on success, negative error code if not found. + */ +static int wd_sched_skey_remove_ctx(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_id) +{ + __u32 i, current_min; + int found = 0; + + if (!cache) { + WD_ERR("invalid: cache pointer is NULL!\n"); + return -WD_EINVAL; + } + + pthread_mutex_lock(&cache->cache_lock); + /* Find and remove the ctx */ + for (i = 0; i < cache->valid_count; i++) { + if (cache->idx_list[i] == ctx_id) { + found = 1; + break; + } + } + + if (!found) { + WD_ERR("invalid: context %u not found in skey cache!\n", ctx_id); + pthread_mutex_unlock(&cache->cache_lock); + return -WD_ENODEV; + } + + /* Shift remaining elements to fill the gap */ + for (; i < cache->valid_count - 1; i++) { + cache->idx_list[i] = cache->idx_list[i + 1]; + __atomic_store_n(&cache->load_values[i], + __atomic_load_n(&cache->load_values[i + 1], __ATOMIC_RELAXED), + __ATOMIC_RELAXED); + } + + /* Clear last position */ + cache->idx_list[cache->valid_count - 1] = INVALID_POS; + __atomic_store_n(&cache->load_values[cache->valid_count - 1], 0, __ATOMIC_RELAXED); + cache->valid_count--; + + /* Reset pointers if cache becomes empty */ + if (!cache->valid_count) { + __atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + } else { + /* Adjust min load index if necessary */ + current_min = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED); + if (current_min >= cache->valid_count) + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + } + pthread_mutex_unlock(&cache->cache_lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_update_min_load - Update cached min load index + * @cache: Pointer to cache structure + * @ctxs: Context array for hw_load lookup, or NULL to use cache->load_values * - * The user must init the schedule info through wd_sched_rr_instance + * Scan valid queues to find the one with minimum load. + * When ctxs is provided, reads per-ctx hw_load for cross-session accuracy. */ -static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, - const int sched_mode) +static void wd_sched_update_min_load(struct wd_sched_domain_idx_cache *cache, + struct wd_ctx_internal *ctxs) { - struct sched_ctx_region *region = NULL; - bool ret; + __u32 min_load = UINT_MAX; + __u32 min_idx = 0; + __u32 i, load; - key->mode = sched_mode; - ret = sched_key_valid(sched_ctx, key); - if (!ret) + if (!cache->valid_count) + return; + + for (i = 0; i < cache->valid_count; i++) { + if (ctxs) + load = __atomic_load_n(&ctxs[cache->idx_list[i]].hw_load, + __ATOMIC_RELAXED); + else + load = __atomic_load_n(&cache->load_values[i], __ATOMIC_RELAXED); + if (load < min_load) { + min_load = load; + min_idx = i; + } + } + + __atomic_store_n(&cache->min_load_idx, min_idx, __ATOMIC_RELAXED); +} + +/** + * wd_sched_skey_pick_next - Pick next ctx from skey domain cache + * @cache: Pointer to cache structure + * @ctx_idx: Output index within cache array + * @ctxs: Context array for hw_load lookup, or NULL + * + * Select next ctx based on scheduling policy: + * - RR: Simple round-robin selection + * - HUNGRY: Choose ctx with minimum load (hw_load if ctxs available) + * + * Returns selected ctx index, or INVALID_POS if no valid ctx. + */ +static __u32 wd_sched_skey_pick_next(struct wd_sched_domain_idx_cache *cache, + __u32 *ctx_idx, struct wd_ctx_internal *ctxs) +{ + __u32 selected_idx; + __u32 op_count; + + if (!cache || !cache->valid_count) return INVALID_POS; - region = sched_get_ctx_range(sched_ctx, key); - if (!region) + switch (cache->policy) { + case SCHED_POLICY_RR: + case SCHED_POLICY_NONE: + case SCHED_POLICY_SINGLE: + case SCHED_POLICY_DEV: + case SCHED_POLICY_LOOP: + case SCHED_POLICY_INSTR: + /* Round-robin: atomic increment and module */ + selected_idx = __atomic_fetch_add(&cache->rr_ptr, 1, __ATOMIC_RELAXED) % + cache->valid_count; + break; + case SCHED_POLICY_HUNGRY: + /* Update min load periodically */ + op_count = __atomic_fetch_add(&cache->op_counter, 1, __ATOMIC_RELAXED); + if (op_count % cache->update_interval == 0) + wd_sched_update_min_load(cache, ctxs); + + /* Load balancing: use cached min load index */ + selected_idx = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED); + break; + default: + WD_ERR("invalid: unknown scheduling policy %d!\n", cache->policy); + selected_idx = INVALID_POS; + break; + } + + /* Ensure index is within valid range */ + if (selected_idx >= cache->valid_count) return INVALID_POS; - return sched_get_next_pos_rr(region, NULL); + *ctx_idx = selected_idx; + return cache->idx_list[selected_idx]; } -static handle_t session_sched_init(handle_t h_sched_ctx, void *sched_param) +/** + * wd_sched_skey_update_load - Update load for a specific ctx + * @cache: Pointer to cache structure + * @ctx_idx: Context index in list + * @delta: Load delta (positive for send, negative for receive) + * + * Atomically update load counter for the specified ctx. + * Returns 0 on success, negative error code if ctx not found. + */ +static int wd_sched_skey_update_load(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_idx, int delta) { - struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - struct sched_params *param = (struct sched_params *)sched_param; - struct sched_key *skey; + /* Atomic update without locking, ctx_idx's value is guaranteed by the caller. */ + if (delta > 0) + __atomic_fetch_add(&cache->load_values[ctx_idx], delta, __ATOMIC_RELAXED); + else { + __atomic_fetch_sub(&cache->load_values[ctx_idx], -delta, __ATOMIC_RELAXED); + if (unlikely(!cache->load_decreased)) + cache->load_decreased = 1; + } + return WD_SUCCESS; +} + +/* ============================================================================ + * Session Key Domain Initialization + * ============================================================================ + */ + +/** + * wd_sched_skey_domain_init - Initialize session domain with min-heap + * @key_domain: Target key domain + * @ctx_idx: context indices idx + * @policy: current session's policy + * + * Initializes dual-domain structure for session. + */ +static int wd_sched_skey_domain_init(struct wd_sched_key_domain *key_domain, + __u32 ctx_idx, __u8 policy) +{ + int ret; + + if (!key_domain) + return -WD_EINVAL; + + ret = wd_sched_skey_cache_init(&key_domain->idx_cache, policy); + if (ret) + return ret; + + ret = wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx); + if (ret) + goto init_err; + + ret = pthread_mutex_init(&key_domain->lock, NULL); + if (ret) + goto add_ctx_err; + + key_domain->expanded_count = 0; + + return WD_SUCCESS; + +add_ctx_err: + wd_sched_skey_remove_ctx(&key_domain->idx_cache, ctx_idx); +init_err: + wd_sched_skey_cache_uninit(&key_domain->idx_cache); + return ret; +} + +/** + * wd_sched_skey_domain_destroy - Release session domain resources + */ +static void wd_sched_skey_domain_destroy(struct wd_sched_key_domain *key_domain) +{ + if (!key_domain) + return; + + pthread_mutex_destroy(&key_domain->lock); + wd_sched_skey_cache_uninit(&key_domain->idx_cache); +} + + +/** + * wd_sched_poll_skey - Poll contexts for scheduler session + * @sched_ctx: Scheduler context + * @skey: Session key + * @expect: Expected number of responses + * @count: Actual response count (output) + * + * Polls all contexts in session domains and updates load values. + */ +static int wd_sched_poll_skey(struct wd_sched_ctx *sched_ctx, struct wd_sched_key *skey, + __u32 expect, __u32 *count) +{ + struct wd_sched_domain_idx_cache *cache; + __u32 ctx_list_num = 0; + __u32 sum_poll_num = 0; + bool hungry_policy; + __u32 poll_num; + __u32 idx, i; + int ret = 0; + + /* Get cache pointer and check if HUNGRY policy */ + cache = &skey->async_domain.idx_cache; + ctx_list_num = cache->valid_count; + hungry_policy = (cache->policy == SCHED_POLICY_HUNGRY); + + /* Poll async domain contexts */ + for (i = 0; i < ctx_list_num; i++) { + idx = cache->idx_list[i]; + if (idx == INVALID_POS) + continue; + poll_num = 0; + ret = sched_ctx->poll_func(idx, expect, &poll_num); + if (poll_num > 0) + sum_poll_num += poll_num; + + /* Update load value for this context */ + if (hungry_policy && poll_num > 0) { + if (skey->ctxs) + __atomic_fetch_sub(&skey->ctxs[idx].hw_load, + poll_num, __ATOMIC_RELAXED); + wd_sched_skey_update_load(cache, i, -poll_num); + } + + if (ret < 0 && ret != -WD_EAGAIN) + break; + } + *count = sum_poll_num; + + return ret; +} + +/* ============================================================================ + * Utility Functions + * ============================================================================ + */ +static inline bool sched_skey_get_ref(struct wd_sched_key *skey) +{ + if (!skey) + return false; + return __atomic_fetch_add(&skey->refcount, 1, __ATOMIC_ACQUIRE) > 0; +} + +static inline bool sched_skey_put_ref(struct wd_sched_key *skey) +{ + return __atomic_fetch_sub(&skey->refcount, 1, __ATOMIC_RELEASE) == 1; +} + +static int sched_skey_param_init(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey) +{ + __u32 i; + + pthread_mutex_lock(&sched_ctx->skey_lock); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + if (!sched_ctx->skey[i]) { + sched_ctx->skey[i] = skey; + if (sched_ctx->skey_num < SKEY_MAX_THREAD_NUM) + sched_ctx->skey_num++; + pthread_mutex_unlock(&sched_ctx->skey_lock); + return 0; + } + } + pthread_mutex_unlock(&sched_ctx->skey_lock); + WD_ERR("invalid: skey node number exceeds SKEY_MAX_THREAD_NUM(%d)!\n", + SKEY_MAX_THREAD_NUM); + return -WD_ENOMEM; +} + +static void sched_skey_param_uninit(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey) +{ + __u32 i; + + if (!sched_ctx || !skey) + return; + + pthread_mutex_lock(&sched_ctx->skey_lock); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + if (sched_ctx->skey[i] == skey) { + sched_ctx->skey[i] = NULL; + pthread_mutex_unlock(&sched_ctx->skey_lock); + return; + } + } + pthread_mutex_unlock(&sched_ctx->skey_lock); + WD_ERR("warning: skey %p not found in sched_ctx array\n", skey); +} + +static handle_t sched_session_common_init(struct wd_sched_ctx *sched_ctx, + struct sched_params *param) +{ + struct wd_sched_key *skey; unsigned int node; if (getcpu(NULL, &node)) { WD_ERR("failed to get node, errno %d!\n", errno); return (handle_t)(-errno); } - if (node == (unsigned int)NUMA_NO_NODE) { - WD_ERR("invalid: failed to get numa node!\n"); - return (handle_t)(-WD_EINVAL); - } if (!sched_ctx) { WD_ERR("invalid: sched ctx is NULL!\n"); return (handle_t)(-WD_EINVAL); } - skey = malloc(sizeof(struct sched_key)); + skey = malloc(sizeof(struct wd_sched_key)); if (!skey) { WD_ERR("failed to alloc memory for session sched key!\n"); return (handle_t)(-WD_ENOMEM); } + memset(skey, 0, sizeof(struct wd_sched_key)); if (!param) { - memset(skey, 0, sizeof(struct sched_key)); - skey->numa_id = sched_ctx->numa_map[node]; + skey->region_id = node; if (wd_need_debug()) WD_DEBUG("session don't set scheduler parameters!\n"); - } else if (param->numa_id < 0) { - skey->type = param->type; - skey->numa_id = sched_ctx->numa_map[node]; } else { + if (sched_ctx->policy == SCHED_POLICY_DEV) + skey->region_id = param->dev_id; + else if (param->numa_id >= 0) + skey->region_id = param->numa_id; + else + skey->region_id = node; skey->type = param->type; - skey->numa_id = param->numa_id; + skey->ctx_prop = param->ctx_prop; } + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + __atomic_store_n(&skey->refcount, 1, __ATOMIC_RELAXED); - if (skey->numa_id < 0) { - WD_ERR("failed to get valid sched numa region!\n"); - goto out; - } + return (handle_t)skey; +} - skey->sync_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); - skey->async_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); - if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { - WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); - goto out; +static __u16 sched_get_poll_skey_idx(struct wd_sched_ctx *sched_ctx) +{ + /* Thread-local sequence number with atomic global counter */ + static __thread __u32 thread_seq = UINT32_MAX; + static __u32 global_seq_counter; + __u32 skey_num = sched_ctx->skey_num; + __u16 start_pos; + + if (unlikely(!sched_ctx || !skey_num)) + return skey_num; + + /* Assign unique sequence number on first call */ + if (unlikely(thread_seq == UINT32_MAX)) { + thread_seq = __atomic_fetch_add(&global_seq_counter, 1, __ATOMIC_RELAXED); + + /* Basic overflow protection */ + if (thread_seq >= UINT32_MAX - 1) + WD_DEBUG("Thread sequence counter approaching limit: %u\n", thread_seq); } - return (handle_t)skey; + /* Calculate start_pos based on thread_seq */ + start_pos = thread_seq % skey_num; -out: - free(skey); - return (handle_t)(-WD_EINVAL); + return start_pos; } -/* - * session_pick_next_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/** + * session_sched_init_ctx - Pre-fetch single context from domain for session + * @sched_ctx: Scheduler context + * @skey: session scheduler param + * @sched_mode: Mode (SYNC/ASYNC) * - * The user must init the schedule info through session_sched_init + * Returns: Context index from domain (compatible with alg_name if skey provided) */ -static __u32 session_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, - const int sched_mode) +static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, int sched_mode) { - struct sched_key *key = (struct sched_key *)sched_key; + struct wd_sched_ctx_domain *domain = NULL; + int region_id = skey->region_id; + __u32 op_type = skey->type; + __u8 prop = skey->ctx_prop; + __u32 ctx_idx; + __u16 r; + + if (sched_mode >= SCHED_MODE_BUTT || + op_type >= sched_ctx->type_num || prop >= UADK_ALG_TYPE_MAX) { + WD_ERR("invalid: region: %d, mode: %d, type: %u!, prop: %u\n", + region_id, sched_mode, op_type, prop); + return INVALID_POS; + } - if (unlikely(!h_sched_ctx || !key)) { - WD_ERR("invalid: sched ctx or key is NULL!\n"); + if (!sched_ctx->domain_hash_table) + return INVALID_POS; + + /* Try current region first */ + if (region_id >= 0) { + if (sched_ctx->policy == SCHED_POLICY_DEV || + region_id < sched_ctx->region_num) { + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + region_id, sched_mode, op_type, prop); + if (domain && domain->valid) + return wd_sched_domain_get_next_rr(domain); + } + } + + /* DEV policy must not cross region */ + if (sched_ctx->policy == SCHED_POLICY_DEV) return INVALID_POS; + + /* Cross-region fallback for other policies */ + for (r = 0; r < sched_ctx->region_num; r++) { + if ((int)r == region_id) + continue; + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + r, sched_mode, op_type, prop); + if (domain && domain->valid) { + ctx_idx = wd_sched_domain_get_next_rr(domain); + if (ctx_idx != INVALID_POS) + return ctx_idx; + } } - /* return in do task */ - if (sched_mode == CTX_MODE_SYNC) - return key->sync_ctxid; - return key->async_ctxid; + return INVALID_POS; } -static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, - __u32 end, __u32 expect, __u32 *count) +/** + * session_sched_domain_destroy - Destroy session domains + * @skey: Session key to destroy domains for + * + * Releases all resources associated with session domains. + */ +static void session_sched_domain_destroy(struct wd_sched_key *skey) { - __u32 poll_num = 0; - __u32 i; - int ret; + if (!skey) + return; - /* i is the pos of sched_ctxs, the max is end */ - for (i = begin; i <= end; i++) { - /* - * RR schedule, one time poll one package, - * poll_num is always not more than one here. - */ - ret = sched_ctx->poll_func(i, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) - return ret; - else if (ret == -EAGAIN) + /* Destroy both sync and async domains */ + wd_sched_skey_domain_destroy(&skey->sync_domain); + wd_sched_skey_domain_destroy(&skey->async_domain); +} + +static inline void sched_skey_poll_release(struct wd_sched_key *skey) +{ + __u32 prev; + + if (!skey) + return; + + prev = __atomic_fetch_sub(&skey->refcount, 1, __ATOMIC_ACQ_REL); + if (prev == 1) { + session_sched_domain_destroy(skey); + free(skey); + } +} + +static void sched_session_uninit(handle_t h_sched_ctx, handle_t h_sched_key) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_key *skey = (struct wd_sched_key *)h_sched_key; + + if (!skey) + return; + + /* Remove from skey array first to prevent new poll discovery. + * Poll threads already holding a pointer are protected by refcount. + */ + if (sched_ctx) + sched_skey_param_uninit(sched_ctx, skey); + + /* Drop creator reference. If no poll holds a ref, free now. + * Otherwise the last poll thread will cleanup via poll_release. + */ + if (sched_skey_put_ref(skey)) { + session_sched_domain_destroy(skey); + free(skey); + } +} + +/** + * session_sched_init_ctx_with_fallback - Pre-fetch a ctx for one sched_mode, + * trying user_prop first then falling + * back through other props in enum order. + * @sched_ctx: Scheduler context + * @skey: Session key (skey->ctx_prop is the user-specified prop; restored on return) + * @sched_mode: SCHED_MODE_SYNC or SCHED_MODE_ASYNC + * + * Returns: ctx index, or INVALID_POS if no prop has a usable ctx. + * + * Falls back only when the user-specified prop's domain is unavailable + * (e.g. HW driver unloaded). Stops at the first prop that yields a ctx; + * does NOT accumulate ctxs across props. + */ +static __u32 session_sched_init_ctx_with_fallback(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + int sched_mode) +{ + __u8 user_prop = skey->ctx_prop; + __u32 ctx_idx; + __u8 p; + + /* Try user-specified prop first */ + ctx_idx = session_sched_init_ctx(sched_ctx, skey, sched_mode); + if (ctx_idx != INVALID_POS) + return ctx_idx; + + /* Fallback: target prop unavailable, try remaining props in enum order */ + for (p = 0; p < UADK_ALG_TYPE_MAX; p++) { + if (p == user_prop) continue; - *count += poll_num; - if (*count == expect) + skey->ctx_prop = p; + ctx_idx = session_sched_init_ctx(sched_ctx, skey, sched_mode); + if (ctx_idx != INVALID_POS) break; } - return 0; + /* Restore user_prop so skey state is not mutated */ + skey->ctx_prop = user_prop; + return ctx_idx; } -static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, - __u32 expect, __u32 *count) +/** + * session_sched_domain_init - Initialize session domains with sync/async + * @sched_ctx: Scheduler context + * @skey: Session key to initialize + * @allow_fallback: If true, fall back to other props when skey->ctx_prop has + * no usable domain; if false, only try skey->ctx_prop. + * + * Pre-fetches sync and async contexts and initializes corresponding domains. + * Returns: 0 on success, negative error code on failure. + */ +static int session_sched_domain_init(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + bool allow_fallback) { - struct sched_ctx_region **region = sched_ctx->sched_info[numa_id].ctx_region; - __u32 begin, end; - __u32 i; - int ret; + __u32 sync_ctx, async_ctx; - for (i = 0; i < sched_ctx->type_num; i++) { - if (!region[SCHED_MODE_ASYNC][i].valid) - continue; + if (!sched_ctx || !skey) { + WD_ERR("invalid: sched_ctx or skey is NULL!\n"); + return -WD_EINVAL; + } + + if (allow_fallback) { + /* Pre-fetch with per-mode prop fallback */ + sync_ctx = session_sched_init_ctx_with_fallback(sched_ctx, skey, + SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx_with_fallback(sched_ctx, skey, + SCHED_MODE_ASYNC); + } else { + sync_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_ASYNC); + } + + if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) { + WD_ERR("invalid: no valid sync_ctx or async_ctx domain!\n"); + return -WD_EINVAL; + } + + /* Initialize sync domain if context is valid */ + if (sync_ctx != INVALID_POS) { + if (wd_sched_skey_domain_init(&skey->sync_domain, sync_ctx, sched_ctx->policy)) { + WD_ERR("failed to init sync domain!\n"); + return -WD_EINVAL; + } + } + + /* Initialize async domain if context is valid */ + if (async_ctx != INVALID_POS) { + if (wd_sched_skey_domain_init(&skey->async_domain, async_ctx, sched_ctx->policy)) { + WD_ERR("failed to init async domain!\n"); + /* Cleanup sync domain if async domain init failed */ + if (sync_ctx != INVALID_POS) + wd_sched_skey_domain_destroy(&skey->sync_domain); + return -WD_EINVAL; + } + } + + return WD_SUCCESS; +} + +/* ============================================================================ + * Scheduler Policy Functions + * ============================================================================ + */ +/** + * round_robin_sched_init - Initialize session with single sync and async ctx + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_param: Scheduling parameters (cannot modify per API contract) + * + * Allocates session key and pre-fetches one sync and one async context. + */ +static handle_t round_robin_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + struct wd_sched_key *skey; + handle_t hskey; + int ret = 0; + + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; + } + + skey = (struct wd_sched_key *)hskey; + /* RR: allow prop fallback so session creation survives target prop unload */ + ret = session_sched_domain_init(sched_ctx, skey, true); + if (ret) { + WD_ERR("failed to initialize session domains!\n"); + free(skey); + return (handle_t)(-WD_EINVAL); + } + + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); + return (handle_t)(-WD_ENOMEM); + } + + return hskey; +} + +/** + * round_robin_pick_next_ctx - Pick context with load-based selection + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) + * + * Returns: Context index with minimum load + * Time complexity: O(1) + */ +static __u32 round_robin_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_key_domain *domain; + __u32 min_ctx, ctx_idx; - begin = region[SCHED_MODE_ASYNC][i].begin; - end = region[SCHED_MODE_ASYNC][i].end; - ret = session_poll_region(sched_ctx, begin, end, expect, count); - if (unlikely(ret)) - return ret; + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; } - return 0; + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; + + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) + return INVALID_POS; + + return min_ctx; } -/* - * session_poll_policy - The polling policy matches the pick next ctx. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @cfg: The global resoure info. - * @expect: User expect poll msg num. - * @count: The actually poll num. +/** + * round_robin_poll_policy - Poll policy for session scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) * - * The user must init the schedule info through wd_sched_rr_instance, the - * func interval will not check the valid, becouse it will affect performance. + * Returns: Status code */ -static int session_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +static int round_robin_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - struct wd_sched_info *sched_info; - __u32 loop_time = 0; - __u32 last_count = 0; - __u16 i, region_mum; - int ret; + struct wd_sched_key *skey; + __u16 i, tpos, start_pos; + __u32 poll_num, skey_num; + int ret = -WD_EAGAIN; + __u32 sum_count = 0; if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); return -WD_EINVAL; } - if (unlikely(sched_ctx->numa_num > NUMA_NUM_NODES)) { - WD_ERR("invalid: ctx's numa number is %u!\n", sched_ctx->numa_num); - return -WD_EINVAL; + /* Randomize the initial query position. */ + skey_num = sched_ctx->skey_num; + start_pos = sched_get_poll_skey_idx(sched_ctx); + if (!skey_num || start_pos >= sched_ctx->skey_num) { + *count = 0; + return -WD_EAGAIN; } - sched_info = sched_ctx->sched_info; - if (sched_ctx->policy == SCHED_POLICY_DEV) - region_mum = sched_ctx->dev_num; - else - region_mum = sched_ctx->numa_num; - - /* - * Try different region's ctx if we can't receive any - * package last time, it is more efficient. In most - * bad situation, poll ends after MAX_POLL_TIMES loop. - */ - while (++loop_time < MAX_POLL_TIMES) { - for (i = 0; i < region_mum;) { - /* If current numa is not valid, find next. */ - if (!sched_info[i].valid) { - i++; - continue; - } + /* Query the queues on each skey separately. */ + for (i = 0; i < skey_num; i++) { + tpos = (start_pos + i) % skey_num; + skey = sched_ctx->skey[tpos]; - last_count = *count; - ret = session_poll_policy_rr(sched_ctx, i, expect, count); - if (unlikely(ret)) - return ret; + if (unlikely(!skey)) + continue; - if (expect == *count) - return 0; + if (!sched_skey_get_ref(skey)) + continue; - /* - * If no package is received, find next numa, - * otherwise, keep receiving packets at this node. - */ - if (last_count == *count) - i++; + if (__atomic_test_and_set(&skey->poll_lock, __ATOMIC_ACQUIRE)) { + sched_skey_poll_release(skey); + continue; } + + ret = wd_sched_poll_skey(sched_ctx, skey, expect, &poll_num); + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + sched_skey_poll_release(skey); + + sum_count += poll_num; + if (unlikely(ret && ret != -WD_EAGAIN)) + goto poll_err; + + if (sum_count >= expect) + break; } - return 0; +poll_err: + *count = sum_count; + if (ret == -WD_EAGAIN) + return 0; + return ret; } static handle_t sched_none_init(handle_t h_sched_ctx, void *sched_param) @@ -396,12 +1464,11 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, } while (loop_times > 0) { - /* Default use ctx 0 */ loop_times--; ret = sched_ctx->poll_func(0, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) + if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; - else if (ret == -EAGAIN) + else if (ret == -WD_EAGAIN) continue; *count += poll_num; @@ -409,7 +1476,7 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, break; } - return 0; + return WD_SUCCESS; } static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) @@ -420,13 +1487,10 @@ static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) static __u32 sched_single_pick_next_ctx(handle_t sched_ctx, void *sched_key, const int sched_mode) { -#define CTX_ASYNC 1 -#define CTX_SYNC 0 - if (sched_mode) - return CTX_ASYNC; + return 1; else - return CTX_SYNC; + return 0; } static int sched_single_poll_policy(handle_t h_sched_ctx, @@ -443,12 +1507,11 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, } while (loop_times > 0) { - /* Default async mode use ctx 1 */ loop_times--; ret = sched_ctx->poll_func(1, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) + if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; - else if (ret == -EAGAIN) + else if (ret == -WD_EAGAIN) continue; *count += poll_num; @@ -456,260 +1519,483 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, break; } - return 0; + return WD_SUCCESS; } -static bool sched_dev_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +/** + * sched_skey_domain_fill - Initialize or append ctx to session domain + * @key_domain: Target domain (sync or async) + * @ctx_idx: Context index to add + * @policy: Scheduler policy + * @inited: Pointer to boolean tracking whether domain has been initialized + * + * First call: full domain init via wd_sched_skey_domain_init(). + * Subsequent calls: append via wd_sched_skey_add_ctx(). + */ +static void sched_skey_domain_fill(struct wd_sched_key_domain *key_domain, + __u32 ctx_idx, __u8 policy, bool *inited) { - bool found = false; - int i; + int ret; - if (key->mode >= SCHED_MODE_BUTT || key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's device id: %u, mode: %u, type: %u!\n", - key->dev_id, key->mode, key->type); - return false; + if (ctx_idx == INVALID_POS) + return; + + if (!*inited) { + ret = wd_sched_skey_domain_init(key_domain, ctx_idx, policy); + if (!ret) + *inited = true; + return; } - for (i = 0; i < sched_ctx->dev_num; i++) { - if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { - found = true; - break; + (void)wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx); +} + +/** + * sched_skey_common_init - Common scheduler init with init-once + add-rest pattern + * @h_sched_ctx: Scheduler handle + * @sched_param: Scheduling parameters + * @prop_begin: First prop type to iterate (inclusive) + * @prop_end: Last prop type to iterate (inclusive) + * + * Shared by Loop, Hungry, and Instr scheduler init functions. + * Iterates [prop_begin, prop_end], fetching sync/async ctxs per prop type. + * First valid ctx initializes the domain, subsequent ctxs are appended. + */ +static handle_t sched_skey_common_init(handle_t h_sched_ctx, void *sched_param, + __u8 prop_begin, __u8 prop_end) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + struct wd_sched_key *skey; + bool sync_inited = false; + bool async_inited = false; + __u32 sync_ctx, async_ctx; + __u32 req_ctx_num = 0; + handle_t hskey; + __u8 def_prop; + int ret, i; + + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; + } + + skey = (struct wd_sched_key *)hskey; + def_prop = skey->ctx_prop; + for (i = prop_begin; i <= prop_end; i++) { + skey->ctx_prop = i; + sync_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_ASYNC); + if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) + continue; + + if (sync_ctx != INVALID_POS) { + sched_skey_domain_fill(&skey->sync_domain, sync_ctx, + sched_ctx->policy, &sync_inited); } + if (async_ctx != INVALID_POS) + sched_skey_domain_fill(&skey->async_domain, async_ctx, + sched_ctx->policy, &async_inited); + + req_ctx_num += 2; + } + if (!req_ctx_num) { + free(skey); + return (handle_t)(-WD_EINVAL); } - if (!found) { - WD_ERR("invalid: dev_id %u is not registered!\n", key->dev_id); - return false; + skey->ctx_prop = def_prop; + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); + return (handle_t)(-WD_ENOMEM); } - return true; + return hskey; } -/* - * sched_dev_get_region - Get ctx region from ctx_map by the wd comp arg +/** + * skey_sched_init - Initialize Hungry scheduler session + */ +static handle_t skey_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + 0, UADK_ALG_TYPE_MAX - 1); + return hskey; +} + +/** + * skey_sched_pick_next_ctx - Pick context from hungry scheduler with load awareness + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) + * + * Returns: Context with minimum load, or expands if threshold exceeded + * Time complexity: O(1) for selection, O(n) if expansion needed */ -static struct sched_ctx_region *sched_dev_get_region(struct wd_sched_ctx *sched_ctx, - const struct sched_key *key) +static __u32 skey_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) { - struct wd_sched_info *sched_info; - int i, region_id; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_domain_idx_cache *idx_cache; + struct wd_sched_key_domain *domain; + __u32 min_ctx, min_load, ctx_idx; - for (i = 0; i < sched_ctx->dev_num; i++) { - if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { - region_id = sched_ctx->dev_id_map[i].region_id; - sched_info = &sched_ctx->sched_info[region_id]; - if (sched_info->ctx_region[key->mode][key->type].valid) - return &sched_info->ctx_region[key->mode][key->type]; + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; + } + + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; + + idx_cache = &domain->idx_cache; + if (sched_mode == SCHED_MODE_SYNC) { + /* Sync: send+recv atomic, spinlock serialized, use RR */ + if (!idx_cache->valid_count) + return INVALID_POS; + + ctx_idx = __atomic_fetch_add(&idx_cache->rr_ptr, 1, __ATOMIC_RELAXED) % + idx_cache->valid_count; + return idx_cache->idx_list[ctx_idx]; + } + + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) + return INVALID_POS; + + if (skey->ctxs) { + min_load = __atomic_load_n(&skey->ctxs[min_ctx].hw_load, __ATOMIC_RELAXED); + if (min_load >= HW_QUEUE_FULL_DEPTH) + return QUEUE_FULL_POS; + __atomic_fetch_add(&skey->ctxs[min_ctx].hw_load, 1, __ATOMIC_RELAXED); + wd_sched_skey_update_load(idx_cache, ctx_idx, 1); + } else { + min_load = __atomic_load_n(&idx_cache->load_values[ctx_idx], __ATOMIC_RELAXED); + if (min_load >= HW_QUEUE_FULL_DEPTH && idx_cache->load_decreased > 0) + return QUEUE_FULL_POS; + wd_sched_skey_update_load(idx_cache, ctx_idx, 1); + } + /* Check if we need to expand context pool */ + + return min_ctx; +} + +/** + * skey_sched_poll_policy - Poll policy for hungry scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) + * + * Returns: Status code + */ +static int skey_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_key *skey; + __u16 i, tpos, start_pos; + __u32 poll_num, skey_num; + int ret = -WD_EAGAIN; + __u32 sum_count = 0; + + if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { + WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); + return -WD_EINVAL; + } + + /* Randomize the initial query position. */ + skey_num = sched_ctx->skey_num; + start_pos = sched_get_poll_skey_idx(sched_ctx); + if (!skey_num || start_pos >= sched_ctx->skey_num) { + *count = 0; + return -WD_EAGAIN; + } + + /* Query the queues on each skey separately. */ + for (i = 0; i < skey_num; i++) { + tpos = (start_pos + i) % skey_num; + skey = sched_ctx->skey[tpos]; + + if (unlikely(!skey)) + continue; + + if (!sched_skey_get_ref(skey)) + continue; + + if (__atomic_test_and_set(&skey->poll_lock, __ATOMIC_ACQUIRE)) { + sched_skey_poll_release(skey); + continue; } + + ret = wd_sched_poll_skey(sched_ctx, skey, expect, &poll_num); + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + sched_skey_poll_release(skey); + /* + * This query returned 0, indicating the hardware + * likely hasn't finished processing yet. + * Implementing a delay and releasing the CPU is a predictive optimization + */ + if (!poll_num) + usleep(1); + + sum_count += poll_num; + if (ret == -WD_EAGAIN) + continue; + if (unlikely(ret) || sum_count >= expect) + break; } + *count = sum_count; - /* - * If the scheduling domain of dev_id does not exist, - * taskes operations cannot be executed using queues from other devices; - * otherwise, an SMMU error will occur. - */ - return NULL; + return ret; } -/* - * session_dev_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/** + * loop_sched_init - Initialize Loop scheduler session + */ +static handle_t loop_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + 0, UADK_ALG_TYPE_MAX - 1); + return hskey; +} + +/** + * loop_sched_pick_next_ctx - Pick context for loop scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) * - * The user must init the schedule info through wd_sched_rr_instance + * Returns: Context index with minimum load + * Time complexity: O(1) + */ +static __u32 loop_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + return round_robin_pick_next_ctx(h_sched_ctx, sched_key, sched_mode); +} + +static int loop_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + return round_robin_poll_policy(h_sched_ctx, expect, count); +} + +/** + * instr_sched_init - Initialize Instr scheduler session */ -static __u32 session_dev_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, - const int sched_mode) +static handle_t instr_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + UADK_ALG_CE_INSTR, + UADK_ALG_SVE_INSTR); + return hskey; +} + +static __u32 instr_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) { - struct sched_ctx_region *region = NULL; - bool ret; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_key_domain *domain; + __u32 min_ctx, ctx_idx; - key->mode = sched_mode; - ret = sched_dev_key_valid(sched_ctx, key); - if (!ret) + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); return INVALID_POS; + } + + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; - region = sched_dev_get_region(sched_ctx, key); - if (!region) + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) return INVALID_POS; - return sched_get_next_pos_rr(region, NULL); + return min_ctx; +} + +/** + * instr_sched_poll_policy - Poll policy for instruction scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) + * + * Returns: Status code + */ +static int instr_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + return round_robin_poll_policy(h_sched_ctx, expect, count); } static handle_t session_dev_sched_init(handle_t h_sched_ctx, void *sched_param) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; struct sched_params *param = (struct sched_params *)sched_param; - struct sched_key *skey; - unsigned int node; + struct wd_sched_key *skey; + handle_t hskey; + int ret = 0; - if (getcpu(NULL, &node)) { - WD_ERR("failed to get numa node, errno %d!\n", errno); - return (handle_t)(-errno); - } - if (node == (unsigned int)NUMA_NO_NODE) { - WD_ERR("invalid: failed to get numa node for dev sched init!\n"); + if (!param) { + WD_ERR("invalid: dev sched param is NULL!\n"); return (handle_t)(-WD_EINVAL); } - if (!sched_ctx) { - WD_ERR("invalid: sched ctx is NULL!\n"); - return (handle_t)(-WD_EINVAL); + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; } - if (!param) { - WD_DEBUG("no-sva session don't set scheduler parameters!\n"); + skey = (struct wd_sched_key *)hskey; + skey->type = param->type; + skey->dev_id = param->dev_id; + + /* DEV: pinned to a specific device, do not cross prop families */ + ret = session_sched_domain_init(sched_ctx, skey, false); + if (ret) { + WD_ERR("failed to initialize session domains!\n"); + free(skey); return (handle_t)(-WD_EINVAL); } - skey = malloc(sizeof(struct sched_key)); - if (!skey) { - WD_ERR("failed to alloc memory for session sched key!\n"); + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); return (handle_t)(-WD_ENOMEM); } - skey->type = param->type; - skey->dev_id = param->dev_id; + return (handle_t)skey; +} + +/** + * wd_sched_set_param - Set scheduler parameters + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_param: Scheduling parameters (cannot modify per API contract) + */ + +static void wd_sched_set_param(handle_t h_sched_ctx, + void *sched_key, void *sched_param) +{ + struct wd_sched_params *params = (struct wd_sched_params *)sched_param; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; - skey->sync_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); - skey->async_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); - if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { - WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); - goto out; + if (unlikely(!params || !skey)) { + WD_ERR("invalid: sched parmas or skey is NULL!\n"); + return; } - return (handle_t)skey; + skey->pkt_size = params->pkt_size; + skey->is_stream = params->data_mode; + skey->prio_mode = params->prio_mode; + + /* Store compat filtering parameters */ + skey->alg_name = params->alg_name; + skey->ctxs = params->ctxs; -out: - free(skey); - return (handle_t)(-WD_EINVAL); } static struct wd_sched sched_table[SCHED_POLICY_BUTT] = { { .name = "RR scheduler", .sched_policy = SCHED_POLICY_RR, - .sched_init = session_sched_init, - .pick_next_ctx = session_sched_pick_next_ctx, - .poll_policy = session_sched_poll_policy, + .sched_init = round_robin_sched_init, + .pick_next_ctx = round_robin_pick_next_ctx, + .poll_policy = round_robin_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "None scheduler", .sched_policy = SCHED_POLICY_NONE, .sched_init = sched_none_init, .pick_next_ctx = sched_none_pick_next_ctx, .poll_policy = sched_none_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "Single scheduler", .sched_policy = SCHED_POLICY_SINGLE, .sched_init = sched_single_init, .pick_next_ctx = sched_single_pick_next_ctx, .poll_policy = sched_single_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "Device RR scheduler", .sched_policy = SCHED_POLICY_DEV, .sched_init = session_dev_sched_init, - .pick_next_ctx = session_sched_pick_next_ctx, - .poll_policy = session_sched_poll_policy, - } + .pick_next_ctx = round_robin_pick_next_ctx, + .poll_policy = round_robin_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Loop scheduler", + .sched_policy = SCHED_POLICY_LOOP, + .sched_init = loop_sched_init, + .pick_next_ctx = loop_sched_pick_next_ctx, + .poll_policy = loop_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Hungry scheduler", + .sched_policy = SCHED_POLICY_HUNGRY, + .sched_init = skey_sched_init, + .pick_next_ctx = skey_sched_pick_next_ctx, + .poll_policy = skey_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Instr scheduler", + .sched_policy = SCHED_POLICY_INSTR, + .sched_init = instr_sched_init, + .pick_next_ctx = instr_sched_pick_next_ctx, + .poll_policy = instr_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, }; -static int wd_sched_get_nearby_numa_id(struct wd_sched_info *sched_info, int node, int numa_num) -{ - int dis = INT32_MAX; - int valid_id = -1; - int i, tmp; - - for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) { - tmp = numa_distance(node, i); - if (dis > tmp) { - valid_id = i; - dis = tmp; - } - } - } - - return valid_id; -} - -static void wd_sched_map_cpus_to_dev(struct wd_sched_ctx *sched_ctx) -{ - struct wd_sched_info *sched_info = sched_ctx->sched_info; - int i, numa_num = sched_ctx->numa_num; - int *numa_map = sched_ctx->numa_map; - - for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) - numa_map[i] = i; - else - numa_map[i] = wd_sched_get_nearby_numa_id(sched_info, i, numa_num); - } -} - -static int wd_instance_dev_region(struct wd_sched_ctx *sched_ctx, - struct sched_params *param) +static int numa_num_check(__u16 region_num) { - struct wd_sched_info *sched_info; - __u32 region_idx = INVALID_POS; - __u8 type, mode; - __u32 dev_id; - int i; - - dev_id = param->dev_id; - type = param->type; - mode = param->mode; - - /* Check whether dev_id has already been registered. */ - for (i = 0; i < sched_ctx->dev_num; i++) { - if (sched_ctx->dev_id_map[i].dev_id == dev_id) { - region_idx = sched_ctx->dev_id_map[i].region_id; - break; - } - } - - /* If not registered, allocate a new region. */ - if (region_idx == INVALID_POS) { - if (sched_ctx->dev_num >= DEVICE_REGION_MAX) { - WD_ERR("too many devices registered!\n"); - return -WD_EINVAL; - } - - region_idx = sched_ctx->dev_num; - sched_ctx->dev_id_map[region_idx].dev_id = dev_id; - sched_ctx->dev_id_map[region_idx].region_id = region_idx; - sched_ctx->dev_num++; + int max_node; - sched_info = &sched_ctx->sched_info[region_idx]; - } else { - sched_info = &sched_ctx->sched_info[region_idx]; + max_node = numa_max_node() + 1; + if (max_node <= 0) { + WD_ERR("invalid: numa max node is %d!\n", max_node); + return -WD_EINVAL; } - /* Check whether the mode and type have already been registered. */ - if (sched_info->ctx_region[mode][type].valid) { - WD_INFO("device %u mode %u type %u already registered\n", - dev_id, mode, type); - return WD_SUCCESS; + if (!region_num || region_num > max_node) { + WD_ERR("invalid: region number is %u!\n", region_num); + return -WD_EINVAL; } - /* Initialize the scheduling region for this mode and type */ - sched_info->ctx_region[mode][type].begin = param->begin; - sched_info->ctx_region[mode][type].end = param->end; - sched_info->ctx_region[mode][type].last = param->begin; - sched_info->ctx_region[mode][type].valid = true; - sched_info->valid = true; - - pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); - - return WD_SUCCESS; + return 0; } +/** + * wd_sched_rr_instance - External API for scheduling region instance + * @sched: Scheduler (cannot modify per API contract) + * @param: Scheduling parameters (cannot modify per API contract) + * + * Creates scheduling region for given parameters. + */ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *param) { - struct wd_sched_info *sched_info = NULL; struct wd_sched_ctx *sched_ctx = NULL; - __u8 type, mode; - int numa_id; + struct wd_sched_ctx_domain *domain; + int region_key; + __u8 mode; + int ret; if (!sched || !sched->h_sched_ctx || !param) { WD_ERR("invalid: sched or sched_params is NULL!\n"); @@ -721,57 +2007,68 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para return -WD_EINVAL; } - numa_id = param->numa_id; - type = param->type; mode = param->mode; sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; - if (sched_ctx->numa_num > 0 && (numa_id >= sched_ctx->numa_num || - numa_id < 0)) { - WD_ERR("invalid: sched_ctx's numa_id is %d, numa_num is %u!\n", - numa_id, sched_ctx->numa_num); - return -WD_EINVAL; + if (sched_ctx->policy == SCHED_POLICY_DEV) { + region_key = param->dev_id; + if (region_key < 0) { + WD_ERR("invalid: dev_id is %d!\n", region_key); + return -WD_EINVAL; + } + } else { + region_key = param->numa_id; + if (region_key >= sched_ctx->region_num || region_key < 0) { + WD_ERR("invalid: region_key is %d, region_num is %u!\n", + region_key, sched_ctx->region_num); + return -WD_EINVAL; + } } - if (type >= sched_ctx->type_num) { - WD_ERR("invalid: sched_ctx's type is %u, type_num is %u!\n", - type, sched_ctx->type_num); + if (param->type >= sched_ctx->type_num) { + WD_ERR("invalid: type is %u, type_num is %u!\n", + param->type, sched_ctx->type_num); return -WD_EINVAL; } if (mode >= SCHED_MODE_BUTT) { - WD_ERR("invalid: sched_ctx's mode is %u, mode_num is %d!\n", - mode, SCHED_MODE_BUTT); + WD_ERR("invalid: mode is %u, mode_num is %u!\n", + mode, sched_ctx->mode_num); return -WD_EINVAL; } - if (sched_ctx->policy == SCHED_POLICY_DEV) - return wd_instance_dev_region(sched_ctx, param); - - sched_info = &sched_ctx->sched_info[numa_id]; - if (!sched_info->ctx_region[mode]) { - WD_ERR("invalid: ctx_region is NULL, numa: %d, mode: %u!\n", - numa_id, mode); - return -WD_EINVAL; + if (param->ctx_prop < 0 || param->ctx_prop >= UADK_ALG_TYPE_MAX) { + WD_INFO("Info: ctx_prop %d exceeds max type!\n", param->ctx_prop); + param->ctx_prop = UADK_ALG_HW; } - sched_info->ctx_region[mode][type].begin = param->begin; - sched_info->ctx_region[mode][type].end = param->end; - sched_info->ctx_region[mode][type].last = param->begin; - sched_info->ctx_region[mode][type].valid = true; - sched_info->valid = true; + domain = wd_sched_hash_table_insert(sched_ctx->domain_hash_table, + region_key, mode, param->type, + param->ctx_prop); + if (!domain) + return -WD_ENOMEM; - wd_sched_map_cpus_to_dev(sched_ctx); - pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); + ret = wd_sched_domain_add_segment(domain, param->begin, param->end); + if (ret) { + WD_ERR("failed to add segment to domain!\n"); + return ret; + } + domain->valid = true; return WD_SUCCESS; } +/** + * wd_sched_rr_release - External API for scheduler release + * @sched: Scheduler to release (cannot modify per API contract) + * + * Releases all scheduler resources. + */ void wd_sched_rr_release(struct wd_sched *sched) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; - int i, j, region_num; + struct wd_sched_key *skey; + __u32 i; if (!sched) return; @@ -780,59 +2077,49 @@ void wd_sched_rr_release(struct wd_sched *sched) if (!sched_ctx) goto ctx_out; - /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ - if (sched_ctx->policy == SCHED_POLICY_DEV) - region_num = DEVICE_REGION_MAX; - else - region_num = sched_ctx->numa_num; + /* Release all session keys - iterate full array to catch residual entries */ + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + skey = sched_ctx->skey[i]; + if (!skey) + continue; - sched_info = sched_ctx->sched_info; - if (!sched_info) - goto info_out; + sched_ctx->skey[i] = NULL; + __atomic_store_n(&skey->refcount, 0, __ATOMIC_RELAXED); + session_sched_domain_destroy(skey); + free(skey); + } + sched_ctx->skey_num = 0; - for (i = 0; i < region_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - if (sched_info[i].ctx_region[j]) { - free(sched_info[i].ctx_region[j]); - sched_info[i].ctx_region[j] = NULL; - } - } + /* Release hash table */ + if (sched_ctx->domain_hash_table) { + wd_sched_hash_table_destroy(sched_ctx->domain_hash_table); + sched_ctx->domain_hash_table = NULL; } -info_out: + pthread_mutex_destroy(&sched_ctx->skey_lock); free(sched_ctx); + ctx_out: free(sched); - return; } -static int numa_num_check(__u16 numa_num) -{ - int max_node; - - max_node = numa_max_node() + 1; - if (max_node <= 0) { - WD_ERR("invalid: numa max node is %d!\n", max_node); - return -WD_EINVAL; - } - - if (!numa_num || numa_num > max_node) { - WD_ERR("invalid: numa number is %u!\n", numa_num); - return -WD_EINVAL; - } - - return 0; -} - +/** + * wd_sched_rr_alloc - External API for scheduler allocation + * @sched_type: Scheduling policy type (cannot modify per API contract) + * @type_num: Number of operation types (cannot modify per API contract) + * @region_num: Number of regions (cannot modify per API contract) + * @func: Poll function (cannot modify per API contract) + * + * Allocates and initializes scheduler with single global hash table. + */ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, - __u16 numa_num, user_poll_func func) + __u16 region_num, user_poll_func func) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; struct wd_sched *sched; - int region_num; - int i, j; + __u32 estimated_entries; + __u32 i; if (sched_type >= SCHED_POLICY_BUTT || !type_num) { WD_ERR("invalid: sched_type is %u or type_num is %u!\n", @@ -846,63 +2133,62 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, return NULL; } - if (sched_type == SCHED_POLICY_DEV) - region_num = DEVICE_REGION_MAX; - else - region_num = numa_num; - - sched_ctx = calloc(1, sizeof(struct wd_sched_ctx) + - sizeof(struct wd_sched_info) * region_num); + sched_ctx = calloc(1, sizeof(struct wd_sched_ctx)); if (!sched_ctx) { WD_ERR("failed to alloc memory for sched_ctx!\n"); goto err_out; } - /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ + /* Cache dimension parameters */ + sched_ctx->type_num = type_num; + sched_ctx->mode_num = SCHED_MODE_BUTT; + sched_ctx->region_num = region_num; + sched_ctx->policy = sched_type; + if (sched_type == SCHED_POLICY_DEV) { - sched_ctx->numa_num = 0; - sched_ctx->dev_num = 0; - for (i = 0; i < DEVICE_REGION_MAX; i++) { - sched_ctx->dev_id_map[i].dev_id = INVALID_POS; - sched_ctx->dev_id_map[i].region_id = INVALID_POS; - } + /* Device mode: region_num is actually device count */ + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } else { - sched_ctx->numa_num = numa_num; - sched_ctx->dev_num = 0; - if (numa_num_check(sched_ctx->numa_num)) + /* NUMA mode: validate region_num */ + if (numa_num_check(region_num)) goto err_out; + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } - sched->h_sched_ctx = (handle_t)sched_ctx; - if (sched_type == SCHED_POLICY_NONE || - sched_type == SCHED_POLICY_SINGLE) - goto simple_ok; - - sched_info = sched_ctx->sched_info; - for (i = 0; i < region_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - sched_info[i].ctx_region[j] = - calloc(1, sizeof(struct sched_ctx_region) * type_num); - if (!sched_info[i].ctx_region[j]) - goto err_out; - } + /* Create single global hash table */ + sched_ctx->domain_hash_table = wd_sched_hash_table_create(estimated_entries); + if (!sched_ctx->domain_hash_table) { + WD_ERR("failed to create hash table!\n"); + goto ctx_out; } -simple_ok: sched_ctx->poll_func = func; - sched_ctx->policy = sched_type; - sched_ctx->type_num = type_num; - memset(sched_ctx->numa_map, -1, sizeof(int) * NUMA_NUM_NODES); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) + sched_ctx->skey[i] = NULL; + + if (pthread_mutex_init(&sched_ctx->skey_lock, NULL)) { + WD_ERR("failed to init skey_lock!\n"); + goto err_destroy_hash; + } + sched_ctx->skey_num = 0; + + sched->h_sched_ctx = (handle_t)sched_ctx; sched->sched_init = sched_table[sched_type].sched_init; + sched->sched_uninit = sched_table[sched_type].sched_uninit; sched->pick_next_ctx = sched_table[sched_type].pick_next_ctx; sched->poll_policy = sched_table[sched_type].poll_policy; sched->sched_policy = sched_type; sched->name = sched_table[sched_type].name; + sched->set_param = sched_table[sched_type].set_param; return sched; +err_destroy_hash: + wd_sched_hash_table_destroy(sched_ctx->domain_hash_table); +ctx_out: + free(sched_ctx); err_out: - wd_sched_rr_release(sched); + free(sched); return NULL; } -- 2.43.0