[PATCH 01/16] uadk: rewrite scheduler with scheduling domain solution
From: Longfang Liu <liulongfang@huawei.com> Conduct a comprehensive update of the scheduler's internal triple-array-based pooling scheme. Adopt new data organization methods such as hash buckets and segment linked lists to achieve efficient insertion and query operations. Implement pre-fetch queue processing for business thread sessions, creating thread-level sub-queue pools for fast packet reception queries. Provide new scheduling algorithms for the new UADK framework. Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- include/wd_alg.h | 11 +- include/wd_alg_common.h | 5 +- include/wd_internal.h | 18 +- include/wd_sched.h | 10 +- wd_sched.c | 2304 ++++++++++++++++++++++++++++++--------- 5 files changed, 1832 insertions(+), 516 deletions(-) diff --git a/include/wd_alg.h b/include/wd_alg.h index 18503ca..1ae1dae 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -64,10 +64,13 @@ extern "C" { #endif enum alg_dev_type { - UADK_ALG_SOFT = 0x0, - UADK_ALG_CE_INSTR = 0x1, - UADK_ALG_SVE_INSTR = 0x2, - UADK_ALG_HW = 0x3 + UADK_ALG_HW = 0x0, + UADK_ALG_CE_INSTR = 0x1, + UADK_ALG_SVE_INSTR = 0x2, + UADK_ALG_SOFT = 0x3, + UADK_ALG_NPU = 0x4, + UADK_ALG_GPU = 0x5, + UADK_ALG_TYPE_MAX, }; /* diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index a294877..ebba7f8 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -41,6 +41,7 @@ extern "C" { /* Key size of digest */ #define MAX_HMAC_KEY_SIZE 128U +#define STATUS_ENABLE (void *)0x1 enum alg_task_type { TASK_MIX = 0x0, @@ -140,7 +141,7 @@ struct wd_ctx_params { }; /* - * struct wd_comp_sched - Define a scheduler. + * struct wd_sched - Define a scheduler. * @name: Name of this scheduler. * @sched_policy: Method for scheduler to perform scheduling * @sched_init: inited the scheduler input parameters. @@ -157,10 +158,12 @@ struct wd_sched { const char *name; int sched_policy; handle_t (*sched_init)(handle_t h_sched_ctx, void *sched_param); + void (*sched_uninit)(handle_t h_sched_ctx, handle_t h_sched_key); __u32 (*pick_next_ctx)(handle_t h_sched_ctx, void *sched_key, const int sched_mode); int (*poll_policy)(handle_t h_sched_ctx, __u32 expect, __u32 *count); + void (*set_param)(handle_t h_sched_ctx, void *sched_key, void *sched_param); handle_t h_sched_ctx; }; diff --git a/include/wd_internal.h b/include/wd_internal.h index 4f19d3a..62cf1a9 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -7,8 +7,10 @@ #define WD_INTERNAL_H #include <pthread.h> +#include <stdatomic.h> #include <stdbool.h> #include "wd.h" +#include "wd_alg.h" #ifdef __cplusplus extern "C" { @@ -41,11 +43,14 @@ struct wd_ce_ctx { }; struct wd_ctx_internal { - handle_t ctx; __u8 op_type; __u8 ctx_mode; + __u8 ctx_type; + handle_t ctx; __u16 sqn; pthread_spinlock_t lock; + struct wd_alg_driver *drv; + __u32 hw_load; }; struct wd_ctx_config_internal { @@ -64,6 +69,17 @@ struct wd_datalist { struct wd_datalist *next; }; +struct wd_sched_params { + __u32 pkt_size; + /* block mode or stream mode */ + __u16 data_mode; + __u16 prio_mode; + + /* Compat filtering parameters for session-ctx matching */ + const char *alg_name; + struct wd_ctx_internal *ctxs; +}; + int memcmp_consttime(const void *s1, const void *s2, size_t n); #ifdef __cplusplus diff --git a/include/wd_sched.h b/include/wd_sched.h index 5baecd3..f81025f 100644 --- a/include/wd_sched.h +++ b/include/wd_sched.h @@ -13,7 +13,8 @@ extern "C" { #endif -#define INVALID_POS 0xFFFFFFFF +#define INVALID_POS 0xFFFF +#define QUEUE_FULL_POS 0x1FFF /* The global policy type */ enum sched_policy_type { @@ -25,6 +26,12 @@ enum sched_policy_type { SCHED_POLICY_SINGLE, /* requests will be sent to ctxs and dev_id */ SCHED_POLICY_DEV, + /* Hard calculation and soft calculation interval loop call */ + SCHED_POLICY_LOOP, + /* Perform heterogeneous calculations through ctx of session key */ + SCHED_POLICY_HUNGRY, + /* Instructions to accelerate heterogeneous computing */ + SCHED_POLICY_INSTR, SCHED_POLICY_BUTT, }; @@ -35,6 +42,7 @@ struct sched_params { __u32 begin; __u32 end; __u32 dev_id; + int ctx_prop; }; typedef int (*user_poll_func)(__u32 pos, __u32 expect, __u32 *count); diff --git a/wd_sched.c b/wd_sched.c index 19936fd..2efa2aa 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -1,374 +1,1442 @@ // SPDX-License-Identifier: Apache-2.0 /* - * Copyright 2020-2021 Huawei Technologies Co.,Ltd. All rights reserved. + * Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved. * Copyright 2020-2021 Linaro ltd. + * + * Scheduler: Simplified Pure Hash Table with Dynamic Context Expansion + * + * Key improvements: + * - Single global hash table with (region_id, mode, op_type, prop) dimensions + * - Segment list for non-contiguous ctx ranges + * - Dual-domain queues for session key. + * - Dynamic ctx expansion in HUNGRY mode based on load threshold + * - Packet reception is handled through the active queues in the session key. + * - Simplified sched_init: only allocate one sync + one async ctx + * - Removed redundant wd_sched_info layer */ #define _GNU_SOURCE #include <stdlib.h> #include <stdbool.h> +#include <string.h> #include <sched.h> #include <numa.h> +#include <limits.h> +#include <pthread.h> #include "wd_sched.h" +#include "wd_alg.h" +#include "wd_internal.h" -#define MAX_POLL_TIMES 1000 +#define MAX_POLL_TIMES 1000 +#define HUNGRY_LOAD_THRESHOLD 256 +#define SKEY_CTX_MAX_NUM 16 +#define SKEY_MAX_THREAD_NUM 64 +#define SKEY_LOAD_UPDATE_INTERVAL 1 +#define HW_QUEUE_FULL_DEPTH 1024 +#define MAX_NUMA_NODES (NUMA_NUM_NODES >> 5) + +/* ============================================================================ + * Hash Table Configuration + * ============================================================================ + */ +#define WD_SCHED_MAX_BUCKETS 512 +#define WD_SCHED_MIN_BUCKETS 32 +#define WD_SCHED_LOAD_FACTOR 0.75f +#define HASH_PRIME1 73 +#define HASH_PRIME2 13 +#define HASH_PRIME3 7 +#define HASH_PRIME4 11 + +/* ============================================================================ + * Scheduling Region Mode + * ============================================================================ + */ enum sched_region_mode { SCHED_MODE_SYNC = 0, SCHED_MODE_ASYNC = 1, SCHED_MODE_BUTT }; -/* - * sched_key - The key if schedule region. - * @numa_id: The schedule numa region id. - * @mode: Sync mode:0, async_mode:1 - * @type: Service type , the value must smaller than type_num. - * @sync_ctxid: alloc ctx id for sync mode - * @async_ctxid: alloc ctx id for async mode - */ -struct sched_key { - int numa_id; - __u8 type; - __u8 mode; - __u32 sync_ctxid; - __u32 async_ctxid; - __u32 dev_id; -}; +/* ============================================================================ + * Segment List for Domain Index Organization + * ============================================================================ + */ -/* - * struct sched_ctx_range - define one ctx pos. - * @begin: the start pos in ctxs of config. - * @end: the end pos in ctxx of config. - * @last: the last one which be distributed. - * @valid: the region used flag. - * @lock: lock the currentscheduling region. - */ -struct sched_ctx_region { +/** + * wd_sched_ctx_segment - Contiguous segment of ctx indices in domain + * @begin: Start index of this segment + * @end: End index of this segment (inclusive) + * @next: Pointer to next segment in the linked list + * + * Supports non-contiguous ctx ranges via segment list. + */ +struct wd_sched_ctx_segment { __u32 begin; __u32 end; - __u32 last; + struct wd_sched_ctx_segment *next; +}; + +/* ============================================================================ + * Session key domain cache processing. + * ============================================================================ + */ + +/** + * wd_sched_domain_idx_cache - Simplified fixed array cache for skey domains + * + * Design principles: + * - Fixed array for cache-friendly memory layout + * - Atomic operations for lock-free load tracking + * - Simple RR and load balancing strategies + * - Maximum 16 queues per thread (typical usage) + */ +struct wd_sched_domain_idx_cache { + /* Queue index array */ + __u32 idx_list[SKEY_CTX_MAX_NUM]; /* Array of ctx indices */ + __u32 load_values[SKEY_CTX_MAX_NUM]; /* Atomic load counters */ + __u32 valid_count; /* Number of valid queues */ + + /* Scheduling state */ + __u32 rr_ptr; /* Round-robin pointer */ + __u32 min_load_idx; /* Cached min load index */ + __u32 op_counter; /* Operation counter for updates */ + __u8 load_decreased; /* Non-zero if poll ever decremented load */ + + /* Configuration */ + __u32 update_interval; /* Min load update interval */ + __u8 policy; /* Scheduling policy */ + + /* Synchronization */ + pthread_mutex_t cache_lock; /* Lock for structure modifications */ +}; + +/** + * wd_sched_ctx_domain - Scheduling domain with four dimensions + * @region_id: Region identifier (numa_id or device_id) + * @mode: Context mode (SYNC/ASYNC) + * @op_type: Operation type + * @prop: Property (e.g., device type: HW, CE, SOFT) + * @segments: Linked list of context ranges + * @segment_count: Number of segments + * @total_ctx_count: Total contexts across all segments + * @current_segment: Current segment pointer for round-robin + * @current_pos: Current position within segment + * @valid: Domain validity flag + * @lock: Synchronization spinlock + */ +struct wd_sched_ctx_domain { + int region_id; + __u8 mode; + __u32 op_type; + __u8 prop; + + struct wd_sched_ctx_segment *segments; + __u32 segment_count; + __u32 total_ctx_count; + + struct wd_sched_ctx_segment *current_segment; + __u32 current_pos; bool valid; + pthread_mutex_t lock; }; -/* - * wd_sched_info - define the context of the scheduler. - * @ctx_region: define the map for the comp ctxs, using for quickly search. - * the x range: two(sync and async), the y range: - * two(e.g. comp and uncomp) the map[x][y]'s value is the ctx - * begin and end pos. - * @valid: the region used flag. - */ -struct wd_sched_info { - struct sched_ctx_region *ctx_region[SCHED_MODE_BUTT]; - bool valid; +/** + * wd_sched_domain_hash_node - Hash table collision chain node + */ +struct wd_sched_domain_hash_node { + struct wd_sched_ctx_domain domain; + struct wd_sched_domain_hash_node *next; }; -struct dev_region_map { +/** + * wd_sched_domain_hash_table - Pure dynamic hash table for scheduling domains + * @buckets: Hash table bucket array + * @bucket_size: Number of buckets + * @lock: Read-write lock for concurrent access + */ +struct wd_sched_domain_hash_table { + struct wd_sched_domain_hash_node **buckets; + __u32 bucket_size; + pthread_mutex_t lock; +}; + +/* ============================================================================ + * Dual-Domain Structure for Session Key + * ============================================================================ + */ + +/** + * wd_sched_key_domain - Session domain with min-heap + * @idx_cache: Index cache with min-heap for load-based selection + * @lock: Synchronization spinlock + * @expanded_count: Track how many times ctx has been expanded + */ +struct wd_sched_key_domain { + struct wd_sched_domain_idx_cache idx_cache; + pthread_mutex_t lock; + __u32 expanded_count; +}; + +/** + * wd_sched_key - Session-level scheduling key + * @region_id: Region identifier + * @type: Operation type + * @mode: Current mode (SYNC/ASYNC) + * @dev_id: Device identifier (for SCHED_POLICY_DEV) + * @ctx_prop: Context property + * @is_stream: Stream mode flag + * @prio_mode: Priority mode + * @pkt_size: Current packet size + * @sync_domain: Min-heap domain for sync contexts + * @async_domain: Min-heap domain for async contexts + * @lock: Synchronization spinlock + */ +struct wd_sched_key { + int region_id; + __u8 type; + __u8 mode; __u32 dev_id; - __u32 region_id; + __u8 ctx_prop; + __u16 is_stream; + __u16 prio_mode; + __u32 pkt_size; + + struct wd_sched_key_domain sync_domain; + struct wd_sched_key_domain async_domain; + + pthread_mutex_t lock; + __u32 poll_lock; + __u32 refcount; + + /* Compat filtering parameters for session-ctx matching */ + const char *alg_name; + struct wd_ctx_internal *ctxs; }; -/* - * wd_sched_ctx - define the context of the scheduler. - * @policy: define the policy of the scheduler. - * @numa_num: the max numa numbers of the scheduler. - * @type_num: the max operation types of the scheduler. - * @poll_func: the task's poll operation function. - * @numa_map: a map of cpus to devices. - * @sched_info: the context of the scheduler. + +/** + * wd_sched_ctx - Main scheduler context + * @policy: Scheduling policy type + * @type_num: Number of operation types + * @mode_num: Number of modes (SYNC/ASYNC) + * @region_num: Number of regions (numa or devices) + * @poll_func: Poll function for receiving responses + * @domain_hash_table: Global hash table for all domains + * @skey_num: Number of active session keys + * @skey_lock: Lock for skey array + * @skey: Array of session keys */ struct wd_sched_ctx { __u32 policy; __u32 type_num; - __u16 numa_num; - __u16 dev_num; + __u32 mode_num; + __u16 region_num; + user_poll_func poll_func; - int numa_map[NUMA_NUM_NODES]; - struct dev_region_map dev_id_map[DEVICE_REGION_MAX]; - struct wd_sched_info sched_info[0]; + struct wd_sched_domain_hash_table *domain_hash_table; + + __u32 skey_num; + pthread_mutex_t skey_lock; + struct wd_sched_key *skey[SKEY_MAX_THREAD_NUM]; }; -static bool sched_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +/* ============================================================================ + * Hash Table Core Operations + * ============================================================================ + */ + +static bool wd_sched_is_prime(__u32 n) { - if (key->numa_id >= sched_ctx->numa_num || key->mode >= SCHED_MODE_BUTT || - key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's numa: %d, mode: %u, type: %u!\n", - key->numa_id, key->mode, key->type); + __u32 i; + + if (n <= 1) return false; + if (n <= 3) + return true; + if (n % 2 == 0 || n % 3 == 0) + return false; + + for (i = 5; i * i <= n; i += 6) { + if (n % i == 0 || n % (i + 2) == 0) + return false; } return true; } -/* - * sched_get_ctx_range - Get ctx range from ctx_map by the wd comp arg +static __u32 wd_sched_find_prime(__u32 n) +{ + while (!wd_sched_is_prime(n)) + n++; + return n; +} + +static __u32 wd_sched_compute_bucket_size(__u32 estimated_entries) +{ + __u32 target_size; + + target_size = (estimated_entries * 4) / 3; + + if (target_size < WD_SCHED_MIN_BUCKETS) + target_size = WD_SCHED_MIN_BUCKETS; + if (target_size > WD_SCHED_MAX_BUCKETS) + target_size = WD_SCHED_MAX_BUCKETS; + + return wd_sched_find_prime(target_size); +} + +/** + * wd_sched_hash_compute - Compute hash value for four-dimensional domain key + * @region_id: Region identifier + * @mode: Context mode + * @op_type: Operation type + * @prop: Property + * @bucket_size: Hash table bucket count + * + * Combines four dimensions using prime number multipliers. */ -static struct sched_ctx_region *sched_get_ctx_range(struct wd_sched_ctx *sched_ctx, - const struct sched_key *key) +static inline __u32 wd_sched_hash_compute(int region_id, __u8 mode, + __u32 op_type, __u8 prop, __u32 bucket_size) { - struct wd_sched_info *sched_info; - int numa_id; + __u32 hash; - sched_info = sched_ctx->sched_info; - if (key->numa_id >= 0 && - sched_info[key->numa_id].ctx_region[key->mode][key->type].valid) - return &sched_info[key->numa_id].ctx_region[key->mode][key->type]; + hash = (region_id * HASH_PRIME1) + (mode * HASH_PRIME2) + + (op_type * HASH_PRIME3) + (prop * HASH_PRIME4); + return hash % bucket_size; +} + +static inline bool wd_sched_domain_key_match( + int region_id1, __u8 mode1, __u32 op_type1, __u8 prop1, + int region_id2, __u8 mode2, __u32 op_type2, __u8 prop2) +{ + return (region_id1 == region_id2 && mode1 == mode2 && + op_type1 == op_type2 && prop1 == prop2); +} + +/** + * wd_sched_hash_table_create - Create hash table + * @estimated_entries: Estimated number of entries + * + * Returns: Initialized hash table or NULL on error + */ +static struct wd_sched_domain_hash_table * +wd_sched_hash_table_create(__u32 estimated_entries) +{ + struct wd_sched_domain_hash_table *table; + __u32 bucket_size; + int ret; - /* If the key->numa_id is not exist, we should scan for a region */ - for (numa_id = 0; numa_id < sched_ctx->numa_num; numa_id++) { - if (sched_info[numa_id].ctx_region[key->mode][key->type].valid) - return &sched_info[numa_id].ctx_region[key->mode][key->type]; + table = calloc(1, sizeof(*table)); + if (!table) + return NULL; + + bucket_size = wd_sched_compute_bucket_size(estimated_entries); + + table->buckets = calloc(bucket_size, sizeof(*table->buckets)); + if (!table->buckets) { + free(table); + return NULL; } - return NULL; + table->bucket_size = bucket_size; + ret = pthread_mutex_init(&table->lock, NULL); + if (ret) { + free(table->buckets); + free(table); + return NULL; + } + + return table; } -/* - * sched_get_next_pos_rr - Get next resource pos by RR schedule. - * The second para is reserved for future. +static void wd_sched_hash_table_destroy(struct wd_sched_domain_hash_table *table) +{ + struct wd_sched_domain_hash_node *node, *next; + struct wd_sched_ctx_segment *seg, *next_seg; + __u32 i; + + if (!table) + return; + + for (i = 0; i < table->bucket_size; i++) { + node = table->buckets[i]; + while (node) { + next = node->next; + + /* Release segment linked list */ + seg = node->domain.segments; + while (seg) { + next_seg = seg->next; + free(seg); + seg = next_seg; + } + + pthread_mutex_destroy(&node->domain.lock); + free(node); + node = next; + } + } + + pthread_mutex_destroy(&table->lock); + free(table->buckets); + free(table); +} + +static struct wd_sched_ctx_domain * +wd_sched_hash_table_lookup(struct wd_sched_domain_hash_table *table, + int region_id, __u8 mode, __u32 op_type, __u8 prop) +{ + struct wd_sched_domain_hash_node *node; + struct wd_sched_ctx_domain *domain = NULL; + __u32 hash_idx; + + if (!table) + return NULL; + + pthread_mutex_lock(&table->lock); + hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size); + node = table->buckets[hash_idx]; + while (node) { + if (wd_sched_domain_key_match( + node->domain.region_id, node->domain.mode, node->domain.op_type, + node->domain.prop, region_id, mode, op_type, prop)) { + domain = &node->domain; + break; + } + node = node->next; + } + pthread_mutex_unlock(&table->lock); + + return domain; +} + +static struct wd_sched_ctx_domain * +wd_sched_hash_table_insert(struct wd_sched_domain_hash_table *table, + int region_id, __u8 mode, __u32 op_type, __u8 prop) +{ + struct wd_sched_domain_hash_node *new_node; + struct wd_sched_ctx_domain *existing; + __u32 hash_idx; + int ret; + + if (!table) + return NULL; + + existing = wd_sched_hash_table_lookup(table, region_id, mode, op_type, prop); + if (existing) + return existing; + + pthread_mutex_lock(&table->lock); + hash_idx = wd_sched_hash_compute(region_id, mode, op_type, prop, table->bucket_size); + /* Alloc and initialize new domain */ + new_node = calloc(1, sizeof(*new_node)); + if (!new_node) { + pthread_mutex_unlock(&table->lock); + return NULL; + } + + /* Initialize new domain */ + new_node->domain.region_id = region_id; + new_node->domain.mode = mode; + new_node->domain.op_type = op_type; + new_node->domain.prop = prop; + new_node->domain.segments = NULL; + new_node->domain.segment_count = 0; + new_node->domain.total_ctx_count = 0; + new_node->domain.current_segment = NULL; + new_node->domain.current_pos = 0; + new_node->domain.valid = false; + + ret = pthread_mutex_init(&new_node->domain.lock, NULL); + if (ret) { + pthread_mutex_unlock(&table->lock); + free(new_node); + return NULL; + } + + new_node->next = table->buckets[hash_idx]; + table->buckets[hash_idx] = new_node; + pthread_mutex_unlock(&table->lock); + + return &new_node->domain; +} + +/* ============================================================================ + * Segment List Operations + * ============================================================================ + */ + +/** + * wd_sched_domain_add_segment - Add context range segment to domain + * @domain: Target domain + * @begin: Start context index + * @end: End context index (inclusive) + * + * Supports non-contiguous context ranges via segment list. */ -static __u32 sched_get_next_pos_rr(struct sched_ctx_region *region, void *para) +static int wd_sched_domain_add_segment(struct wd_sched_ctx_domain *domain, + __u32 begin, __u32 end) { + struct wd_sched_ctx_segment *seg, *new_seg; + + if (!domain || begin > end) + return -WD_EINVAL; + + new_seg = calloc(1, sizeof(*new_seg)); + if (!new_seg) + return -WD_ENOMEM; + + new_seg->begin = begin; + new_seg->end = end; + new_seg->next = NULL; + + pthread_mutex_lock(&domain->lock); + + /* Append to segment list tail */ + if (!domain->segments) { + domain->segments = new_seg; + } else { + seg = domain->segments; + while (seg->next) + seg = seg->next; + seg->next = new_seg; + } + + domain->segment_count++; + domain->total_ctx_count += (end - begin + 1); + + /* Initialize polling state */ + if (!domain->current_segment) + domain->current_segment = domain->segments; + + pthread_mutex_unlock(&domain->lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_domain_get_next_rr - Get next context via round-robin from domain + * @domain: Source domain + * + * Returns: Next global queue index in round-robin order + * Time complexity: O(1) + */ +static __u32 wd_sched_domain_get_next_rr(struct wd_sched_ctx_domain *domain) +{ + __u32 ctx_idx; __u32 pos; - pthread_mutex_lock(®ion->lock); + if (!domain || !domain->segments || !domain->total_ctx_count) + return INVALID_POS; - pos = region->last; + pthread_mutex_lock(&domain->lock); + if (!domain->current_segment) + domain->current_segment = domain->segments; - if (pos < region->end) - region->last++; - else - region->last = region->begin; + pos = domain->current_pos; + + /* Calculate global queue number: segment.begin + relative position */ + ctx_idx = domain->current_segment->begin + pos; - pthread_mutex_unlock(®ion->lock); + /* Move to next position */ + if (pos + 1 < domain->current_segment->end - domain->current_segment->begin + 1) { + /* Within same segment */ + domain->current_pos = pos + 1; + } else if (domain->current_segment->next) { + /* Move to next segment */ + domain->current_segment = domain->current_segment->next; + domain->current_pos = 0; + } else { + /* Loop back to beginning */ + domain->current_segment = domain->segments; + domain->current_pos = 0; + } + + pthread_mutex_unlock(&domain->lock); - return pos; + return ctx_idx; } -/* - * session_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/* ============================================================================ + * SKey Domain Cache Management Functions + * ============================================================================ + */ +/** + * wd_sched_skey_cache_init - Initialize skey domain cache + * @cache: Pointer to cache structure + * @policy: Scheduling policy * @sched_type: Scheduling policy type (cannot modify per + * API contract) + * + * Initialize fixed array cache with invalid positions and zero loads. + */ +static int wd_sched_skey_cache_init(struct wd_sched_domain_idx_cache *cache, + __u8 policy) +{ + int i; + + if (!cache) { + WD_ERR("invalid: cache pointer is NULL!\n"); + return -WD_EINVAL; + } + + /* Initialize array with invalid positions */ + for (i = 0; i < SKEY_CTX_MAX_NUM; i++) { + cache->idx_list[i] = INVALID_POS; + __atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED); + } + + /* Initialize atomic counters */ + __atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->op_counter, 0, __ATOMIC_RELAXED); + cache->load_decreased = 0; + + /* Set configuration */ + cache->valid_count = 0; + cache->update_interval = SKEY_LOAD_UPDATE_INTERVAL; + cache->policy = policy; + + /* Initialize structure lock */ + if (pthread_mutex_init(&cache->cache_lock, NULL)) { + WD_ERR("failed to init cache lock!\n"); + return -WD_EINVAL; + } + + return WD_SUCCESS; +} +/** + * wd_sched_skey_cache_uninit - Cleanup skey domain cache + * @cache: Pointer to cache structure + * + * Release resources and reset cache state. + */ +static void wd_sched_skey_cache_uninit(struct wd_sched_domain_idx_cache *cache) +{ + if (!cache) + return; + + pthread_mutex_destroy(&cache->cache_lock); + + /* Reset cache state */ + for (int i = 0; i < SKEY_CTX_MAX_NUM; i++) { + cache->idx_list[i] = INVALID_POS; + __atomic_store_n(&cache->load_values[i], 0, __ATOMIC_RELAXED); + } + + cache->valid_count = 0; + cache->load_decreased = 0; +} +/** + * wd_sched_skey_add_ctx - Add ctx to skey domain cache + * @cache: Pointer to cache structure + * @ctx_id: Context ID to add + * + * Add ctx to next available position in fixed array. + * Returns 0 on success, negative error code on failure. + */ +static int wd_sched_skey_add_ctx(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_id) +{ + __u32 i; + + if (!cache || ctx_id == INVALID_POS) { + WD_ERR("invalid: parameters are NULL!\n"); + return -WD_EINVAL; + } + + pthread_mutex_lock(&cache->cache_lock); + /* Check if cache is full */ + if (cache->valid_count >= SKEY_CTX_MAX_NUM) { + pthread_mutex_unlock(&cache->cache_lock); + WD_ERR("invalid: skey cache full, cannot add more queues!\n"); + return -WD_EINVAL; + } + + /* Check for duplicate ctx_id */ + for (i = 0; i < cache->valid_count; i++) { + if (cache->idx_list[i] == ctx_id) { + WD_ERR("invalid: context %u already exists in skey cache at pos %u!\n", + ctx_id, i); + pthread_mutex_unlock(&cache->cache_lock); + return -WD_EEXIST; + } + } + + /* Update min load index if as the new ctx */ + __atomic_store_n(&cache->min_load_idx, cache->valid_count, __ATOMIC_RELAXED); + + /* Add to next available position */ + cache->idx_list[cache->valid_count] = ctx_id; + __atomic_store_n(&cache->load_values[cache->valid_count], 0, __ATOMIC_RELAXED); + cache->valid_count++; + pthread_mutex_unlock(&cache->cache_lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_skey_remove_ctx - Remove ctx from skey domain cache + * @cache: Pointer to cache structure + * @ctx_id: Context ID to remove + * + * Remove ctx by shifting array elements to maintain continuity. + * Returns 0 on success, negative error code if not found. + */ +static int wd_sched_skey_remove_ctx(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_id) +{ + __u32 i, current_min; + int found = 0; + + if (!cache) { + WD_ERR("invalid: cache pointer is NULL!\n"); + return -WD_EINVAL; + } + + pthread_mutex_lock(&cache->cache_lock); + /* Find and remove the ctx */ + for (i = 0; i < cache->valid_count; i++) { + if (cache->idx_list[i] == ctx_id) { + found = 1; + break; + } + } + + if (!found) { + WD_ERR("invalid: context %u not found in skey cache!\n", ctx_id); + pthread_mutex_unlock(&cache->cache_lock); + return -WD_ENODEV; + } + + /* Shift remaining elements to fill the gap */ + for (; i < cache->valid_count - 1; i++) { + cache->idx_list[i] = cache->idx_list[i + 1]; + __atomic_store_n(&cache->load_values[i], + __atomic_load_n(&cache->load_values[i + 1], __ATOMIC_RELAXED), + __ATOMIC_RELAXED); + } + + /* Clear last position */ + cache->idx_list[cache->valid_count - 1] = INVALID_POS; + __atomic_store_n(&cache->load_values[cache->valid_count - 1], 0, __ATOMIC_RELAXED); + cache->valid_count--; + + /* Reset pointers if cache becomes empty */ + if (!cache->valid_count) { + __atomic_store_n(&cache->rr_ptr, 0, __ATOMIC_RELAXED); + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + } else { + /* Adjust min load index if necessary */ + current_min = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED); + if (current_min >= cache->valid_count) + __atomic_store_n(&cache->min_load_idx, 0, __ATOMIC_RELAXED); + } + pthread_mutex_unlock(&cache->cache_lock); + + return WD_SUCCESS; +} + +/** + * wd_sched_update_min_load - Update cached min load index + * @cache: Pointer to cache structure + * @ctxs: Context array for hw_load lookup, or NULL to use cache->load_values * - * The user must init the schedule info through wd_sched_rr_instance + * Scan valid queues to find the one with minimum load. + * When ctxs is provided, reads per-ctx hw_load for cross-session accuracy. */ -static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, - const int sched_mode) +static void wd_sched_update_min_load(struct wd_sched_domain_idx_cache *cache, + struct wd_ctx_internal *ctxs) { - struct sched_ctx_region *region = NULL; - bool ret; + __u32 min_load = UINT_MAX; + __u32 min_idx = 0; + __u32 i, load; - key->mode = sched_mode; - ret = sched_key_valid(sched_ctx, key); - if (!ret) + if (!cache->valid_count) + return; + + for (i = 0; i < cache->valid_count; i++) { + if (ctxs) + load = __atomic_load_n(&ctxs[cache->idx_list[i]].hw_load, + __ATOMIC_RELAXED); + else + load = __atomic_load_n(&cache->load_values[i], __ATOMIC_RELAXED); + if (load < min_load) { + min_load = load; + min_idx = i; + } + } + + __atomic_store_n(&cache->min_load_idx, min_idx, __ATOMIC_RELAXED); +} + +/** + * wd_sched_skey_pick_next - Pick next ctx from skey domain cache + * @cache: Pointer to cache structure + * @ctx_idx: Output index within cache array + * @ctxs: Context array for hw_load lookup, or NULL + * + * Select next ctx based on scheduling policy: + * - RR: Simple round-robin selection + * - HUNGRY: Choose ctx with minimum load (hw_load if ctxs available) + * + * Returns selected ctx index, or INVALID_POS if no valid ctx. + */ +static __u32 wd_sched_skey_pick_next(struct wd_sched_domain_idx_cache *cache, + __u32 *ctx_idx, struct wd_ctx_internal *ctxs) +{ + __u32 selected_idx; + __u32 op_count; + + if (!cache || !cache->valid_count) return INVALID_POS; - region = sched_get_ctx_range(sched_ctx, key); - if (!region) + switch (cache->policy) { + case SCHED_POLICY_RR: + case SCHED_POLICY_NONE: + case SCHED_POLICY_SINGLE: + case SCHED_POLICY_DEV: + case SCHED_POLICY_LOOP: + case SCHED_POLICY_INSTR: + /* Round-robin: atomic increment and module */ + selected_idx = __atomic_fetch_add(&cache->rr_ptr, 1, __ATOMIC_RELAXED) % + cache->valid_count; + break; + case SCHED_POLICY_HUNGRY: + /* Update min load periodically */ + op_count = __atomic_fetch_add(&cache->op_counter, 1, __ATOMIC_RELAXED); + if (op_count % cache->update_interval == 0) + wd_sched_update_min_load(cache, ctxs); + + /* Load balancing: use cached min load index */ + selected_idx = __atomic_load_n(&cache->min_load_idx, __ATOMIC_RELAXED); + break; + default: + WD_ERR("invalid: unknown scheduling policy %d!\n", cache->policy); + selected_idx = INVALID_POS; + break; + } + + /* Ensure index is within valid range */ + if (selected_idx >= cache->valid_count) return INVALID_POS; - return sched_get_next_pos_rr(region, NULL); + *ctx_idx = selected_idx; + return cache->idx_list[selected_idx]; } -static handle_t session_sched_init(handle_t h_sched_ctx, void *sched_param) +/** + * wd_sched_skey_update_load - Update load for a specific ctx + * @cache: Pointer to cache structure + * @ctx_idx: Context index in list + * @delta: Load delta (positive for send, negative for receive) + * + * Atomically update load counter for the specified ctx. + * Returns 0 on success, negative error code if ctx not found. + */ +static int wd_sched_skey_update_load(struct wd_sched_domain_idx_cache *cache, + __u32 ctx_idx, int delta) { - struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - struct sched_params *param = (struct sched_params *)sched_param; - struct sched_key *skey; + /* Atomic update without locking, ctx_idx's value is guaranteed by the caller. */ + if (delta > 0) + __atomic_fetch_add(&cache->load_values[ctx_idx], delta, __ATOMIC_RELAXED); + else { + __atomic_fetch_sub(&cache->load_values[ctx_idx], -delta, __ATOMIC_RELAXED); + if (unlikely(!cache->load_decreased)) + cache->load_decreased = 1; + } + return WD_SUCCESS; +} + +/* ============================================================================ + * Session Key Domain Initialization + * ============================================================================ + */ + +/** + * wd_sched_skey_domain_init - Initialize session domain with min-heap + * @key_domain: Target key domain + * @ctx_idx: context indices idx + * @policy: current session's policy + * + * Initializes dual-domain structure for session. + */ +static int wd_sched_skey_domain_init(struct wd_sched_key_domain *key_domain, + __u32 ctx_idx, __u8 policy) +{ + int ret; + + if (!key_domain) + return -WD_EINVAL; + + ret = wd_sched_skey_cache_init(&key_domain->idx_cache, policy); + if (ret) + return ret; + + ret = wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx); + if (ret) + goto init_err; + + ret = pthread_mutex_init(&key_domain->lock, NULL); + if (ret) + goto add_ctx_err; + + key_domain->expanded_count = 0; + + return WD_SUCCESS; + +add_ctx_err: + wd_sched_skey_remove_ctx(&key_domain->idx_cache, ctx_idx); +init_err: + wd_sched_skey_cache_uninit(&key_domain->idx_cache); + return ret; +} + +/** + * wd_sched_skey_domain_destroy - Release session domain resources + */ +static void wd_sched_skey_domain_destroy(struct wd_sched_key_domain *key_domain) +{ + if (!key_domain) + return; + + pthread_mutex_destroy(&key_domain->lock); + wd_sched_skey_cache_uninit(&key_domain->idx_cache); +} + + +/** + * wd_sched_poll_skey - Poll contexts for scheduler session + * @sched_ctx: Scheduler context + * @skey: Session key + * @expect: Expected number of responses + * @count: Actual response count (output) + * + * Polls all contexts in session domains and updates load values. + */ +static int wd_sched_poll_skey(struct wd_sched_ctx *sched_ctx, struct wd_sched_key *skey, + __u32 expect, __u32 *count) +{ + struct wd_sched_domain_idx_cache *cache; + __u32 ctx_list_num = 0; + __u32 sum_poll_num = 0; + bool hungry_policy; + __u32 poll_num; + __u32 idx, i; + int ret = 0; + + /* Get cache pointer and check if HUNGRY policy */ + cache = &skey->async_domain.idx_cache; + ctx_list_num = cache->valid_count; + hungry_policy = (cache->policy == SCHED_POLICY_HUNGRY); + + /* Poll async domain contexts */ + for (i = 0; i < ctx_list_num; i++) { + idx = cache->idx_list[i]; + if (idx == INVALID_POS) + continue; + poll_num = 0; + ret = sched_ctx->poll_func(idx, expect, &poll_num); + if (poll_num > 0) + sum_poll_num += poll_num; + + /* Update load value for this context */ + if (hungry_policy && poll_num > 0) { + if (skey->ctxs) + __atomic_fetch_sub(&skey->ctxs[idx].hw_load, + poll_num, __ATOMIC_RELAXED); + wd_sched_skey_update_load(cache, i, -poll_num); + } + + if (ret < 0 && ret != -WD_EAGAIN) + break; + } + *count = sum_poll_num; + + return ret; +} + +/* ============================================================================ + * Utility Functions + * ============================================================================ + */ +static inline bool sched_skey_get_ref(struct wd_sched_key *skey) +{ + if (!skey) + return false; + return __atomic_fetch_add(&skey->refcount, 1, __ATOMIC_ACQUIRE) > 0; +} + +static inline bool sched_skey_put_ref(struct wd_sched_key *skey) +{ + return __atomic_fetch_sub(&skey->refcount, 1, __ATOMIC_RELEASE) == 1; +} + +static int sched_skey_param_init(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey) +{ + __u32 i; + + pthread_mutex_lock(&sched_ctx->skey_lock); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + if (!sched_ctx->skey[i]) { + sched_ctx->skey[i] = skey; + if (sched_ctx->skey_num < SKEY_MAX_THREAD_NUM) + sched_ctx->skey_num++; + pthread_mutex_unlock(&sched_ctx->skey_lock); + return 0; + } + } + pthread_mutex_unlock(&sched_ctx->skey_lock); + WD_ERR("invalid: skey node number exceeds SKEY_MAX_THREAD_NUM(%d)!\n", + SKEY_MAX_THREAD_NUM); + return -WD_ENOMEM; +} + +static void sched_skey_param_uninit(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey) +{ + __u32 i; + + if (!sched_ctx || !skey) + return; + + pthread_mutex_lock(&sched_ctx->skey_lock); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + if (sched_ctx->skey[i] == skey) { + sched_ctx->skey[i] = NULL; + pthread_mutex_unlock(&sched_ctx->skey_lock); + return; + } + } + pthread_mutex_unlock(&sched_ctx->skey_lock); + WD_ERR("warning: skey %p not found in sched_ctx array\n", skey); +} + +static handle_t sched_session_common_init(struct wd_sched_ctx *sched_ctx, + struct sched_params *param) +{ + struct wd_sched_key *skey; unsigned int node; if (getcpu(NULL, &node)) { WD_ERR("failed to get node, errno %d!\n", errno); return (handle_t)(-errno); } - if (node == (unsigned int)NUMA_NO_NODE) { - WD_ERR("invalid: failed to get numa node!\n"); - return (handle_t)(-WD_EINVAL); - } if (!sched_ctx) { WD_ERR("invalid: sched ctx is NULL!\n"); return (handle_t)(-WD_EINVAL); } - skey = malloc(sizeof(struct sched_key)); + skey = malloc(sizeof(struct wd_sched_key)); if (!skey) { WD_ERR("failed to alloc memory for session sched key!\n"); return (handle_t)(-WD_ENOMEM); } + memset(skey, 0, sizeof(struct wd_sched_key)); if (!param) { - memset(skey, 0, sizeof(struct sched_key)); - skey->numa_id = sched_ctx->numa_map[node]; + skey->region_id = node; if (wd_need_debug()) WD_DEBUG("session don't set scheduler parameters!\n"); - } else if (param->numa_id < 0) { - skey->type = param->type; - skey->numa_id = sched_ctx->numa_map[node]; } else { + if (sched_ctx->policy == SCHED_POLICY_DEV) + skey->region_id = param->dev_id; + else if (param->numa_id >= 0) + skey->region_id = param->numa_id; + else + skey->region_id = node; skey->type = param->type; - skey->numa_id = param->numa_id; + skey->ctx_prop = param->ctx_prop; } + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + __atomic_store_n(&skey->refcount, 1, __ATOMIC_RELAXED); - if (skey->numa_id < 0) { - WD_ERR("failed to get valid sched numa region!\n"); - goto out; - } + return (handle_t)skey; +} - skey->sync_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); - skey->async_ctxid = session_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); - if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { - WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); - goto out; +static __u16 sched_get_poll_skey_idx(struct wd_sched_ctx *sched_ctx) +{ + /* Thread-local sequence number with atomic global counter */ + static __thread __u32 thread_seq = UINT32_MAX; + static __u32 global_seq_counter; + __u32 skey_num = sched_ctx->skey_num; + __u16 start_pos; + + if (unlikely(!sched_ctx || !skey_num)) + return skey_num; + + /* Assign unique sequence number on first call */ + if (unlikely(thread_seq == UINT32_MAX)) { + thread_seq = __atomic_fetch_add(&global_seq_counter, 1, __ATOMIC_RELAXED); + + /* Basic overflow protection */ + if (thread_seq >= UINT32_MAX - 1) + WD_DEBUG("Thread sequence counter approaching limit: %u\n", thread_seq); } - return (handle_t)skey; + /* Calculate start_pos based on thread_seq */ + start_pos = thread_seq % skey_num; -out: - free(skey); - return (handle_t)(-WD_EINVAL); + return start_pos; } -/* - * session_pick_next_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/** + * session_sched_init_ctx - Pre-fetch single context from domain for session + * @sched_ctx: Scheduler context + * @skey: session scheduler param + * @sched_mode: Mode (SYNC/ASYNC) * - * The user must init the schedule info through session_sched_init + * Returns: Context index from domain (compatible with alg_name if skey provided) */ -static __u32 session_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, - const int sched_mode) +static __u32 session_sched_init_ctx(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, int sched_mode) { - struct sched_key *key = (struct sched_key *)sched_key; + struct wd_sched_ctx_domain *domain = NULL; + int region_id = skey->region_id; + __u32 op_type = skey->type; + __u8 prop = skey->ctx_prop; + __u32 ctx_idx; + __u16 r; + + if (sched_mode >= SCHED_MODE_BUTT || + op_type >= sched_ctx->type_num || prop >= UADK_ALG_TYPE_MAX) { + WD_ERR("invalid: region: %d, mode: %d, type: %u!, prop: %u\n", + region_id, sched_mode, op_type, prop); + return INVALID_POS; + } - if (unlikely(!h_sched_ctx || !key)) { - WD_ERR("invalid: sched ctx or key is NULL!\n"); + if (!sched_ctx->domain_hash_table) + return INVALID_POS; + + /* Try current region first */ + if (region_id >= 0) { + if (sched_ctx->policy == SCHED_POLICY_DEV || + region_id < sched_ctx->region_num) { + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + region_id, sched_mode, op_type, prop); + if (domain && domain->valid) + return wd_sched_domain_get_next_rr(domain); + } + } + + /* DEV policy must not cross region */ + if (sched_ctx->policy == SCHED_POLICY_DEV) return INVALID_POS; + + /* Cross-region fallback for other policies */ + for (r = 0; r < sched_ctx->region_num; r++) { + if ((int)r == region_id) + continue; + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + r, sched_mode, op_type, prop); + if (domain && domain->valid) { + ctx_idx = wd_sched_domain_get_next_rr(domain); + if (ctx_idx != INVALID_POS) + return ctx_idx; + } } - /* return in do task */ - if (sched_mode == CTX_MODE_SYNC) - return key->sync_ctxid; - return key->async_ctxid; + return INVALID_POS; } -static int session_poll_region(struct wd_sched_ctx *sched_ctx, __u32 begin, - __u32 end, __u32 expect, __u32 *count) +/** + * session_sched_domain_destroy - Destroy session domains + * @skey: Session key to destroy domains for + * + * Releases all resources associated with session domains. + */ +static void session_sched_domain_destroy(struct wd_sched_key *skey) { - __u32 poll_num = 0; - __u32 i; - int ret; + if (!skey) + return; - /* i is the pos of sched_ctxs, the max is end */ - for (i = begin; i <= end; i++) { - /* - * RR schedule, one time poll one package, - * poll_num is always not more than one here. - */ - ret = sched_ctx->poll_func(i, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) - return ret; - else if (ret == -EAGAIN) + /* Destroy both sync and async domains */ + wd_sched_skey_domain_destroy(&skey->sync_domain); + wd_sched_skey_domain_destroy(&skey->async_domain); +} + +static inline void sched_skey_poll_release(struct wd_sched_key *skey) +{ + __u32 prev; + + if (!skey) + return; + + prev = __atomic_fetch_sub(&skey->refcount, 1, __ATOMIC_ACQ_REL); + if (prev == 1) { + session_sched_domain_destroy(skey); + free(skey); + } +} + +static void sched_session_uninit(handle_t h_sched_ctx, handle_t h_sched_key) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_key *skey = (struct wd_sched_key *)h_sched_key; + + if (!skey) + return; + + /* Remove from skey array first to prevent new poll discovery. + * Poll threads already holding a pointer are protected by refcount. + */ + if (sched_ctx) + sched_skey_param_uninit(sched_ctx, skey); + + /* Drop creator reference. If no poll holds a ref, free now. + * Otherwise the last poll thread will cleanup via poll_release. + */ + if (sched_skey_put_ref(skey)) { + session_sched_domain_destroy(skey); + free(skey); + } +} + +/** + * session_sched_init_ctx_with_fallback - Pre-fetch a ctx for one sched_mode, + * trying user_prop first then falling + * back through other props in enum order. + * @sched_ctx: Scheduler context + * @skey: Session key (skey->ctx_prop is the user-specified prop; restored on return) + * @sched_mode: SCHED_MODE_SYNC or SCHED_MODE_ASYNC + * + * Returns: ctx index, or INVALID_POS if no prop has a usable ctx. + * + * Falls back only when the user-specified prop's domain is unavailable + * (e.g. HW driver unloaded). Stops at the first prop that yields a ctx; + * does NOT accumulate ctxs across props. + */ +static __u32 session_sched_init_ctx_with_fallback(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + int sched_mode) +{ + __u8 user_prop = skey->ctx_prop; + __u32 ctx_idx; + __u8 p; + + /* Try user-specified prop first */ + ctx_idx = session_sched_init_ctx(sched_ctx, skey, sched_mode); + if (ctx_idx != INVALID_POS) + return ctx_idx; + + /* Fallback: target prop unavailable, try remaining props in enum order */ + for (p = 0; p < UADK_ALG_TYPE_MAX; p++) { + if (p == user_prop) continue; - *count += poll_num; - if (*count == expect) + skey->ctx_prop = p; + ctx_idx = session_sched_init_ctx(sched_ctx, skey, sched_mode); + if (ctx_idx != INVALID_POS) break; } - return 0; + /* Restore user_prop so skey state is not mutated */ + skey->ctx_prop = user_prop; + return ctx_idx; } -static int session_poll_policy_rr(struct wd_sched_ctx *sched_ctx, int numa_id, - __u32 expect, __u32 *count) +/** + * session_sched_domain_init - Initialize session domains with sync/async + * @sched_ctx: Scheduler context + * @skey: Session key to initialize + * @allow_fallback: If true, fall back to other props when skey->ctx_prop has + * no usable domain; if false, only try skey->ctx_prop. + * + * Pre-fetches sync and async contexts and initializes corresponding domains. + * Returns: 0 on success, negative error code on failure. + */ +static int session_sched_domain_init(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + bool allow_fallback) { - struct sched_ctx_region **region = sched_ctx->sched_info[numa_id].ctx_region; - __u32 begin, end; - __u32 i; - int ret; + __u32 sync_ctx, async_ctx; - for (i = 0; i < sched_ctx->type_num; i++) { - if (!region[SCHED_MODE_ASYNC][i].valid) - continue; + if (!sched_ctx || !skey) { + WD_ERR("invalid: sched_ctx or skey is NULL!\n"); + return -WD_EINVAL; + } + + if (allow_fallback) { + /* Pre-fetch with per-mode prop fallback */ + sync_ctx = session_sched_init_ctx_with_fallback(sched_ctx, skey, + SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx_with_fallback(sched_ctx, skey, + SCHED_MODE_ASYNC); + } else { + sync_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_ASYNC); + } + + if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) { + WD_ERR("invalid: no valid sync_ctx or async_ctx domain!\n"); + return -WD_EINVAL; + } + + /* Initialize sync domain if context is valid */ + if (sync_ctx != INVALID_POS) { + if (wd_sched_skey_domain_init(&skey->sync_domain, sync_ctx, sched_ctx->policy)) { + WD_ERR("failed to init sync domain!\n"); + return -WD_EINVAL; + } + } + + /* Initialize async domain if context is valid */ + if (async_ctx != INVALID_POS) { + if (wd_sched_skey_domain_init(&skey->async_domain, async_ctx, sched_ctx->policy)) { + WD_ERR("failed to init async domain!\n"); + /* Cleanup sync domain if async domain init failed */ + if (sync_ctx != INVALID_POS) + wd_sched_skey_domain_destroy(&skey->sync_domain); + return -WD_EINVAL; + } + } + + return WD_SUCCESS; +} + +/* ============================================================================ + * Scheduler Policy Functions + * ============================================================================ + */ +/** + * round_robin_sched_init - Initialize session with single sync and async ctx + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_param: Scheduling parameters (cannot modify per API contract) + * + * Allocates session key and pre-fetches one sync and one async context. + */ +static handle_t round_robin_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + struct wd_sched_key *skey; + handle_t hskey; + int ret = 0; + + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; + } + + skey = (struct wd_sched_key *)hskey; + /* RR: allow prop fallback so session creation survives target prop unload */ + ret = session_sched_domain_init(sched_ctx, skey, true); + if (ret) { + WD_ERR("failed to initialize session domains!\n"); + free(skey); + return (handle_t)(-WD_EINVAL); + } + + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); + return (handle_t)(-WD_ENOMEM); + } + + return hskey; +} + +/** + * round_robin_pick_next_ctx - Pick context with load-based selection + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) + * + * Returns: Context index with minimum load + * Time complexity: O(1) + */ +static __u32 round_robin_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_key_domain *domain; + __u32 min_ctx, ctx_idx; - begin = region[SCHED_MODE_ASYNC][i].begin; - end = region[SCHED_MODE_ASYNC][i].end; - ret = session_poll_region(sched_ctx, begin, end, expect, count); - if (unlikely(ret)) - return ret; + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; } - return 0; + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; + + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) + return INVALID_POS; + + return min_ctx; } -/* - * session_poll_policy - The polling policy matches the pick next ctx. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @cfg: The global resoure info. - * @expect: User expect poll msg num. - * @count: The actually poll num. +/** + * round_robin_poll_policy - Poll policy for session scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) * - * The user must init the schedule info through wd_sched_rr_instance, the - * func interval will not check the valid, becouse it will affect performance. + * Returns: Status code */ -static int session_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +static int round_robin_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; - struct wd_sched_info *sched_info; - __u32 loop_time = 0; - __u32 last_count = 0; - __u16 i, region_mum; - int ret; + struct wd_sched_key *skey; + __u16 i, tpos, start_pos; + __u32 poll_num, skey_num; + int ret = -WD_EAGAIN; + __u32 sum_count = 0; if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); return -WD_EINVAL; } - if (unlikely(sched_ctx->numa_num > NUMA_NUM_NODES)) { - WD_ERR("invalid: ctx's numa number is %u!\n", sched_ctx->numa_num); - return -WD_EINVAL; + /* Randomize the initial query position. */ + skey_num = sched_ctx->skey_num; + start_pos = sched_get_poll_skey_idx(sched_ctx); + if (!skey_num || start_pos >= sched_ctx->skey_num) { + *count = 0; + return -WD_EAGAIN; } - sched_info = sched_ctx->sched_info; - if (sched_ctx->policy == SCHED_POLICY_DEV) - region_mum = sched_ctx->dev_num; - else - region_mum = sched_ctx->numa_num; - - /* - * Try different region's ctx if we can't receive any - * package last time, it is more efficient. In most - * bad situation, poll ends after MAX_POLL_TIMES loop. - */ - while (++loop_time < MAX_POLL_TIMES) { - for (i = 0; i < region_mum;) { - /* If current numa is not valid, find next. */ - if (!sched_info[i].valid) { - i++; - continue; - } + /* Query the queues on each skey separately. */ + for (i = 0; i < skey_num; i++) { + tpos = (start_pos + i) % skey_num; + skey = sched_ctx->skey[tpos]; - last_count = *count; - ret = session_poll_policy_rr(sched_ctx, i, expect, count); - if (unlikely(ret)) - return ret; + if (unlikely(!skey)) + continue; - if (expect == *count) - return 0; + if (!sched_skey_get_ref(skey)) + continue; - /* - * If no package is received, find next numa, - * otherwise, keep receiving packets at this node. - */ - if (last_count == *count) - i++; + if (__atomic_test_and_set(&skey->poll_lock, __ATOMIC_ACQUIRE)) { + sched_skey_poll_release(skey); + continue; } + + ret = wd_sched_poll_skey(sched_ctx, skey, expect, &poll_num); + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + sched_skey_poll_release(skey); + + sum_count += poll_num; + if (unlikely(ret && ret != -WD_EAGAIN)) + goto poll_err; + + if (sum_count >= expect) + break; } - return 0; +poll_err: + *count = sum_count; + if (ret == -WD_EAGAIN) + return 0; + return ret; } static handle_t sched_none_init(handle_t h_sched_ctx, void *sched_param) @@ -396,12 +1464,11 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, } while (loop_times > 0) { - /* Default use ctx 0 */ loop_times--; ret = sched_ctx->poll_func(0, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) + if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; - else if (ret == -EAGAIN) + else if (ret == -WD_EAGAIN) continue; *count += poll_num; @@ -409,7 +1476,7 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, break; } - return 0; + return WD_SUCCESS; } static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) @@ -420,13 +1487,10 @@ static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) static __u32 sched_single_pick_next_ctx(handle_t sched_ctx, void *sched_key, const int sched_mode) { -#define CTX_ASYNC 1 -#define CTX_SYNC 0 - if (sched_mode) - return CTX_ASYNC; + return 1; else - return CTX_SYNC; + return 0; } static int sched_single_poll_policy(handle_t h_sched_ctx, @@ -443,12 +1507,11 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, } while (loop_times > 0) { - /* Default async mode use ctx 1 */ loop_times--; ret = sched_ctx->poll_func(1, 1, &poll_num); - if ((ret < 0) && (ret != -EAGAIN)) + if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; - else if (ret == -EAGAIN) + else if (ret == -WD_EAGAIN) continue; *count += poll_num; @@ -456,260 +1519,483 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, break; } - return 0; + return WD_SUCCESS; } -static bool sched_dev_key_valid(struct wd_sched_ctx *sched_ctx, const struct sched_key *key) +/** + * sched_skey_domain_fill - Initialize or append ctx to session domain + * @key_domain: Target domain (sync or async) + * @ctx_idx: Context index to add + * @policy: Scheduler policy + * @inited: Pointer to boolean tracking whether domain has been initialized + * + * First call: full domain init via wd_sched_skey_domain_init(). + * Subsequent calls: append via wd_sched_skey_add_ctx(). + */ +static void sched_skey_domain_fill(struct wd_sched_key_domain *key_domain, + __u32 ctx_idx, __u8 policy, bool *inited) { - bool found = false; - int i; + int ret; - if (key->mode >= SCHED_MODE_BUTT || key->type >= sched_ctx->type_num) { - WD_ERR("invalid: sched key's device id: %u, mode: %u, type: %u!\n", - key->dev_id, key->mode, key->type); - return false; + if (ctx_idx == INVALID_POS) + return; + + if (!*inited) { + ret = wd_sched_skey_domain_init(key_domain, ctx_idx, policy); + if (!ret) + *inited = true; + return; } - for (i = 0; i < sched_ctx->dev_num; i++) { - if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { - found = true; - break; + (void)wd_sched_skey_add_ctx(&key_domain->idx_cache, ctx_idx); +} + +/** + * sched_skey_common_init - Common scheduler init with init-once + add-rest pattern + * @h_sched_ctx: Scheduler handle + * @sched_param: Scheduling parameters + * @prop_begin: First prop type to iterate (inclusive) + * @prop_end: Last prop type to iterate (inclusive) + * + * Shared by Loop, Hungry, and Instr scheduler init functions. + * Iterates [prop_begin, prop_end], fetching sync/async ctxs per prop type. + * First valid ctx initializes the domain, subsequent ctxs are appended. + */ +static handle_t sched_skey_common_init(handle_t h_sched_ctx, void *sched_param, + __u8 prop_begin, __u8 prop_end) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct sched_params *param = (struct sched_params *)sched_param; + struct wd_sched_key *skey; + bool sync_inited = false; + bool async_inited = false; + __u32 sync_ctx, async_ctx; + __u32 req_ctx_num = 0; + handle_t hskey; + __u8 def_prop; + int ret, i; + + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; + } + + skey = (struct wd_sched_key *)hskey; + def_prop = skey->ctx_prop; + for (i = prop_begin; i <= prop_end; i++) { + skey->ctx_prop = i; + sync_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_SYNC); + async_ctx = session_sched_init_ctx(sched_ctx, skey, SCHED_MODE_ASYNC); + if (sync_ctx == INVALID_POS && async_ctx == INVALID_POS) + continue; + + if (sync_ctx != INVALID_POS) { + sched_skey_domain_fill(&skey->sync_domain, sync_ctx, + sched_ctx->policy, &sync_inited); } + if (async_ctx != INVALID_POS) + sched_skey_domain_fill(&skey->async_domain, async_ctx, + sched_ctx->policy, &async_inited); + + req_ctx_num += 2; + } + if (!req_ctx_num) { + free(skey); + return (handle_t)(-WD_EINVAL); } - if (!found) { - WD_ERR("invalid: dev_id %u is not registered!\n", key->dev_id); - return false; + skey->ctx_prop = def_prop; + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); + return (handle_t)(-WD_ENOMEM); } - return true; + return hskey; } -/* - * sched_dev_get_region - Get ctx region from ctx_map by the wd comp arg +/** + * skey_sched_init - Initialize Hungry scheduler session + */ +static handle_t skey_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + 0, UADK_ALG_TYPE_MAX - 1); + return hskey; +} + +/** + * skey_sched_pick_next_ctx - Pick context from hungry scheduler with load awareness + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) + * + * Returns: Context with minimum load, or expands if threshold exceeded + * Time complexity: O(1) for selection, O(n) if expansion needed */ -static struct sched_ctx_region *sched_dev_get_region(struct wd_sched_ctx *sched_ctx, - const struct sched_key *key) +static __u32 skey_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) { - struct wd_sched_info *sched_info; - int i, region_id; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_domain_idx_cache *idx_cache; + struct wd_sched_key_domain *domain; + __u32 min_ctx, min_load, ctx_idx; - for (i = 0; i < sched_ctx->dev_num; i++) { - if (key->dev_id == sched_ctx->dev_id_map[i].dev_id) { - region_id = sched_ctx->dev_id_map[i].region_id; - sched_info = &sched_ctx->sched_info[region_id]; - if (sched_info->ctx_region[key->mode][key->type].valid) - return &sched_info->ctx_region[key->mode][key->type]; + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); + return INVALID_POS; + } + + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; + + idx_cache = &domain->idx_cache; + if (sched_mode == SCHED_MODE_SYNC) { + /* Sync: send+recv atomic, spinlock serialized, use RR */ + if (!idx_cache->valid_count) + return INVALID_POS; + + ctx_idx = __atomic_fetch_add(&idx_cache->rr_ptr, 1, __ATOMIC_RELAXED) % + idx_cache->valid_count; + return idx_cache->idx_list[ctx_idx]; + } + + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) + return INVALID_POS; + + if (skey->ctxs) { + min_load = __atomic_load_n(&skey->ctxs[min_ctx].hw_load, __ATOMIC_RELAXED); + if (min_load >= HW_QUEUE_FULL_DEPTH) + return QUEUE_FULL_POS; + __atomic_fetch_add(&skey->ctxs[min_ctx].hw_load, 1, __ATOMIC_RELAXED); + wd_sched_skey_update_load(idx_cache, ctx_idx, 1); + } else { + min_load = __atomic_load_n(&idx_cache->load_values[ctx_idx], __ATOMIC_RELAXED); + if (min_load >= HW_QUEUE_FULL_DEPTH && idx_cache->load_decreased > 0) + return QUEUE_FULL_POS; + wd_sched_skey_update_load(idx_cache, ctx_idx, 1); + } + /* Check if we need to expand context pool */ + + return min_ctx; +} + +/** + * skey_sched_poll_policy - Poll policy for hungry scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) + * + * Returns: Status code + */ +static int skey_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + struct wd_sched_key *skey; + __u16 i, tpos, start_pos; + __u32 poll_num, skey_num; + int ret = -WD_EAGAIN; + __u32 sum_count = 0; + + if (unlikely(!count || !sched_ctx || !sched_ctx->poll_func)) { + WD_ERR("invalid: sched ctx or poll_func is NULL or count is zero!\n"); + return -WD_EINVAL; + } + + /* Randomize the initial query position. */ + skey_num = sched_ctx->skey_num; + start_pos = sched_get_poll_skey_idx(sched_ctx); + if (!skey_num || start_pos >= sched_ctx->skey_num) { + *count = 0; + return -WD_EAGAIN; + } + + /* Query the queues on each skey separately. */ + for (i = 0; i < skey_num; i++) { + tpos = (start_pos + i) % skey_num; + skey = sched_ctx->skey[tpos]; + + if (unlikely(!skey)) + continue; + + if (!sched_skey_get_ref(skey)) + continue; + + if (__atomic_test_and_set(&skey->poll_lock, __ATOMIC_ACQUIRE)) { + sched_skey_poll_release(skey); + continue; } + + ret = wd_sched_poll_skey(sched_ctx, skey, expect, &poll_num); + __atomic_clear(&skey->poll_lock, __ATOMIC_RELEASE); + sched_skey_poll_release(skey); + /* + * This query returned 0, indicating the hardware + * likely hasn't finished processing yet. + * Implementing a delay and releasing the CPU is a predictive optimization + */ + if (!poll_num) + usleep(1); + + sum_count += poll_num; + if (ret == -WD_EAGAIN) + continue; + if (unlikely(ret) || sum_count >= expect) + break; } + *count = sum_count; - /* - * If the scheduling domain of dev_id does not exist, - * taskes operations cannot be executed using queues from other devices; - * otherwise, an SMMU error will occur. - */ - return NULL; + return ret; } -/* - * session_dev_sched_init_ctx - Get one ctx from ctxs by the sched_ctx and arg. - * @sched_ctx: Schedule ctx, reference the struct sample_sched_ctx. - * @sched_key: The key of schedule region. - * @sched_mode: The sched async/sync mode. +/** + * loop_sched_init - Initialize Loop scheduler session + */ +static handle_t loop_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + 0, UADK_ALG_TYPE_MAX - 1); + return hskey; +} + +/** + * loop_sched_pick_next_ctx - Pick context for loop scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_mode: Mode (cannot modify per API contract) * - * The user must init the schedule info through wd_sched_rr_instance + * Returns: Context index with minimum load + * Time complexity: O(1) + */ +static __u32 loop_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) +{ + return round_robin_pick_next_ctx(h_sched_ctx, sched_key, sched_mode); +} + +static int loop_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + return round_robin_poll_policy(h_sched_ctx, expect, count); +} + +/** + * instr_sched_init - Initialize Instr scheduler session */ -static __u32 session_dev_sched_init_ctx(struct wd_sched_ctx *sched_ctx, struct sched_key *key, - const int sched_mode) +static handle_t instr_sched_init(handle_t h_sched_ctx, void *sched_param) +{ + handle_t hskey = sched_skey_common_init(h_sched_ctx, sched_param, + UADK_ALG_CE_INSTR, + UADK_ALG_SVE_INSTR); + return hskey; +} + +static __u32 instr_sched_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, + const int sched_mode) { - struct sched_ctx_region *region = NULL; - bool ret; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_key_domain *domain; + __u32 min_ctx, ctx_idx; - key->mode = sched_mode; - ret = sched_dev_key_valid(sched_ctx, key); - if (!ret) + if (unlikely(!h_sched_ctx || !skey)) { + WD_ERR("invalid: sched ctx or key is NULL!\n"); return INVALID_POS; + } + + if (sched_mode == SCHED_MODE_SYNC) + domain = &skey->sync_domain; + else + domain = &skey->async_domain; - region = sched_dev_get_region(sched_ctx, key); - if (!region) + /* Get current minimum load context */ + min_ctx = wd_sched_skey_pick_next(&domain->idx_cache, &ctx_idx, skey->ctxs); + if (min_ctx == INVALID_POS) return INVALID_POS; - return sched_get_next_pos_rr(region, NULL); + return min_ctx; +} + +/** + * instr_sched_poll_policy - Poll policy for instruction scheduler + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @expect: Expected number of responses (cannot modify per API contract) + * @count: Actual response count (cannot modify per API contract) + * + * Returns: Status code + */ +static int instr_sched_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) +{ + return round_robin_poll_policy(h_sched_ctx, expect, count); } static handle_t session_dev_sched_init(handle_t h_sched_ctx, void *sched_param) { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; struct sched_params *param = (struct sched_params *)sched_param; - struct sched_key *skey; - unsigned int node; + struct wd_sched_key *skey; + handle_t hskey; + int ret = 0; - if (getcpu(NULL, &node)) { - WD_ERR("failed to get numa node, errno %d!\n", errno); - return (handle_t)(-errno); - } - if (node == (unsigned int)NUMA_NO_NODE) { - WD_ERR("invalid: failed to get numa node for dev sched init!\n"); + if (!param) { + WD_ERR("invalid: dev sched param is NULL!\n"); return (handle_t)(-WD_EINVAL); } - if (!sched_ctx) { - WD_ERR("invalid: sched ctx is NULL!\n"); - return (handle_t)(-WD_EINVAL); + hskey = sched_session_common_init(sched_ctx, param); + if (WD_IS_ERR(hskey)) { + WD_ERR("failed to init session schedule key!\n"); + return hskey; } - if (!param) { - WD_DEBUG("no-sva session don't set scheduler parameters!\n"); + skey = (struct wd_sched_key *)hskey; + skey->type = param->type; + skey->dev_id = param->dev_id; + + /* DEV: pinned to a specific device, do not cross prop families */ + ret = session_sched_domain_init(sched_ctx, skey, false); + if (ret) { + WD_ERR("failed to initialize session domains!\n"); + free(skey); return (handle_t)(-WD_EINVAL); } - skey = malloc(sizeof(struct sched_key)); - if (!skey) { - WD_ERR("failed to alloc memory for session sched key!\n"); + ret = sched_skey_param_init(sched_ctx, skey); + if (ret) { + WD_ERR("failed to register skey in sched_ctx array!\n"); + session_sched_domain_destroy(skey); + free(skey); return (handle_t)(-WD_ENOMEM); } - skey->type = param->type; - skey->dev_id = param->dev_id; + return (handle_t)skey; +} + +/** + * wd_sched_set_param - Set scheduler parameters + * @h_sched_ctx: Scheduler handle (cannot modify per API contract) + * @sched_key: Session key (cannot modify per API contract) + * @sched_param: Scheduling parameters (cannot modify per API contract) + */ + +static void wd_sched_set_param(handle_t h_sched_ctx, + void *sched_key, void *sched_param) +{ + struct wd_sched_params *params = (struct wd_sched_params *)sched_param; + struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; - skey->sync_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_SYNC); - skey->async_ctxid = session_dev_sched_init_ctx(sched_ctx, skey, CTX_MODE_ASYNC); - if (skey->sync_ctxid == INVALID_POS && skey->async_ctxid == INVALID_POS) { - WD_ERR("failed to get valid sync_ctxid or async_ctxid!\n"); - goto out; + if (unlikely(!params || !skey)) { + WD_ERR("invalid: sched parmas or skey is NULL!\n"); + return; } - return (handle_t)skey; + skey->pkt_size = params->pkt_size; + skey->is_stream = params->data_mode; + skey->prio_mode = params->prio_mode; + + /* Store compat filtering parameters */ + skey->alg_name = params->alg_name; + skey->ctxs = params->ctxs; -out: - free(skey); - return (handle_t)(-WD_EINVAL); } static struct wd_sched sched_table[SCHED_POLICY_BUTT] = { { .name = "RR scheduler", .sched_policy = SCHED_POLICY_RR, - .sched_init = session_sched_init, - .pick_next_ctx = session_sched_pick_next_ctx, - .poll_policy = session_sched_poll_policy, + .sched_init = round_robin_sched_init, + .pick_next_ctx = round_robin_pick_next_ctx, + .poll_policy = round_robin_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "None scheduler", .sched_policy = SCHED_POLICY_NONE, .sched_init = sched_none_init, .pick_next_ctx = sched_none_pick_next_ctx, .poll_policy = sched_none_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "Single scheduler", .sched_policy = SCHED_POLICY_SINGLE, .sched_init = sched_single_init, .pick_next_ctx = sched_single_pick_next_ctx, .poll_policy = sched_single_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, }, { .name = "Device RR scheduler", .sched_policy = SCHED_POLICY_DEV, .sched_init = session_dev_sched_init, - .pick_next_ctx = session_sched_pick_next_ctx, - .poll_policy = session_sched_poll_policy, - } + .pick_next_ctx = round_robin_pick_next_ctx, + .poll_policy = round_robin_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Loop scheduler", + .sched_policy = SCHED_POLICY_LOOP, + .sched_init = loop_sched_init, + .pick_next_ctx = loop_sched_pick_next_ctx, + .poll_policy = loop_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Hungry scheduler", + .sched_policy = SCHED_POLICY_HUNGRY, + .sched_init = skey_sched_init, + .pick_next_ctx = skey_sched_pick_next_ctx, + .poll_policy = skey_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, { + .name = "Instr scheduler", + .sched_policy = SCHED_POLICY_INSTR, + .sched_init = instr_sched_init, + .pick_next_ctx = instr_sched_pick_next_ctx, + .poll_policy = instr_sched_poll_policy, + .set_param = wd_sched_set_param, + .sched_uninit = sched_session_uninit, + }, }; -static int wd_sched_get_nearby_numa_id(struct wd_sched_info *sched_info, int node, int numa_num) -{ - int dis = INT32_MAX; - int valid_id = -1; - int i, tmp; - - for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) { - tmp = numa_distance(node, i); - if (dis > tmp) { - valid_id = i; - dis = tmp; - } - } - } - - return valid_id; -} - -static void wd_sched_map_cpus_to_dev(struct wd_sched_ctx *sched_ctx) -{ - struct wd_sched_info *sched_info = sched_ctx->sched_info; - int i, numa_num = sched_ctx->numa_num; - int *numa_map = sched_ctx->numa_map; - - for (i = 0; i < numa_num; i++) { - if (sched_info[i].valid) - numa_map[i] = i; - else - numa_map[i] = wd_sched_get_nearby_numa_id(sched_info, i, numa_num); - } -} - -static int wd_instance_dev_region(struct wd_sched_ctx *sched_ctx, - struct sched_params *param) +static int numa_num_check(__u16 region_num) { - struct wd_sched_info *sched_info; - __u32 region_idx = INVALID_POS; - __u8 type, mode; - __u32 dev_id; - int i; - - dev_id = param->dev_id; - type = param->type; - mode = param->mode; - - /* Check whether dev_id has already been registered. */ - for (i = 0; i < sched_ctx->dev_num; i++) { - if (sched_ctx->dev_id_map[i].dev_id == dev_id) { - region_idx = sched_ctx->dev_id_map[i].region_id; - break; - } - } - - /* If not registered, allocate a new region. */ - if (region_idx == INVALID_POS) { - if (sched_ctx->dev_num >= DEVICE_REGION_MAX) { - WD_ERR("too many devices registered!\n"); - return -WD_EINVAL; - } - - region_idx = sched_ctx->dev_num; - sched_ctx->dev_id_map[region_idx].dev_id = dev_id; - sched_ctx->dev_id_map[region_idx].region_id = region_idx; - sched_ctx->dev_num++; + int max_node; - sched_info = &sched_ctx->sched_info[region_idx]; - } else { - sched_info = &sched_ctx->sched_info[region_idx]; + max_node = numa_max_node() + 1; + if (max_node <= 0) { + WD_ERR("invalid: numa max node is %d!\n", max_node); + return -WD_EINVAL; } - /* Check whether the mode and type have already been registered. */ - if (sched_info->ctx_region[mode][type].valid) { - WD_INFO("device %u mode %u type %u already registered\n", - dev_id, mode, type); - return WD_SUCCESS; + if (!region_num || region_num > max_node) { + WD_ERR("invalid: region number is %u!\n", region_num); + return -WD_EINVAL; } - /* Initialize the scheduling region for this mode and type */ - sched_info->ctx_region[mode][type].begin = param->begin; - sched_info->ctx_region[mode][type].end = param->end; - sched_info->ctx_region[mode][type].last = param->begin; - sched_info->ctx_region[mode][type].valid = true; - sched_info->valid = true; - - pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); - - return WD_SUCCESS; + return 0; } +/** + * wd_sched_rr_instance - External API for scheduling region instance + * @sched: Scheduler (cannot modify per API contract) + * @param: Scheduling parameters (cannot modify per API contract) + * + * Creates scheduling region for given parameters. + */ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *param) { - struct wd_sched_info *sched_info = NULL; struct wd_sched_ctx *sched_ctx = NULL; - __u8 type, mode; - int numa_id; + struct wd_sched_ctx_domain *domain; + int region_key; + __u8 mode; + int ret; if (!sched || !sched->h_sched_ctx || !param) { WD_ERR("invalid: sched or sched_params is NULL!\n"); @@ -721,57 +2007,68 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para return -WD_EINVAL; } - numa_id = param->numa_id; - type = param->type; mode = param->mode; sched_ctx = (struct wd_sched_ctx *)sched->h_sched_ctx; - if (sched_ctx->numa_num > 0 && (numa_id >= sched_ctx->numa_num || - numa_id < 0)) { - WD_ERR("invalid: sched_ctx's numa_id is %d, numa_num is %u!\n", - numa_id, sched_ctx->numa_num); - return -WD_EINVAL; + if (sched_ctx->policy == SCHED_POLICY_DEV) { + region_key = param->dev_id; + if (region_key < 0) { + WD_ERR("invalid: dev_id is %d!\n", region_key); + return -WD_EINVAL; + } + } else { + region_key = param->numa_id; + if (region_key >= sched_ctx->region_num || region_key < 0) { + WD_ERR("invalid: region_key is %d, region_num is %u!\n", + region_key, sched_ctx->region_num); + return -WD_EINVAL; + } } - if (type >= sched_ctx->type_num) { - WD_ERR("invalid: sched_ctx's type is %u, type_num is %u!\n", - type, sched_ctx->type_num); + if (param->type >= sched_ctx->type_num) { + WD_ERR("invalid: type is %u, type_num is %u!\n", + param->type, sched_ctx->type_num); return -WD_EINVAL; } if (mode >= SCHED_MODE_BUTT) { - WD_ERR("invalid: sched_ctx's mode is %u, mode_num is %d!\n", - mode, SCHED_MODE_BUTT); + WD_ERR("invalid: mode is %u, mode_num is %u!\n", + mode, sched_ctx->mode_num); return -WD_EINVAL; } - if (sched_ctx->policy == SCHED_POLICY_DEV) - return wd_instance_dev_region(sched_ctx, param); - - sched_info = &sched_ctx->sched_info[numa_id]; - if (!sched_info->ctx_region[mode]) { - WD_ERR("invalid: ctx_region is NULL, numa: %d, mode: %u!\n", - numa_id, mode); - return -WD_EINVAL; + if (param->ctx_prop < 0 || param->ctx_prop >= UADK_ALG_TYPE_MAX) { + WD_INFO("Info: ctx_prop %d exceeds max type!\n", param->ctx_prop); + param->ctx_prop = UADK_ALG_HW; } - sched_info->ctx_region[mode][type].begin = param->begin; - sched_info->ctx_region[mode][type].end = param->end; - sched_info->ctx_region[mode][type].last = param->begin; - sched_info->ctx_region[mode][type].valid = true; - sched_info->valid = true; + domain = wd_sched_hash_table_insert(sched_ctx->domain_hash_table, + region_key, mode, param->type, + param->ctx_prop); + if (!domain) + return -WD_ENOMEM; - wd_sched_map_cpus_to_dev(sched_ctx); - pthread_mutex_init(&sched_info->ctx_region[mode][type].lock, NULL); + ret = wd_sched_domain_add_segment(domain, param->begin, param->end); + if (ret) { + WD_ERR("failed to add segment to domain!\n"); + return ret; + } + domain->valid = true; return WD_SUCCESS; } +/** + * wd_sched_rr_release - External API for scheduler release + * @sched: Scheduler to release (cannot modify per API contract) + * + * Releases all scheduler resources. + */ void wd_sched_rr_release(struct wd_sched *sched) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; - int i, j, region_num; + struct wd_sched_key *skey; + __u32 i; if (!sched) return; @@ -780,59 +2077,49 @@ void wd_sched_rr_release(struct wd_sched *sched) if (!sched_ctx) goto ctx_out; - /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ - if (sched_ctx->policy == SCHED_POLICY_DEV) - region_num = DEVICE_REGION_MAX; - else - region_num = sched_ctx->numa_num; + /* Release all session keys - iterate full array to catch residual entries */ + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) { + skey = sched_ctx->skey[i]; + if (!skey) + continue; - sched_info = sched_ctx->sched_info; - if (!sched_info) - goto info_out; + sched_ctx->skey[i] = NULL; + __atomic_store_n(&skey->refcount, 0, __ATOMIC_RELAXED); + session_sched_domain_destroy(skey); + free(skey); + } + sched_ctx->skey_num = 0; - for (i = 0; i < region_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - if (sched_info[i].ctx_region[j]) { - free(sched_info[i].ctx_region[j]); - sched_info[i].ctx_region[j] = NULL; - } - } + /* Release hash table */ + if (sched_ctx->domain_hash_table) { + wd_sched_hash_table_destroy(sched_ctx->domain_hash_table); + sched_ctx->domain_hash_table = NULL; } -info_out: + pthread_mutex_destroy(&sched_ctx->skey_lock); free(sched_ctx); + ctx_out: free(sched); - return; } -static int numa_num_check(__u16 numa_num) -{ - int max_node; - - max_node = numa_max_node() + 1; - if (max_node <= 0) { - WD_ERR("invalid: numa max node is %d!\n", max_node); - return -WD_EINVAL; - } - - if (!numa_num || numa_num > max_node) { - WD_ERR("invalid: numa number is %u!\n", numa_num); - return -WD_EINVAL; - } - - return 0; -} - +/** + * wd_sched_rr_alloc - External API for scheduler allocation + * @sched_type: Scheduling policy type (cannot modify per API contract) + * @type_num: Number of operation types (cannot modify per API contract) + * @region_num: Number of regions (cannot modify per API contract) + * @func: Poll function (cannot modify per API contract) + * + * Allocates and initializes scheduler with single global hash table. + */ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, - __u16 numa_num, user_poll_func func) + __u16 region_num, user_poll_func func) { - struct wd_sched_info *sched_info; struct wd_sched_ctx *sched_ctx; struct wd_sched *sched; - int region_num; - int i, j; + __u32 estimated_entries; + __u32 i; if (sched_type >= SCHED_POLICY_BUTT || !type_num) { WD_ERR("invalid: sched_type is %u or type_num is %u!\n", @@ -846,63 +2133,62 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, return NULL; } - if (sched_type == SCHED_POLICY_DEV) - region_num = DEVICE_REGION_MAX; - else - region_num = numa_num; - - sched_ctx = calloc(1, sizeof(struct wd_sched_ctx) + - sizeof(struct wd_sched_info) * region_num); + sched_ctx = calloc(1, sizeof(struct wd_sched_ctx)); if (!sched_ctx) { WD_ERR("failed to alloc memory for sched_ctx!\n"); goto err_out; } - /* In SCHED_POLICY_DEV mode, numa_num mean device numbers */ + /* Cache dimension parameters */ + sched_ctx->type_num = type_num; + sched_ctx->mode_num = SCHED_MODE_BUTT; + sched_ctx->region_num = region_num; + sched_ctx->policy = sched_type; + if (sched_type == SCHED_POLICY_DEV) { - sched_ctx->numa_num = 0; - sched_ctx->dev_num = 0; - for (i = 0; i < DEVICE_REGION_MAX; i++) { - sched_ctx->dev_id_map[i].dev_id = INVALID_POS; - sched_ctx->dev_id_map[i].region_id = INVALID_POS; - } + /* Device mode: region_num is actually device count */ + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } else { - sched_ctx->numa_num = numa_num; - sched_ctx->dev_num = 0; - if (numa_num_check(sched_ctx->numa_num)) + /* NUMA mode: validate region_num */ + if (numa_num_check(region_num)) goto err_out; + estimated_entries = region_num * type_num * SCHED_MODE_BUTT * UADK_ALG_TYPE_MAX; } - sched->h_sched_ctx = (handle_t)sched_ctx; - if (sched_type == SCHED_POLICY_NONE || - sched_type == SCHED_POLICY_SINGLE) - goto simple_ok; - - sched_info = sched_ctx->sched_info; - for (i = 0; i < region_num; i++) { - for (j = 0; j < SCHED_MODE_BUTT; j++) { - sched_info[i].ctx_region[j] = - calloc(1, sizeof(struct sched_ctx_region) * type_num); - if (!sched_info[i].ctx_region[j]) - goto err_out; - } + /* Create single global hash table */ + sched_ctx->domain_hash_table = wd_sched_hash_table_create(estimated_entries); + if (!sched_ctx->domain_hash_table) { + WD_ERR("failed to create hash table!\n"); + goto ctx_out; } -simple_ok: sched_ctx->poll_func = func; - sched_ctx->policy = sched_type; - sched_ctx->type_num = type_num; - memset(sched_ctx->numa_map, -1, sizeof(int) * NUMA_NUM_NODES); + for (i = 0; i < SKEY_MAX_THREAD_NUM; i++) + sched_ctx->skey[i] = NULL; + + if (pthread_mutex_init(&sched_ctx->skey_lock, NULL)) { + WD_ERR("failed to init skey_lock!\n"); + goto err_destroy_hash; + } + sched_ctx->skey_num = 0; + + sched->h_sched_ctx = (handle_t)sched_ctx; sched->sched_init = sched_table[sched_type].sched_init; + sched->sched_uninit = sched_table[sched_type].sched_uninit; sched->pick_next_ctx = sched_table[sched_type].pick_next_ctx; sched->poll_policy = sched_table[sched_type].poll_policy; sched->sched_policy = sched_type; sched->name = sched_table[sched_type].name; + sched->set_param = sched_table[sched_type].set_param; return sched; +err_destroy_hash: + wd_sched_hash_table_destroy(sched_ctx->domain_hash_table); +ctx_out: + free(sched_ctx); err_out: - wd_sched_rr_release(sched); + free(sched); return NULL; } -- 2.43.0
From: Longfang Liu <liulongfang@huawei.com> In the new UADK framework, update the dynamic registration management for user-space driver algorithm-driver pairs. After decoupling the framework and driver layers, ensure service scheduling, resource initialization, and the integrity of packet send/receive functionality, while maintaining the normal operation of existing features. Signed-off-by: Longfang Liu <liulongfang@huawei.com> --- Makefile.am | 3 +- include/wd_alg.h | 148 ++++++--- include/wd_internal.h | 1 - libwd.map | 7 +- wd_alg.c | 742 +++++++++++++++++++++++++++++------------- 5 files changed, 622 insertions(+), 279 deletions(-) diff --git a/Makefile.am b/Makefile.am index df873ce..5cc8c39 100644 --- a/Makefile.am +++ b/Makefile.am @@ -89,8 +89,7 @@ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ wd_ecc.c wd_ecc.h wd_ecc_drv.h \ wd_digest.c wd_digest.h wd_digest_drv.h \ wd_util.c wd_util.h \ - wd_sched.c wd_sched.h \ - wd.c wd.h + wd_sched.c wd_sched.h libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ lib/crypto/aes.c lib/crypto/sm4.c lib/crypto/galois.c \ diff --git a/include/wd_alg.h b/include/wd_alg.h index 1ae1dae..9edcbf1 100644 --- a/include/wd_alg.h +++ b/include/wd_alg.h @@ -64,16 +64,43 @@ extern "C" { #endif enum alg_dev_type { - UADK_ALG_HW = 0x0, - UADK_ALG_CE_INSTR = 0x1, - UADK_ALG_SVE_INSTR = 0x2, - UADK_ALG_SOFT = 0x3, - UADK_ALG_NPU = 0x4, - UADK_ALG_GPU = 0x5, + UADK_ALG_HW, + UADK_ALG_CE_INSTR, + UADK_ALG_SVE_INSTR, + UADK_ALG_SOFT, + UADK_ALG_NPU, + UADK_ALG_GPU, UADK_ALG_TYPE_MAX, }; -/* +enum alg_drv_type { + ALG_DRV_HW = 0x0, + ALG_DRV_CE_INS, + ALG_DRV_SVE_INS, + ALG_DRV_SOFT, + ALG_DRV_INS, + ALG_DRV_FB, +}; + +/** + * struct wd_ctx_alloc_params - Minimal parameters for single context allocation. + * + * Used to pass only necessary information to driver's alloc_ctx callback. + * Keeps driver layer simple and focused. + * + * @ctx_mode: CTX_MODE_SYNC or CTX_MODE_ASYNC + * @op_type: Operation type + * @bmp: NUMA node bitmask (optional, NULL if not needed) + */ +struct wd_drv_ctx_params { + __u8 ctx_mode; + __u8 op_type; + int numa_id; + bool epoll_en; + struct bitmask *bmp; +}; + +/** * @drv_name: name of the current device driver * @alg_name: name of the algorithm supported by the driver * @priority: priority of the type of algorithm supported by the driver @@ -88,7 +115,8 @@ enum alg_dev_type { * execute the algorithm task * @op_type_num: number of modes in which the device executes the * algorithm business and requires queues to be executed separately - * @priv: pointer of priv ctx + * @priv_size: parameter memory size passed between the internal + * interfaces of the driver * @fallback: soft calculation driver handle when performing soft * calculation supplement * @init: callback interface for initializing device drivers @@ -100,6 +128,10 @@ enum alg_dev_type { * @get_usage: callback interface used to obtain the * utilization rate of devices. * @get_extend_ops: callback interface to get private operation of drivers. + * @alloc_ctx: Allocate contexts for this driver. + * HW drivers use wd_hw_alloc_ctx. + * Non-HW drivers use wd_drv_alloc_ctx_array. + * @free_ctx: Release all resources allocated by alloc_ctx. */ struct wd_alg_driver { const char *drv_name; @@ -108,15 +140,21 @@ struct wd_alg_driver { int calc_type; int queue_num; int op_type_num; - void *priv; + int priv_size; + int ops_size; + int *drv_data; + void *extend_ops; handle_t fallback; - int (*init)(struct wd_alg_driver *drv, void *conf); - void (*exit)(struct wd_alg_driver *drv); - int (*send)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); - int (*recv)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); + int (*init)(void *conf, void *priv); + void (*exit)(void *priv); + int (*send)(handle_t ctx, void *drv_msg); + int (*recv)(handle_t ctx, void *drv_msg); int (*get_usage)(void *param); int (*get_extend_ops)(void *ops); + + int (*alloc_ctx)(char *alg_name, void *params, handle_t *ctx); + void (*free_ctx)(handle_t ctx); }; struct hisi_dev_usage { @@ -134,68 +172,84 @@ struct hisi_dev_usage { int wd_alg_driver_register(struct wd_alg_driver *drv); void wd_alg_driver_unregister(struct wd_alg_driver *drv); -/* - * @alg_name: name of the algorithm supported by the driver - * @drv_name: name of the current device driver +#define MAX_DRV_ALG_NUM 64 +/** + * Secondary structure: Algorithm entry (only algorithm-specific attributes) + * @alg_name: Specific algorithm name, e.g., "cbc(aes)" + * @avaiblable: Availability depends on specific CE/SVE instructions + */ +struct wd_alg_entry { + char alg_name[ALG_NAME_SIZE]; + bool available; +}; + +/** + * Primary structure: Driver node (List backbone, contains driver-level shared attributes) + * @drv_name: name of the current device driver e.g., "hisi_sec" + * @alg_type: Algorithm class, e.g., "cipher" (Promoted to driver level) * @available: Indicates whether the current driver still has resources available * @priority: priority of the type of algorithm supported by the driver - * @calc_type: the calculation method of algorithm supported by the driver - * @refcnt: the number of times the algorithm driver is being cited by the task + * @calc_type: Driver calc type (HW, CE, SVE, SOFT) + * @refcnt: Driver-level global reference count * - * @drv: device Drivers Supporting Algorithms + * @drv: Pointer to driver implementation + * @algs: Static array for supported algorithms + * @alg_count: Current number of registered algorithms * @next: pointer to the next node of the algorithm linked list */ -struct wd_alg_list { - char alg_name[ALG_NAME_SIZE]; +struct wd_drv_node { char drv_name[DEV_NAME_LEN]; - bool available; - int priority; - int calc_type; - int refcnt; - - struct wd_alg_driver *drv; - struct wd_alg_list *next; char alg_type[ALG_NAME_SIZE]; + int priority; + int calc_type; + int refcnt; + struct wd_alg_driver *drv; + struct wd_alg_entry algs[MAX_DRV_ALG_NUM]; + int alg_count; + struct wd_drv_node *next; }; -/* +int wd_get_drv_array(const char *alg_type, int task_type, const char *drv_name, + struct wd_alg_driver ***drv_array, __u32 *drv_count); +void wd_put_drv_array(struct wd_alg_driver **drv_array, __u32 drv_count); + +void wd_alg_drv_ref_inc(struct wd_alg_driver **drv_array, __u32 drv_count); +void wd_alg_drv_ref_dec(struct wd_alg_driver **drv_array, __u32 drv_count); + +/** * wd_request_drv() - Apply for an algorithm driver. * @alg_name: task algorithm name. - * @hw_mask: the flag of shield hardware device drivers. + * @drv_type: the type of shield hardware device drivers. * * Returns the applied algorithm driver, non means error. */ -struct wd_alg_driver *wd_request_drv(const char *alg_name, bool hw_mask); -void wd_release_drv(struct wd_alg_driver *drv); +struct wd_alg_driver *wd_request_drv(const char *alg_name, int drv_type); -/* +/** * wd_drv_alg_support() - Check the algorithms supported by the driver. * @alg_name: task algorithm name. - * @drv: a device driver that supports an algorithm. + * @param: a device queue parameters. * * Return check result. */ -bool wd_drv_alg_support(const char *alg_name, - struct wd_alg_driver *drv); +bool wd_drv_alg_support(const char *alg_name, void *param); -/* - * wd_enable_drv() - Re-enable use of the current device driver. - * @drv: a device driver that supports an algorithm. +/** + * wd_alg_match_drv() - Check if a given algorithm matches a specific driver. + * @drv: Pointer to the driver instance + * @alg_name: Specific algorithm name to check (e.g., "cbc(aes)") + * + * Return: true if supported and available, false otherwise. */ -void wd_enable_drv(struct wd_alg_driver *drv); -void wd_disable_drv(struct wd_alg_driver *drv); +bool wd_alg_match_drv(struct wd_alg_driver *drv, const char *alg_name); -int wd_alg_driver_init(struct wd_alg_driver *drv, void *conf); -void wd_alg_driver_exit(struct wd_alg_driver *drv); -int wd_alg_driver_send(struct wd_alg_driver *drv, handle_t ctx, void *msg); -int wd_alg_driver_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg); int wd_alg_get_dev_usage(const char *dev_name, const char *alg_type, __u8 op_type); int wd_get_alg_type(const char *alg_name, char *alg_type); -struct wd_alg_list *wd_get_alg_head(void); +struct wd_drv_node *wd_get_alg_head(void); #ifdef WD_STATIC_DRV -/* +/** * duplicate drivers will be skipped when it register to alg_list */ void hisi_sec2_probe(void); diff --git a/include/wd_internal.h b/include/wd_internal.h index 62cf1a9..3e8622c 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -7,7 +7,6 @@ #define WD_INTERNAL_H #include <pthread.h> -#include <stdatomic.h> #include <stdbool.h> #include "wd.h" #include "wd_alg.h" diff --git a/libwd.map b/libwd.map index 0635198..683a88f 100644 --- a/libwd.map +++ b/libwd.map @@ -45,11 +45,16 @@ global: wd_alg_driver_register; wd_alg_driver_unregister; wd_request_drv; - wd_release_drv; wd_drv_alg_support; wd_enable_drv; wd_disable_drv; wd_get_alg_head; + wd_alg_match_drv; + wd_get_drv_array; + wd_put_drv_array; + wd_alg_drv_ref_inc; + wd_alg_drv_ref_dec; + wd_alg_driver_init; wd_alg_driver_exit; wd_alg_driver_send; diff --git a/wd_alg.c b/wd_alg.c index 1e4f14a..e288bd0 100644 --- a/wd_alg.c +++ b/wd_alg.c @@ -6,23 +6,32 @@ #define _GNU_SOURCE #include <dirent.h> #include <errno.h> +#include <stdio.h> #include <stdbool.h> #include <stdlib.h> #include <pthread.h> #include <sys/auxv.h> #include "wd.h" -#include "wd_alg.h" +#include "wd_alg_common.h" #define SYS_CLASS_DIR "/sys/class/uacce" -#define SVA_FILE_NAME "flags" -#define DEV_SVA_SIZE 32 -#define STR_DECIMAL 0xA -static struct wd_alg_list alg_list_head; -static struct wd_alg_list *alg_list_tail = &alg_list_head; +/* Registry structure (List manager) */ +struct wd_alg_registry { + struct wd_drv_node *head; + struct wd_drv_node *tail; + pthread_mutex_t mutex; + int drv_type_num; /* Number of unique driver nodes in the list */ +}; -static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static struct wd_drv_node drv_list_head; +static struct wd_alg_registry alg_registry = { + .head = &drv_list_head, + .tail = &drv_list_head, + .mutex = PTHREAD_MUTEX_INITIALIZER, + .drv_type_num = 0, +}; struct acc_alg_item { const char *name; @@ -30,25 +39,25 @@ struct acc_alg_item { }; static struct acc_alg_item alg_options[] = { - {"zlib", "zlib"}, - {"gzip", "gzip"}, - {"deflate", "deflate"}, - {"lz77_zstd", "lz77_zstd"}, - {"lz4", "lz4"}, - {"lz77_only", "lz77_only"}, + {"zlib", "comp"}, + {"gzip", "comp"}, + {"deflate", "comp"}, + {"lz77_zstd", "comp"}, + {"lz4", "comp"}, + {"lz77_only", "comp"}, {"hashagg", "hashagg"}, {"udma", "udma"}, {"hashjoin", "hashjoin"}, - {"gather", "gather"}, + {"gather", "hashjoin"}, {"join-gather", "hashjoin"}, {"rsa", "rsa"}, {"dh", "dh"}, - {"ecdh", "ecdh"}, - {"x25519", "x25519"}, - {"x448", "x448"}, - {"ecdsa", "ecdsa"}, - {"sm2", "sm2"}, + {"ecdh", "ecc"}, + {"x25519", "ecc"}, + {"x448", "ecc"}, + {"ecdsa", "ecc"}, + {"sm2", "ecc"}, {"ecb(aes)", "cipher"}, {"cbc(aes)", "cipher"}, @@ -72,6 +81,7 @@ static struct acc_alg_item alg_options[] = { {"cbc(des)", "cipher"}, {"ecb(des3_ede)", "cipher"}, {"cbc(des3_ede)", "cipher"}, + {"xts-gb(sm4)", "cipher"}, {"ccm(aes)", "aead"}, {"gcm(aes)", "aead"}, @@ -108,7 +118,7 @@ int wd_get_alg_type(const char *alg_name, char *alg_type) for (i = 0; i < ARRAY_SIZE(alg_options); i++) { if (strcmp(alg_name, alg_options[i].name) == 0) { (void)strcpy(alg_type, alg_options[i].algtype); - return 0; + return WD_SUCCESS; } } @@ -122,7 +132,7 @@ static bool wd_check_accel_dev(const char *dev_name) wd_class = opendir(SYS_CLASS_DIR); if (!wd_class) { - WD_ERR("UADK framework isn't enabled in system!\n"); + WD_ERR("invalid: UADK framework isn't enabled in system!\n"); return false; } @@ -143,20 +153,17 @@ static bool wd_check_accel_dev(const char *dev_name) static bool wd_check_ce_support(const char *alg_name) { - unsigned long support_sm3 = 0; - unsigned long support_sm4 = 0; + unsigned long hwcaps = 0; const char *alg_tail; size_t tail_len; size_t alg_len; - #if defined(__aarch64__) - unsigned long hwcaps = 0; - + #if defined(__arm__) || defined(__arm) + hwcaps = getauxval(AT_HWCAP2); + #elif defined(__aarch64__) hwcaps = getauxval(AT_HWCAP); - support_sm3 = hwcaps & HWCAP_CE_SM3; - support_sm4 = hwcaps & HWCAP_CE_SM4; #endif - if (!strcmp("sm3", alg_name) && support_sm3) + if (!strcmp("sm3", alg_name) && (hwcaps & HWCAP_CE_SM3)) return true; alg_len = strlen(alg_name); @@ -165,7 +172,7 @@ static bool wd_check_ce_support(const char *alg_name) return false; alg_tail = alg_name + (alg_len - tail_len); - if (!strcmp("(sm4)", alg_tail) && support_sm4) + if (!strcmp("(sm4)", alg_tail) && (hwcaps & HWCAP_CE_SM4)) return true; return false; @@ -177,9 +184,8 @@ static bool wd_check_sve_support(void) #if defined(__aarch64__) hwcaps = getauxval(AT_HWCAP); - hwcaps &= HWCAP_SVE; #endif - if (hwcaps) + if (hwcaps & HWCAP_SVE) return true; return false; @@ -212,47 +218,36 @@ static bool wd_alg_check_available(int calc_type, return ret; } -static bool wd_alg_driver_match(struct wd_alg_driver *drv, - struct wd_alg_list *node) +/** + * Mapping from task_type to calc_type filter: + * + * TASK_HW → calc_type == UADK_ALG_HW + * TASK_INSTR → calc_type != UADK_ALG_HW (CE_INSTR | SVE_INSTR | SOFT) + * TASK_MIX → all calc_type values + */ +static inline bool wd_alg_drv_type_match(int task_type, int drv_calc_type) { - if (strcmp(drv->alg_name, node->alg_name)) - return false; - - if (strcmp(drv->drv_name, node->drv_name)) - return false; - - if (drv->priority != node->priority) - return false; - - if (drv->calc_type != node->calc_type) + switch (task_type) { + case TASK_HW: + return drv_calc_type == UADK_ALG_HW || + drv_calc_type == UADK_ALG_NPU || + drv_calc_type == UADK_ALG_GPU; + case TASK_INSTR: + return drv_calc_type == UADK_ALG_CE_INSTR || + drv_calc_type == UADK_ALG_SVE_INSTR; + case TASK_MIX: + return true; + default: return false; - - return true; -} - -static bool wd_alg_repeat_check(struct wd_alg_driver *drv) -{ - struct wd_alg_list *npre = &alg_list_head; - struct wd_alg_list *pnext = NULL; - - pthread_mutex_lock(&mutex); - pnext = npre->next; - while (pnext) { - if (wd_alg_driver_match(drv, pnext)) { - pthread_mutex_unlock(&mutex); - return true; - } - npre = pnext; - pnext = pnext->next; } - pthread_mutex_unlock(&mutex); - - return false; } int wd_alg_driver_register(struct wd_alg_driver *drv) { - struct wd_alg_list *new_alg; + struct wd_drv_node *node; + struct wd_drv_node *target_node = NULL; + char alg_type[ALG_NAME_SIZE]; + int i, ret; if (!drv) { WD_ERR("invalid: register drv is NULL!\n"); @@ -264,150 +259,261 @@ int wd_alg_driver_register(struct wd_alg_driver *drv) return -WD_EINVAL; } - if (wd_alg_repeat_check(drv)) - return 0; - - new_alg = calloc(1, sizeof(struct wd_alg_list)); - if (!new_alg) { - WD_ERR("failed to alloc alg driver memory!\n"); - return -WD_ENOMEM; + ret = wd_get_alg_type(drv->alg_name, alg_type); + if (ret) { + WD_ERR("failed to get alg_type for %s!\n", drv->alg_name); + return -WD_EINVAL; } - (void)wd_get_alg_type(drv->alg_name, new_alg->alg_type); - strncpy(new_alg->alg_name, drv->alg_name, ALG_NAME_SIZE - 1); - strncpy(new_alg->drv_name, drv->drv_name, DEV_NAME_LEN - 1); - new_alg->priority = drv->priority; - new_alg->calc_type = drv->calc_type; - new_alg->drv = drv; - new_alg->refcnt = 0; - new_alg->next = NULL; - - new_alg->available = wd_alg_check_available(drv->calc_type, - drv->alg_name, drv->drv_name); - if (!new_alg->available) { - free(new_alg); - return -WD_ENODEV; + pthread_mutex_lock(&alg_registry.mutex); + node = alg_registry.head->next; + /* Search for an existing node with the same drv_name */ + while (node) { + if (strcmp(node->drv_name, drv->drv_name) == 0 && + strcmp(node->alg_type, alg_type) == 0) { + target_node = node; + break; + } + node = node->next; } - pthread_mutex_lock(&mutex); - alg_list_tail->next = new_alg; - alg_list_tail = new_alg; - pthread_mutex_unlock(&mutex); + if (target_node) { + /* Consistency check: a driver must strictly have uniform properties */ + if (target_node->priority != drv->priority || + target_node->calc_type != drv->calc_type) { + WD_ERR("invalid: driver %s attributes mismatch on re-register!\n", + drv->drv_name); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_EINVAL; + } - return 0; + /* Check if alg_name already exists in this driver's array */ + for (i = 0; i < target_node->alg_count; i++) { + if (strcmp(target_node->algs[i].alg_name, drv->alg_name) == 0) { + /* Algorithm already registered, skip duplicate */ + pthread_mutex_unlock(&alg_registry.mutex); + return WD_SUCCESS; + } + } + + /* Check array capacity */ + if (target_node->alg_count >= MAX_DRV_ALG_NUM) { + WD_ERR("invalid: driver %s alg array overflow (max %d)!\n", + drv->drv_name, MAX_DRV_ALG_NUM); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_ENOMEM; + } + + /* Add new algorithm to existing driver node */ + strncpy(target_node->algs[target_node->alg_count].alg_name, + drv->alg_name, ALG_NAME_SIZE - 1); + target_node->algs[target_node->alg_count].alg_name[ALG_NAME_SIZE - 1] = '\0'; + target_node->algs[target_node->alg_count].available = + wd_alg_check_available(drv->calc_type, drv->alg_name, drv->drv_name); + if (!target_node->algs[target_node->alg_count].available) { + WD_ERR("invalid: driver %s alg %s not available on current system!\n", + drv->drv_name, drv->alg_name); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_ENODEV; + } + target_node->alg_count++; + } else { + /* Create a new driver node */ + target_node = calloc(1, sizeof(struct wd_drv_node)); + if (!target_node) { + WD_ERR("failed to alloc drv node memory!\n"); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_ENOMEM; + } + + strncpy(target_node->drv_name, drv->drv_name, DEV_NAME_LEN - 1); + target_node->drv_name[DEV_NAME_LEN - 1] = '\0'; + snprintf(target_node->alg_type, ALG_NAME_SIZE, "%s", alg_type); + target_node->priority = drv->priority; + target_node->calc_type = drv->calc_type; + target_node->drv = drv; + target_node->refcnt = 0; + target_node->alg_count = 0; + + /* Add the first algorithm to the new node's array */ + strncpy(target_node->algs[0].alg_name, drv->alg_name, ALG_NAME_SIZE - 1); + target_node->algs[0].alg_name[ALG_NAME_SIZE - 1] = '\0'; + target_node->algs[0].available = + wd_alg_check_available(drv->calc_type, drv->alg_name, drv->drv_name); + if (!target_node->algs[0].available) { + free(target_node); + WD_INFO("info: driver %s alg %s has no device register!\n", + drv->drv_name, drv->alg_name); + pthread_mutex_unlock(&alg_registry.mutex); + return -WD_ENODEV; + } + target_node->alg_count = 1; + target_node->next = NULL; + + /* Append to list tail */ + alg_registry.tail->next = target_node; + alg_registry.tail = target_node; + __atomic_fetch_add(&alg_registry.drv_type_num, 1, __ATOMIC_RELAXED); + } + + pthread_mutex_unlock(&alg_registry.mutex); + return WD_SUCCESS; } void wd_alg_driver_unregister(struct wd_alg_driver *drv) { - struct wd_alg_list *npre = &alg_list_head; - struct wd_alg_list *pnext = npre->next; + struct wd_drv_node *npre = alg_registry.head; + struct wd_drv_node *pnext; + char alg_type[ALG_NAME_SIZE]; + int i, ret; - /* Alg driver list has no drivers */ - if (!pnext || !drv) + if (!npre || !drv) return; - pthread_mutex_lock(&mutex); + ret = wd_get_alg_type(drv->alg_name, alg_type); + if (ret) { + WD_ERR("failed to get alg_type for %s!\n", drv->alg_name); + return; + } + + pthread_mutex_lock(&alg_registry.mutex); + /* Find the driver node matching drv_name */ + pnext = npre->next; while (pnext) { - if (wd_alg_driver_match(drv, pnext)) + if (strcmp(drv->drv_name, pnext->drv_name) == 0 && + strcmp(pnext->alg_type, alg_type) == 0) break; npre = pnext; pnext = pnext->next; } - /* The current algorithm is not registered */ if (!pnext) { - pthread_mutex_unlock(&mutex); + pthread_mutex_unlock(&alg_registry.mutex); return; } - /* Used to locate the problem and ensure symmetrical use driver */ - if (pnext->refcnt > 0) - WD_ERR("driver<%s> still in used: %d\n", pnext->drv_name, pnext->refcnt); + /* Find and remove the specific alg_name from the node's array */ + for (i = 0; i < pnext->alg_count; i++) { + if (strcmp(pnext->algs[i].alg_name, drv->alg_name) == 0) { + /* Compact the array: move the last element to the removed slot */ + if (i != pnext->alg_count - 1) + pnext->algs[i] = pnext->algs[pnext->alg_count - 1]; + pnext->alg_count--; + break; + } + } + + /* If the driver no longer supports any algorithms, remove the entire node */ + if (!pnext->alg_count) { + if (pnext->refcnt > 0) + WD_INFO("info: release driver <%s> as it is still in use: %d!\n", + pnext->drv_name, pnext->refcnt); + + if (pnext == alg_registry.tail) + alg_registry.tail = npre; - if (pnext == alg_list_tail) - alg_list_tail = npre; + npre->next = pnext->next; + free(pnext); + if (alg_registry.drv_type_num > 0) + __atomic_fetch_sub(&alg_registry.drv_type_num, 1, __ATOMIC_RELAXED); + } - npre->next = pnext->next; - free(pnext); - pthread_mutex_unlock(&mutex); + pthread_mutex_unlock(&alg_registry.mutex); } -struct wd_alg_list *wd_get_alg_head(void) +struct wd_drv_node *wd_get_alg_head(void) { - return &alg_list_head; + return alg_registry.head; } -bool wd_drv_alg_support(const char *alg_name, - struct wd_alg_driver *drv) +/** + * wd_alg_match_drv() - Check if a given algorithm match a specific driver. + * @drv: Pointer to the driver instance + * @alg_name: Specific algorithm name to check (e.g., "cbc(aes)") + * + * Uses the new hierarchical structure: finds the driver node, then searches + * its internal static algorithm array. + * + * Return: true if supported and available, false otherwise. + */ +bool wd_alg_match_drv(struct wd_alg_driver *drv, const char *alg_name) { - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; + struct wd_drv_node *node; + int i; - if (!alg_name || !drv) + if (!drv || !alg_name) return false; - while (pnext) { - if (!strcmp(alg_name, pnext->alg_name) && - !strcmp(drv->drv_name, pnext->drv_name)) { - return true; + pthread_mutex_lock(&alg_registry.mutex); + node = alg_registry.head->next; + while (node) { + if (node->drv == drv) { + /* Found the driver node, now search its algs array */ + for (i = 0; i < node->alg_count; i++) { + if (!strcmp(node->algs[i].alg_name, alg_name) && + node->algs[i].available) { + pthread_mutex_unlock(&alg_registry.mutex); + return true; + } + } + /* Driver found, but algorithm not in its array or not available */ + pthread_mutex_unlock(&alg_registry.mutex); + return false; } - pnext = pnext->next; + node = node->next; } + pthread_mutex_unlock(&alg_registry.mutex); return false; } -void wd_enable_drv(struct wd_alg_driver *drv) +bool wd_drv_alg_support(const char *alg_name, void *param) { - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; + struct wd_ctx_config_internal *config = param; + struct wd_drv_node *head = alg_registry.head; + struct wd_drv_node *node; + __u32 i; + int j; - if (!pnext || !drv) - return; - - pthread_mutex_lock(&mutex); - while (pnext) { - if (wd_alg_driver_match(drv, pnext)) - break; - pnext = pnext->next; - } - - if (pnext) - pnext->available = wd_alg_check_available(drv->calc_type, - drv->alg_name, drv->drv_name); - pthread_mutex_unlock(&mutex); -} - -void wd_disable_drv(struct wd_alg_driver *drv) -{ - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; - - if (!pnext || !drv) - return; + if (!alg_name || !config || !config->ctxs) + return false; - pthread_mutex_lock(&mutex); - while (pnext) { - if (wd_alg_driver_match(drv, pnext) && pnext->available) - break; - pnext = pnext->next; + pthread_mutex_lock(&alg_registry.mutex); + /* Check whether the currently allocated ctxs supports the specified algorithm. */ + for (i = 0; i < config->ctx_num; i++) { + if (!config->ctxs[i].drv) + continue; + node = head->next; + while (node) { + /* Query the position of the driver matching the context in the list. */ + if (!strcmp(config->ctxs[i].drv->drv_name, node->drv_name)) { + for (j = 0; j < node->alg_count; j++) { + if (!strcmp(alg_name, node->algs[j].alg_name) && + node->algs[j].available) { + pthread_mutex_unlock(&alg_registry.mutex); + return true; + } + } + } + node = node->next; + } } + pthread_mutex_unlock(&alg_registry.mutex); - if (pnext) - pnext->available = false; - pthread_mutex_unlock(&mutex); + return false; } -struct wd_alg_driver *wd_request_drv(const char *alg_name, bool hw_mask) +struct wd_alg_driver *wd_request_drv(const char *alg_name, int drv_type) { - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; - struct wd_alg_list *select_node = NULL; + struct wd_drv_node *node = alg_registry.head->next; + struct wd_drv_node *drv_node = NULL; struct wd_alg_driver *drv = NULL; + bool type_match = false; int tmp_priority = -1; + int i; - if (!pnext) { - WD_ERR("invalid: requset drv pnext is NULL!\n"); + if (!node) { + WD_ERR("invalid: request drv node is NULL!\n"); return NULL; } @@ -416,106 +522,286 @@ struct wd_alg_driver *wd_request_drv(const char *alg_name, bool hw_mask) return NULL; } - /* Check the list to get an best driver */ - pthread_mutex_lock(&mutex); - while (pnext) { - /* hw_mask true mean not to used hardware dev */ - if ((hw_mask && pnext->drv->calc_type == UADK_ALG_HW) || - (!hw_mask && pnext->drv->calc_type != UADK_ALG_HW)) { - pnext = pnext->next; - continue; - } - - if (!strcmp(alg_name, pnext->alg_name) && pnext->available && - pnext->drv->priority > tmp_priority) { - tmp_priority = pnext->drv->priority; - select_node = pnext; - drv = pnext->drv; + pthread_mutex_lock(&alg_registry.mutex); + while (node) { + type_match = false; + /* Check calc_type against requested drv_type */ + if (drv_type == ALG_DRV_FB && (node->calc_type == UADK_ALG_SOFT || + node->calc_type == UADK_ALG_CE_INSTR || + node->calc_type == UADK_ALG_SVE_INSTR)) + type_match = true; + + if (type_match && node->drv->priority > tmp_priority) { + /* Check if this driver supports the requested alg_name */ + for (i = 0; i < node->alg_count; i++) { + if (!strcmp(alg_name, node->algs[i].alg_name) && + node->algs[i].available) { + drv_node = node; + drv = node->drv; + tmp_priority = node->drv->priority; + break; + } + } } - pnext = pnext->next; + node = node->next; } - if (select_node) - select_node->refcnt++; - pthread_mutex_unlock(&mutex); + /* Increment refcnt on the selected driver node */ + if (drv) + drv_node->refcnt++; + pthread_mutex_unlock(&alg_registry.mutex); return drv; } -void wd_release_drv(struct wd_alg_driver *drv) +int wd_alg_get_dev_usage(const char *dev_name, const char *alg_type, __u8 alg_op_type) { - struct wd_alg_list *head = &alg_list_head; - struct wd_alg_list *pnext = head->next; - struct wd_alg_list *select_node = NULL; + struct wd_drv_node *node = alg_registry.head->next; + struct hisi_dev_usage dev_usage; + struct wd_alg_driver *drv; - if (!pnext || !drv) - return; + if (!dev_name || !alg_type) { + WD_ERR("invalid: dev_name or alg_type is NULL!\n"); + return -WD_EINVAL; + } - pthread_mutex_lock(&mutex); - while (pnext) { - if (wd_alg_driver_match(drv, pnext) && pnext->refcnt > 0) { - select_node = pnext; + while (node) { + /* Match dev_name and alg_type at the driver node level */ + if (strstr(dev_name, node->drv_name) && + !strcmp(alg_type, node->alg_type)) break; - } - pnext = pnext->next; + + node = node->next; } - if (select_node) - select_node->refcnt--; - pthread_mutex_unlock(&mutex); -} + if (!node) + return -WD_EACCES; -int wd_alg_driver_init(struct wd_alg_driver *drv, void *conf) -{ - return drv->init(drv, conf); -} + drv = node->drv; + if (!drv->get_usage) + return -WD_EINVAL; -void wd_alg_driver_exit(struct wd_alg_driver *drv) -{ - drv->exit(drv); + dev_usage.drv = drv; + dev_usage.alg_op_type = alg_op_type; + dev_usage.dev_name = dev_name; + + return drv->get_usage(&dev_usage); } -int wd_alg_driver_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +/** + * wd_put_drv_array() - Release driver array allocated by wd_get_drv_array(). + * + * Frees the driver pointer array. Does NOT touch the drivers themselves + * (refcount managed separately by wd_alg_drv_ref_inc/dec). + * + * @drv_array: Driver array from wd_get_drv_array() + * @drv_count: Number of entries (unused, for API symmetry) + */ +void wd_put_drv_array(struct wd_alg_driver **drv_array, const __u32 drv_count) { - return drv->send(drv, ctx, msg); + if (drv_array) + free(drv_array); } -int wd_alg_driver_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int wd_compare_drv_priority(const void *a, const void *b) { - return drv->recv(drv, ctx, msg); + struct wd_alg_driver *driver_a = *(struct wd_alg_driver **)a; + struct wd_alg_driver *driver_b = *(struct wd_alg_driver **)b; + + /* Higher priority value should come first */ + if (driver_a->priority > driver_b->priority) + return -1; + /* a has lower priority, should come after b */ + else if (driver_a->priority < driver_b->priority) + return 1; + /* equal priority */ + return 0; } -int wd_alg_get_dev_usage(const char *dev_name, const char *alg_type, __u8 alg_op_type) +/** + * wd_get_drv_array() - Discover all unique drivers matching alg_type and task_type. + * + * @alg_type: Algorithm class string ("cipher", "digest", "aead", "comp", etc.) + * @task_type: TASK_HW (hardware only), TASK_INSTR (instruction only), TASK_MIX (all) + * @drv_array: Output - newly allocated array of unique wd_alg_driver* pointers, + * caller must free with plain free() + * @drv_count: Output - number of unique drivers found + * + * Traverses wd_drv_node list once: + * 1. Matches by alg_type at node level (no need to traverse algs array for this). + * 2. Filters by task_type using wd_alg_drv_type_match(). + * 3. Deduplicates is inherently solved (each node is a unique driver). + * + * This is a PURE QUERY — no reference counting or resource allocation side effects. + * Reference counting is done separately by wd_alg_drv_ref_inc/dec(). + * + * Return: 0 on success, negative on failure. + */ +int wd_get_drv_array(const char *alg_type, int task_type, const char *drv_name, + struct wd_alg_driver ***drv_array, __u32 *drv_count) { - struct wd_alg_list *pnext = alg_list_head.next; - struct hisi_dev_usage dev_usage; - struct wd_alg_driver *drv; - size_t len; + struct wd_drv_node *head, *node; + struct wd_alg_driver **drivers; + __u32 max_driver_count; + __u32 current_count = 0; + bool has_available_alg; + int i; + + if (!alg_type || !drv_array || !drv_count) { + WD_ERR("invalid: NULL parameter!\n"); + return -WD_EINVAL; + } - if (!dev_name || !alg_type) { - WD_ERR("dev_name or alg_type is NULL!\n"); + *drv_array = NULL; + *drv_count = 0; + head = wd_get_alg_head(); + if (!head) { + WD_ERR("failed to get alg list head!\n"); return -WD_EINVAL; } - while (pnext) { - len = strlen(pnext->drv_name); - if (!strncmp(dev_name, pnext->drv_name, len) && *(dev_name + len) == '-' && - !strcmp(alg_type, pnext->alg_type) && pnext->drv->priv) - break; + max_driver_count = __atomic_load_n(&alg_registry.drv_type_num, __ATOMIC_RELAXED); + if (!max_driver_count) { + WD_ERR("invalid: no drivers registered for alg_type: %s\n", alg_type); + return -WD_EINVAL; + } - pnext = pnext->next; + drivers = calloc(max_driver_count, sizeof(struct wd_alg_driver *)); + if (!drivers) { + WD_ERR("failed to allocate drivers array!\n"); + return -WD_ENOMEM; } - if (!pnext) - return -WD_EACCES; + /* + * Single traversal of wd_drv_node list: + * - Match by alg_type at node level + * - Filter by task_type + * - Deduplication inherently solved + */ + node = head->next; + while (node) { + if (strcmp(node->alg_type, alg_type) == 0 && + wd_alg_drv_type_match(task_type, node->calc_type)) { + + if (drv_name && strcmp(node->drv_name, drv_name) != 0) { + node = node->next; + continue; + } + + /* Check if at least one algorithm in this driver is available */ + has_available_alg = false; + for (i = 0; i < node->alg_count; i++) { + if (node->algs[i].available) { + has_available_alg = true; + break; + } + } + + if (!has_available_alg) { + node = node->next; + continue; + } + + if (current_count >= max_driver_count) { + WD_ERR("failed to check driver array overflow!\n"); + goto query_failed; + } + drivers[current_count] = node->drv; + current_count++; + } + node = node->next; + } - drv = pnext->drv; - if (!drv->get_usage) - return -WD_EINVAL; + if (!current_count) { + WD_ERR("invalid: no available drivers for alg_type: %s, task_type: %d\n", + alg_type, task_type); + goto query_failed; + } - dev_usage.drv = drv; - dev_usage.alg_op_type = alg_op_type; - dev_usage.dev_name = dev_name; + /* Sort drivers by priority (higher priority first) */ + if (current_count > 1) + qsort(drivers, current_count, sizeof(struct wd_alg_driver *), + wd_compare_drv_priority); - return drv->get_usage(&dev_usage); + WD_DEBUG("Driver discovery: %u unique drivers for alg_type=%s\n", + current_count, alg_type); + *drv_array = drivers; + *drv_count = current_count; + + return WD_SUCCESS; + +query_failed: + free(drivers); + return -WD_EINVAL; +} + +/** + * wd_alg_drv_ref_inc() - Increment reference count for each unique driver. + * + * @drv_array: Array of unique driver pointers + * @drv_count: Number of drivers in the array + * + * For each unique driver, finds its node in wd_drv_node list and + * increments refcnt by exactly 1. This ensures refcnt reflects the + * number of configs using the driver, not the number of ctxs. + * + * Must be called after wd_get_drv_array() and after ctx binding. + */ +void wd_alg_drv_ref_inc(struct wd_alg_driver **drv_array, __u32 drv_count) +{ + struct wd_drv_node *node; + __u32 i; + + if (!drv_array || !drv_count) + return; + + pthread_mutex_lock(&alg_registry.mutex); + for (i = 0; i < drv_count; i++) { + if (!drv_array[i]) + continue; + /* Directly find the unique driver node and increment refcnt */ + node = alg_registry.head->next; + while (node) { + if (node->drv == drv_array[i]) { + node->refcnt++; + break; + } + node = node->next; + } + } + pthread_mutex_unlock(&alg_registry.mutex); +} + +/** + * wd_alg_drv_ref_dec() - Decrement reference count for each unique driver. + * + * @drv_array: Array of unique driver pointers + * @drv_count: Number of drivers in the array + * + * Inverse of wd_alg_drv_ref_inc(). Decrements refcnt by 1 for each + * unique driver. Must be called during cleanup. + */ +void wd_alg_drv_ref_dec(struct wd_alg_driver **drv_array, __u32 drv_count) +{ + struct wd_drv_node *node; + __u32 i; + + if (!drv_array || !drv_count) + return; + + pthread_mutex_lock(&alg_registry.mutex); + for (i = 0; i < drv_count; i++) { + if (!drv_array[i]) + continue; + /* Directly find the unique driver node and decrement refcnt */ + node = alg_registry.head->next; + while (node) { + if (node->drv == drv_array[i] && node->refcnt > 0) { + node->refcnt--; + break; + } + node = node->next; + } + } + pthread_mutex_unlock(&alg_registry.mutex); } -- 2.43.0
From: Junchong Pan <panjunchong@h-partners.com> In the updated UADK framework, the management functions for hardware queue allocation and release have been decoupled from the framework layer to the user-space driver layer. However, for existing uacce hardware queues and pure software computation queues, we have created a set of common allocation and release interfaces, thereby reducing redundancy in the driver layer code and improving the reusability of the framework code. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> --- Makefile.am | 9 +- drv/wd_drv.c | 340 +++++++++++ drv/wd_drv.h | 62 ++ wd_util.c | 1623 ++++++++++++++++++++++++++++++++------------------ 4 files changed, 1457 insertions(+), 577 deletions(-) create mode 100644 drv/wd_drv.c create mode 100644 drv/wd_drv.h diff --git a/Makefile.am b/Makefile.am index 5cc8c39..c08c825 100644 --- a/Makefile.am +++ b/Makefile.am @@ -84,6 +84,7 @@ libhisi_zip_la_SOURCES=drv/hisi_comp.c hisi_comp.h drv/hisi_qm_udrv.c \ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ wd_aead.c wd_aead.h wd_aead_drv.h \ + wd.c wd.h wd_alg.h \ wd_rsa.c wd_rsa.h wd_rsa_drv.h \ wd_dh.c wd_dh.h wd_dh_drv.h \ wd_ecc.c wd_ecc.h wd_ecc_drv.h \ @@ -93,24 +94,24 @@ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ lib/crypto/aes.c lib/crypto/sm4.c lib/crypto/galois.c \ - hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h sm4.h galois.h + hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h sm4.h galois.h \n drv/wd_drv.h drv/wd_drv.c libhisi_hpre_la_SOURCES=drv/hisi_hpre.c drv/hisi_qm_udrv.c \ hisi_qm_udrv.h if ARCH_ARM64 libisa_ce_la_SOURCES=arm_arch_ce.h drv/isa_ce_sm3.c drv/isa_ce_sm3_armv8.S isa_ce_sm3.h \ - drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h + drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h wd_util.c wd_util.h \n drv/wd_drv.h drv/wd_drv.c libisa_sve_la_SOURCES=drv/hash_mb/hash_mb.c wd_digest_drv.h drv/hash_mb/hash_mb.h \ drv/hash_mb/sm3_sve_common.S drv/hash_mb/sm3_mb_asimd_x1.S \ drv/hash_mb/sm3_mb_asimd_x4.S drv/hash_mb/sm3_mb_sve.S \ drv/hash_mb/md5_sve_common.S drv/hash_mb/md5_mb_asimd_x1.S \ - drv/hash_mb/md5_mb_asimd_x4.S drv/hash_mb/md5_mb_sve.S + drv/hash_mb/md5_mb_asimd_x4.S drv/hash_mb/md5_mb_sve.S \n drv/wd_drv.h drv/wd_drv.c endif libhisi_dae_la_SOURCES=drv/hisi_dae.c hisi_dae.h drv/hisi_qm_udrv.c \ - hisi_qm_udrv.h drv/hisi_dae_join_gather.c drv/hisi_dae_common.c + hisi_qm_udrv.h drv/hisi_dae_join_gather.c drv/hisi_dae_common.c \n drv/wd_drv.h drv/wd_drv.c libhisi_udma_la_SOURCES=drv/hisi_udma.c drv/hisi_qm_udrv.c \ hisi_qm_udrv.h diff --git a/drv/wd_drv.c b/drv/wd_drv.c new file mode 100644 index 0000000..d011d3f --- /dev/null +++ b/drv/wd_drv.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: Apache-2.0 +/* Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved. */ +#include <stdlib.h> +#include <sched.h> + +#include "wd_internal.h" +#include "wd_alg.h" +#include "wd_util.h" +#include "wd_drv.h" + +int wd_soft_alloc_ctx(char *alg_name, void *params, handle_t *ctx) +{ + struct wd_drv_ctx_params *ctx_params = (struct wd_drv_ctx_params *)params; + struct wd_soft_ctx *sfctx; + + if (!params || !ctx) { + WD_ERR("invalid: params, or ctx is NULL!\n"); + return -WD_EINVAL; + } + + /* Allocate ONE software context structure */ + sfctx = calloc(1, sizeof(struct wd_soft_ctx)); + if (!sfctx) { + WD_ERR("failed to alloc ctx!\n"); + return -WD_ENOMEM; + } + + /* Initialize as software context */ + sfctx->fd = -1; + sfctx->ctx_type = UADK_ALG_SOFT; + pthread_spin_init(&sfctx->slock, PTHREAD_PROCESS_PRIVATE); + pthread_spin_init(&sfctx->rlock, PTHREAD_PROCESS_PRIVATE); + + /* Return context handle */ + *ctx = (handle_t)sfctx; + + return WD_SUCCESS; +} + +void wd_soft_free_ctx(handle_t ctx) +{ + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; + + if (!sfctx) { + WD_ERR("invalid: ctx is NULL!\n"); + return; + } + + /* Simply free the allocated wd_ctx_h structure */ + pthread_spin_destroy(&sfctx->slock); + pthread_spin_destroy(&sfctx->rlock); + free(sfctx); +} + +static int wd_compare_dev_distance(const void *a, const void *b) +{ + struct uacce_dev_list *node_a = *(struct uacce_dev_list **)a; + struct uacce_dev_list *node_b = *(struct uacce_dev_list **)b; + unsigned int curr_node; + int dist_a, dist_b; + + if (getcpu(NULL, &curr_node) || curr_node == (unsigned int)NUMA_NO_NODE) + return 0; + + dist_a = numa_distance((int)curr_node, node_a->dev->numa_id); + dist_b = numa_distance((int)curr_node, node_b->dev->numa_id); + + return dist_a - dist_b; +} + +static struct uacce_dev_list *wd_sort_dev_list(struct uacce_dev_list *list, int list_count) +{ + struct uacce_dev_list *p, **nodes = NULL; + struct uacce_dev_list *result = NULL; + int i; + + if (!list || !list_count) + return NULL; + + /* Convert to array */ + nodes = calloc(list_count, sizeof(struct uacce_dev_list *)); + if (!nodes) + return list; /* Return original list on allocation failure */ + + p = list; + for (i = 0; i < list_count; i++) { + if (!p) + break; + nodes[i] = p; + p = p->next; + } + + /* Sort by NUMA distance */ + qsort(nodes, list_count, sizeof(struct uacce_dev_list *), wd_compare_dev_distance); + + /* Rebuild sorted list */ + for (i = 0; i < list_count; i++) { + nodes[i]->next = NULL; + if (!result) + result = nodes[i]; + else + wd_add_dev_to_list(result, nodes[i]); + } + free(nodes); + + return result; +} + +struct uacce_dev_list *wd_get_usable_list(struct uacce_dev_list *list, int target_numa) +{ + struct uacce_dev_list *p, *node, *result = NULL; + struct uacce_dev_list *ret, *head = NULL; + struct uacce_dev *dev; + int count = 0; + int numa_id; + + p = list; + while (p) { + dev = p->dev; + numa_id = dev->numa_id; + if (numa_id != target_numa) { + p = p->next; + continue; + } + + node = calloc(1, sizeof(*node)); + if (!node) { + ret = WD_ERR_PTR(-WD_ENOMEM); + goto out_free_list; + } + + node->dev = wd_clone_dev(dev); + if (!node->dev) { + ret = WD_ERR_PTR(-WD_ENOMEM); + goto out_free_node; + } + + if (!head) + head = node; + else + wd_add_dev_to_list(head, node); + + count++; + p = p->next; + } + + if (!count) + return NULL; + + /* Sort by NUMA distance */ + result = wd_sort_dev_list(head, count); + if (!result) { + ret = WD_ERR_PTR(-WD_ENODEV); + goto out_free_list; + } + + return result; + +out_free_node: + free(node); +out_free_list: + wd_free_list_accels(head); + return ret; +} + +/** + * wd_hw_alloc_ctx() - HW driver's alloc_ctx callback. + * + * Allocates ONE hardware context from UACCE device. + * Device selection strategy: + * 1. Filter devices by bmp (NUMA bitmask) + * 2. Sort by NUMA distance (nearest first) + * 3. Prefer devices on target_numa; fall back to others in distance order + * + * @alg_name: The algorithm name + * @params: Minimal allocation parameters (ctx_mode, op_type, numa_id, bmp) + * @ctx: (output) Allocated context handle + * + * Return: 0 on success, negative on failure + */ +int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) +{ + struct wd_drv_ctx_params *ctx_params = (struct wd_drv_ctx_params *)params; + struct uacce_dev_list *dev_list, *used_list = NULL; + char alg_type[CRYPTO_MAX_ALG_NAME]; + struct uacce_dev_list *curr; + struct wd_ctx_h *ctx_h; + int target_numa; + handle_t hctx; + int ret; + + if (!params || !ctx) { + WD_ERR("invalid: parameters are NULL!\n"); + return -WD_EINVAL; + } + target_numa = ctx_params->numa_id; + + /* Get algorithm type and device list */ + ret = wd_get_alg_type(alg_name, alg_type); + if (ret) { + WD_ERR("invalid: alg_name is NULL!\n"); + return -WD_EINVAL; + } + + dev_list = wd_get_accel_list(alg_type); + if (!dev_list) { + WD_ERR("failed to get device list for alg %s\n", alg_name); + return -WD_ENODEV; + } + + /* Filter by bmp and sort by NUMA distance */ + used_list = wd_get_usable_list(dev_list, target_numa); + if (WD_IS_ERR(used_list) || !used_list) { + WD_INFO("Info: No usable device detected on numa<%d>\n", target_numa); + used_list = NULL; + ret = -WD_ENODEV; + goto out; + } + + curr = used_list; + while (curr) { + if (curr->dev) { + hctx = wd_request_ctx(curr->dev); + if (hctx) + goto success; + } + curr = curr->next; + } + + WD_ERR("failed to request ctx on NUMA node %d for %s\n", + target_numa, alg_name); + ret = -WD_EBUSY; + goto out; + +success: + ctx_h = (struct wd_ctx_h *)hctx; + ctx_h->priv = NULL; + ctx_h->ctx_type = UADK_ALG_HW; + *ctx = hctx; + ret = 0; +out: + if (dev_list) + wd_free_list_accels(dev_list); + if (used_list && !WD_IS_ERR(used_list)) + wd_free_list_accels(used_list); + + return ret; +} + +/** + * wd_hw_free_ctx() - HW driver's free_ctx callback. + * + * Releases ONE hardware context back to UACCE device. + * + * @ctx: The context handle to release + */ +void wd_hw_free_ctx(handle_t ctx) +{ + struct wd_ctx_h *ctx_h = (struct wd_ctx_h *)ctx; + + if (!ctx_h) { + WD_ERR("invalid: ctx is NULL!\n"); + return; + } + + /* Release hardware context back to device */ + wd_release_ctx(ctx); +} + +int wd_get_sqe_from_queue(struct wd_soft_ctx *sctx, __u32 tag_id) +{ + struct wd_soft_sqe *sqe = NULL; + + if (!sctx) { + WD_ERR("invalid: sctx is NULL!\n"); + return -WD_EINVAL; + } + + pthread_spin_lock(&sctx->slock); + sqe = &sctx->qfifo[sctx->head]; + if (!sqe->used && !sqe->complete) { // find the next not used sqe + sctx->head++; + if (unlikely(sctx->head == MAX_SOFT_QUEUE_LENGTH)) + sctx->head = 0; + + sqe->used = 1; + sqe->complete = 1; + sqe->id = tag_id; + sqe->result = 0; + __atomic_fetch_add(&sctx->run_num, 0x1, __ATOMIC_ACQUIRE); + pthread_spin_unlock(&sctx->slock); + } else { + pthread_spin_unlock(&sctx->slock); + return -WD_EBUSY; + } + + return WD_SUCCESS; +} + +int wd_put_sqe_to_queue(struct wd_soft_ctx *sctx, __u32 *tag_id, __u8 *result) +{ + struct wd_soft_sqe *sqe = NULL; + + /* The queue is not used */ + if (!sctx || !tag_id || !result || sctx->run_num < 1) + return -WD_EAGAIN; + + if (pthread_spin_trylock(&sctx->rlock)) + return -WD_EAGAIN; + sqe = &sctx->qfifo[sctx->tail]; + if (sqe->used && sqe->complete) { // find a used sqe + sctx->tail++; + if (unlikely(sctx->tail == MAX_SOFT_QUEUE_LENGTH)) + sctx->tail = 0; + + *tag_id = sqe->id; + *result = sqe->result; + sqe->used = 0x0; + sqe->complete = 0x0; + __atomic_fetch_sub(&sctx->run_num, 0x1, __ATOMIC_ACQUIRE); + pthread_spin_unlock(&sctx->rlock); + } else { + pthread_spin_unlock(&sctx->rlock); + return -WD_EAGAIN; + } + + return WD_SUCCESS; +} + +int wd_queue_is_busy(struct wd_soft_ctx *sctx) +{ + /* The queue is not used */ + if (!sctx) + return -WD_EINVAL; + + if (__atomic_load_n(&sctx->run_num, __ATOMIC_ACQUIRE) >= MAX_SOFT_QUEUE_LENGTH - 1) + return -WD_EBUSY; + + return WD_SUCCESS; +} diff --git a/drv/wd_drv.h b/drv/wd_drv.h new file mode 100644 index 0000000..17c7cc1 --- /dev/null +++ b/drv/wd_drv.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* Copyright 2020-2026 Huawei Technologies Co.,Ltd. All rights reserved. */ +#ifndef __WD_DRV_H +#define __WD_DRV_H + +#include <numa.h> +#include <stdlib.h> +#include <stdio.h> +#include <sys/ipc.h> +#include <sys/shm.h> +#include <pthread.h> + +#include "wd.h" +#include "wd_alg.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define MAX_SOFT_QUEUE_LENGTH 1024U + +/** + * default queue length set to 1024 + */ +struct wd_soft_sqe { + __u8 used; + __u8 result; + __u8 complete; + __u8 rsv; + __u32 id; +}; + +struct wd_soft_ctx { + __u8 ctx_type; + __u8 rsv[3]; + pthread_spinlock_t slock; + pthread_spinlock_t rlock; + __u32 head; + __u32 tail; + __u32 run_num; + int fd; + void *priv; + struct wd_soft_sqe qfifo[MAX_SOFT_QUEUE_LENGTH]; +}; + +/* Public function declarations */ +int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx); +void wd_hw_free_ctx(handle_t ctx); + +int wd_soft_alloc_ctx(char *alg_name, void *params, handle_t *ctx); +void wd_soft_free_ctx(handle_t ctx); + +int wd_queue_is_busy(struct wd_soft_ctx *sctx); +int wd_get_sqe_from_queue(struct wd_soft_ctx *sctx, __u32 tag_id); +int wd_put_sqe_to_queue(struct wd_soft_ctx *sctx, __u32 *tag_id, __u8 *result); +struct uacce_dev_list *wd_get_usable_list(struct uacce_dev_list *list, int target_numa); + +#ifdef __cplusplus +} +#endif + +#endif /* __WD_DRV_H */ diff --git a/wd_util.c b/wd_util.c index 707cdb5..1b4c74e 100644 --- a/wd_util.c +++ b/wd_util.c @@ -8,15 +8,15 @@ #include <dirent.h> #include <dlfcn.h> #include <pthread.h> +#include <stdlib.h> #include <string.h> #include <ctype.h> #include "wd_sched.h" #include "wd_util.h" #include "wd_alg.h" #include "wd_bmm.h" +#include "wd_internal.h" -#define WD_ASYNC_DEF_POLL_NUM 1 -#define WD_ASYNC_DEF_QUEUE_DEPTH 1024 #define WD_BALANCE_THRHD 1280 #define WD_RECV_MAX_CNT_SLEEP 60000000 #define WD_RECV_MAX_CNT_NOSLEEP 200000000 @@ -25,18 +25,14 @@ #define MAX(a, b) ((a) > (b) ? (a) : (b)) #define WD_INIT_SLEEP_UTIME 1000 -#define WD_INIT_RETRY_TIMES 10000 #define US2S(us) ((us) >> 20) #define WD_INIT_RETRY_TIMEOUT 3 -#define WD_SOFT_CTX_NUM 2 -#define WD_SOFT_SYNC_CTX 0 -#define WD_SOFT_ASYNC_CTX 1 - #define WD_DRV_LIB_DIR "uadk" #define WD_DRV_CONF_FILE "uadk.cnf" #define WD_PATH_DIR_NUM 2 +#define UADK_MAX_NUMA_NODES 64 struct msg_pool { /* message array allocated dynamically */ @@ -56,7 +52,7 @@ static const char *comp_ctx_type[2][2] = { }; /* define two ctx mode here for cipher and other alg */ -static const char *ctx_type[2][1] = { {"sync:"}, {"async:"} }; +static const char *ctx_mode_type[2][1] = { {"sync:"}, {"async:"} }; static const char *wd_env_name[WD_TYPE_MAX] = { "WD_COMP_CTX_NUM", @@ -78,7 +74,7 @@ struct drv_lib_list { static void *wd_internal_alloc(void *usr, size_t size) { - if (size != 0) + if (size) return malloc(size); else return NULL; @@ -86,7 +82,7 @@ static void *wd_internal_alloc(void *usr, size_t size) static void wd_internal_free(void *usr, void *va) { - if (va != NULL) + if (va) free(va); } @@ -163,33 +159,8 @@ int wd_mem_ops_init(handle_t h_ctx, struct wd_mm_ops *mm_ops, int mem_type) return 0; } -static int wd_parse_dev_id(handle_t h_ctx) -{ - struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; - char *dev_path = ctx->dev_path; - char *last_str = NULL; - char *endptr; - int dev_id; - - if (!dev_path) - return -WD_EINVAL; - - /* Find the last '-' in the string. */ - last_str = strrchr(dev_path, '-'); - if (!last_str || *(last_str + 1) == '\0') - return -WD_EINVAL; - - /* Parse the following number */ - dev_id = strtol(last_str + 1, &endptr, DECIMAL_NUMBER); - /* Check whether it is truly all digits */ - if (*endptr != '\0' || dev_id < 0) - return -WD_EINVAL; - - return dev_id; -} - static void clone_ctx_to_internal(struct wd_ctx *ctx, - struct wd_ctx_internal *ctx_in) + struct wd_ctx_internal *ctx_in) { ctx_in->ctx = ctx->ctx; ctx_in->op_type = ctx->op_type; @@ -242,7 +213,7 @@ int wd_init_ctx_config(struct wd_ctx_config_internal *in, struct wd_ctx_config *cfg) { struct wd_ctx_internal *ctxs; - char *alg_name; + const char *alg_name; __u32 i, j; int ret; @@ -264,9 +235,8 @@ int wd_init_ctx_config(struct wd_ctx_config_internal *in, for (i = 0; i < cfg->ctx_num; i++) { if (!cfg->ctxs[i].ctx) { - WD_ERR("invalid: ctx is NULL!\n"); - ret = -WD_EINVAL; - goto err_out; + WD_ERR("invalid: ctx<%u> is NULL!\n", i); + break; } clone_ctx_to_internal(cfg->ctxs + i, ctxs + i); ret = pthread_spin_init(&ctxs[i].lock, PTHREAD_PROCESS_SHARED); @@ -305,6 +275,12 @@ err_shm_del: return ret; } +static void wd_sched_set_param_default(handle_t h_sched_ctx, + void *sched_key, void *sched_param) +{ + +} + int wd_init_sched(struct wd_sched *in, struct wd_sched *from) { if (!from->name || !from->sched_init || @@ -319,8 +295,15 @@ int wd_init_sched(struct wd_sched *in, struct wd_sched *from) return -WD_ENOMEM; in->sched_init = from->sched_init; + in->sched_uninit = from->sched_uninit; in->pick_next_ctx = from->pick_next_ctx; in->poll_policy = from->poll_policy; + in->set_param = from->set_param; + + if (!from->set_param) { + WD_ERR("set param is NULL, use default!\n"); + in->set_param = wd_sched_set_param_default; + } return 0; } @@ -334,15 +317,17 @@ void wd_clear_sched(struct wd_sched *in) in->h_sched_ctx = 0; in->name = NULL; in->sched_init = NULL; + in->sched_uninit = NULL; in->pick_next_ctx = NULL; in->poll_policy = NULL; + in->set_param = NULL; } void wd_clear_ctx_config(struct wd_ctx_config_internal *in) { __u32 i; - for (i = 0; i < in->ctx_num; i++) + for (i = 0; in->ctxs && i < in->ctx_num; i++) pthread_spin_destroy(&in->ctxs[i].lock); in->priv = NULL; @@ -352,7 +337,6 @@ void wd_clear_ctx_config(struct wd_ctx_config_internal *in) in->ctxs = NULL; } - wd_remove_ctx_list(); wd_shm_delete(in); } @@ -470,7 +454,7 @@ void *wd_find_msg_in_pool(struct wd_async_msg_pool *pool, struct msg_pool *p; __u32 msg_num; - if ((__u32)ctx_idx > pool->pool_num) { + if ((__u32)ctx_idx >= pool->pool_num) { WD_ERR("invalid: message ctx id index is %d!\n", ctx_idx); return NULL; } @@ -478,7 +462,7 @@ void *wd_find_msg_in_pool(struct wd_async_msg_pool *pool, msg_num = p->msg_num; /* tag value start from 1 */ - if (tag == 0 || tag > msg_num) { + if (!tag || tag > msg_num) { WD_ERR("invalid: message cache tag is %u!\n", tag); return NULL; } @@ -707,7 +691,7 @@ static int wd_alloc_numa(struct wd_env_config *config, /* get numa num and device num of each numa from uacce_dev list */ config->numa_num = wd_get_dev_numa(head, numa_dev_num, max_node); - if (config->numa_num == 0 || config->numa_num > max_node) { + if (!config->numa_num || config->numa_num > max_node) { WD_ERR("invalid: numa number is %u!\n", config->numa_num); ret = -WD_ENODEV; goto free_list; @@ -743,7 +727,7 @@ static int is_number(const char *str) return 0; len = strlen(str); - if (len == 0) + if (!len) return 0; if (len != 1 && str[0] == '0') @@ -875,7 +859,7 @@ static int get_and_fill_ctx_num(struct wd_env_config_per_numa *config_numa, for (i = 0; i < CTX_MODE_MAX; i++) for (j = 0; j < config_numa->op_type_num; j++) { if (config_numa->op_type_num == 1) - type = ctx_type[i][j]; + type = ctx_mode_type[i][j]; else type = comp_ctx_type[i][j]; @@ -1265,6 +1249,7 @@ static int wd_sched_fill_table(struct wd_env_config_per_numa *config_numa, param.type = i; param.begin = ctx_table[mode][i].begin; param.end = ctx_table[mode][i].end; + param.ctx_prop = UADK_ALG_HW; ret = wd_sched_rr_instance(sched, ¶m); if (ret) return ret; @@ -1439,6 +1424,9 @@ int wd_check_ctx(struct wd_ctx_config_internal *config, __u8 mode, __u32 idx) { struct wd_ctx_internal *ctx; + if (unlikely(idx == QUEUE_FULL_POS)) + return -WD_EBUSY; + if (unlikely(idx >= config->ctx_num)) { WD_ERR("failed to pick a proper ctx: idx %u!\n", idx); return -WD_EINVAL; @@ -1476,8 +1464,8 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en) return 0; } -int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_handle, - handle_t ctx, void *msg, __u64 *balance, bool epoll_en) +int wd_handle_msg_sync(struct wd_msg_handle *msg_handle, handle_t ctx, + void *msg, __u64 *balance, bool epoll_en) { __u64 timeout = WD_RECV_MAX_CNT_NOSLEEP; __u64 rx_cnt = 0; @@ -1486,7 +1474,7 @@ int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_hand if (balance) timeout = WD_RECV_MAX_CNT_SLEEP; - ret = msg_handle->send(drv, ctx, msg); + ret = msg_handle->send(ctx, msg); if (unlikely(ret < 0)) { WD_ERR("failed to send msg to hw, ret = %d!\n", ret); return ret; @@ -1499,7 +1487,7 @@ int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_hand WD_ERR("wd ctx wait timeout(%d)!\n", ret); } - ret = msg_handle->recv(drv, ctx, msg); + ret = msg_handle->recv(ctx, msg); if (ret != -WD_EAGAIN) { if (unlikely(ret < 0)) { WD_ERR("failed to recv msg: error = %d!\n", ret); @@ -1527,18 +1515,50 @@ int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_hand int wd_init_param_check(struct wd_ctx_config *config, struct wd_sched *sched) { if (!config || !config->ctxs || !config->ctxs[0].ctx) { - WD_ERR("invalid: config or config->ctxs is NULL!\n"); + WD_ERR("invalid: wd_ctx_config is NULL!\n"); return -WD_EINVAL; } if (!sched) { - WD_ERR("invalid: sched is NULL!\n"); + WD_ERR("invalid: wd_sched is NULL!\n"); return -WD_EINVAL; } return 0; } +int wd_alg_try_init(enum wd_status *status) +{ + enum wd_status expected; + __u32 count = 0; + bool ret; + + /* + * Here is aimed to protect the security of the initialization interface + * in the multi-thread scenario. Only one thread can get the WD_INITING + * status to initialize algorithm. Other thread will wait for the result. + * And the algorithm initialization interfaces is a liner process. + * So the initing thread will return a result to notify other thread go on. + */ + do { + expected = WD_UNINIT; + ret = __atomic_compare_exchange_n(status, &expected, WD_INITING, true, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + if (expected == WD_INIT) { + WD_ERR("The algorithm has been initialized!\n"); + return -WD_EEXIST; + } + usleep(WD_INIT_SLEEP_UTIME); + + if (US2S(WD_INIT_SLEEP_UTIME * ++count) >= WD_INIT_RETRY_TIMEOUT) { + WD_ERR("The algorithm initialize wait timeout!\n"); + return -WD_ETIMEDOUT; + } + } while (!ret); + + return 0; +} + static int wd_alg_init_fallback(struct wd_alg_driver *fb_driver) { if (!fb_driver->init) { @@ -1561,47 +1581,103 @@ static void wd_alg_uninit_fallback(struct wd_alg_driver *fb_driver) fb_driver->exit(NULL); } -int wd_alg_init_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver) +static int wd_ctx_init_driver(struct wd_ctx_config_internal *config, + struct wd_alg_driver *driver) { + void *priv; int ret; - if (!driver->init) { - driver->fallback = 0; - WD_ERR("driver have no init interface.\n"); - ret = -WD_EINVAL; - goto err_alloc; + if (!driver || !driver->priv_size) + return -WD_EINVAL; + + if (driver->ops_size) { + driver->extend_ops = calloc(1, driver->ops_size); + if (!driver->extend_ops) + return -WD_ENOMEM; + } else { + driver->extend_ops = NULL; + } + + priv = calloc(1, driver->priv_size); + if (!priv) { + if (driver->extend_ops) + free(driver->extend_ops); + + return -WD_ENOMEM; } - ret = driver->init(driver, config); + ret = driver->init(config, priv); if (ret < 0) { - WD_ERR("driver init failed.\n"); - goto err_alloc; + if (driver->extend_ops) + free(driver->extend_ops); + + free(priv); + return ret; } + driver->drv_data = priv; if (driver->fallback) { ret = wd_alg_init_fallback((struct wd_alg_driver *)driver->fallback); - if (ret) { + if (ret) driver->fallback = 0; - WD_ERR("soft alg driver init failed.\n"); - } } return 0; +} -err_alloc: - return ret; +static void wd_ctx_uninit_driver(struct wd_alg_driver *driver) +{ + void *priv; + + if (!driver) + return; + priv = driver->drv_data; + if (!priv) + return; + driver->exit(priv); + free(priv); + driver->drv_data = NULL; + if (driver->extend_ops) { + free(driver->extend_ops); + driver->extend_ops = NULL; + } + + if (driver->fallback) + wd_alg_uninit_fallback((struct wd_alg_driver *)driver->fallback); } -void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver) +int wd_alg_init_driver(struct wd_ctx_config_internal *config) { - driver->exit(driver); + __u32 i, j; + int ret; + + /* Only initialize the drivers that have been filtered and selected for use. */ + for (i = 0; i < config->drv_count; i++) { + ret = wd_ctx_init_driver(config, config->drv_array[i]); + if (ret) + goto init_err; + } + + return 0; + +init_err: + for (j = 0; j < i; j++) + wd_ctx_uninit_driver(config->drv_array[j]); /* Ctx config just need clear once */ wd_clear_ctx_config(config); - if (driver->fallback) - wd_alg_uninit_fallback((struct wd_alg_driver *)driver->fallback); + return ret; +} + +void wd_alg_uninit_driver(struct wd_ctx_config_internal *config) +{ + __u32 i; + + for (i = 0; i < config->drv_count; i++) + wd_ctx_uninit_driver(config->drv_array[i]); + + /* Ctx config just need clear once */ + wd_clear_ctx_config(config); } void wd_dlclose_drv(void *dlh_list) @@ -1633,12 +1709,11 @@ static void add_lib_to_list(struct drv_lib_list *head, tmp->next = node; } -static int wd_set_ctx_nums(struct wd_ctx_params *ctx_params, struct uacce_dev_list *list, +static int wd_set_ctx_nums(struct wd_ctx_params *ctx_params, const char *section, __u32 op_type_num, int is_comp) { struct wd_ctx_nums *ctxs = ctx_params->ctx_set_num; int ret, ctx_num, node; - struct uacce_dev *dev; char *ctx_section; const char *type; __u32 i, j; @@ -1657,13 +1732,16 @@ static int wd_set_ctx_nums(struct wd_ctx_params *ctx_params, struct uacce_dev_li if (!ctx_num) return 0; - dev = wd_find_dev_by_numa(list, node); - if (WD_IS_ERR(dev)) - return -WD_ENODEV; + /* Validate node is within the system's NUMA node range */ + if (node < 0 || node > numa_max_node()) { + WD_ERR("invalid: numa node %d exceeds system max node %d!\n", + node, numa_max_node()); + return -WD_EINVAL; + } for (i = 0; i < CTX_MODE_MAX; i++) { for (j = 0; j < op_type_num; j++) { - type = is_comp ? comp_ctx_type[i][j] : ctx_type[i][0]; + type = is_comp ? comp_ctx_type[i][j] : ctx_mode_type[i][0]; if (strncmp(section, type, strlen(type))) continue; @@ -1685,14 +1763,12 @@ static int wd_set_ctx_nums(struct wd_ctx_params *ctx_params, struct uacce_dev_li static int wd_env_set_ctx_nums(const char *alg_name, const char *name, const char *var_s, struct wd_ctx_params *ctx_params, __u32 op_type_num) { - char alg_type[CRYPTO_MAX_ALG_NAME]; char *left, *section, *start; - struct uacce_dev_list *list; int is_comp; int ret; /* COMP environment variable's format is different, mark it */ - is_comp = strncmp(name, "WD_COMP_CTX_NUM", strlen(name)) ? 0 : 1; + is_comp = strncmp(name, "WD_COMP_CTX_NUM", sizeof("WD_COMP_CTX_NUM") - 1) ? 0 : 1; if (is_comp && op_type_num > ARRAY_SIZE(comp_ctx_type)) return -WD_EINVAL; @@ -1700,26 +1776,13 @@ static int wd_env_set_ctx_nums(const char *alg_name, const char *name, const cha if (!start) return -WD_ENOMEM; - ret = wd_get_alg_type(alg_name, alg_type); - if (ret) - goto free_start; - - list = wd_get_accel_list(alg_type); - if (!list) { - WD_ERR("failed to get devices!\n"); - ret = -WD_ENODEV; - goto free_start; - } - left = start; while ((section = strsep(&left, ","))) { - ret = wd_set_ctx_nums(ctx_params, list, section, op_type_num, is_comp); + ret = wd_set_ctx_nums(ctx_params, section, op_type_num, is_comp); if (ret < 0) break; } - wd_free_list_accels(list); -free_start: free(start); return ret; } @@ -1731,8 +1794,8 @@ void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params) int wd_ctx_param_init(struct wd_ctx_params *ctx_params, struct wd_ctx_params *user_ctx_params, - struct wd_alg_driver *driver, - enum wd_type type, int max_op_type) + char *alg, enum wd_type type, + int max_op_type) { const char *env_name = wd_env_name[type]; const char *var_s; @@ -1746,9 +1809,9 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, /* Only hw driver support environment variable */ var_s = secure_getenv(env_name); - if (var_s && strlen(var_s) && driver->calc_type == UADK_ALG_HW) { + if (var_s && strlen(var_s)) { /* environment variable has the highest priority */ - ret = wd_env_set_ctx_nums(driver->alg_name, env_name, var_s, + ret = wd_env_set_ctx_nums(alg, env_name, var_s, ctx_params, max_op_type); if (ret) { WD_ERR("fail to init ctx nums from %s!\n", env_name); @@ -1758,38 +1821,26 @@ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, } else { /* environment variable is not set, try to use user_ctx_params first */ if (user_ctx_params) { - if (user_ctx_params->bmp) { - copy_bitmask_to_bitmask(user_ctx_params->bmp, ctx_params->bmp); - } else { - /* default value */ - numa_bitmask_setall(ctx_params->bmp); - } - ctx_params->cap = user_ctx_params->cap; - ctx_params->ctx_set_num = user_ctx_params->ctx_set_num; - ctx_params->op_type_num = user_ctx_params->op_type_num; - if (ctx_params->op_type_num > (__u32)max_op_type) { + copy_bitmask_to_bitmask(user_ctx_params->bmp, ctx_params->bmp); + if (user_ctx_params->op_type_num > (__u32)max_op_type) { WD_ERR("fail to check user op type numbers.\n"); numa_free_nodemask(ctx_params->bmp); return -WD_EINVAL; } + ctx_params->cap = user_ctx_params->cap; + ctx_params->ctx_set_num = user_ctx_params->ctx_set_num; + ctx_params->op_type_num = user_ctx_params->op_type_num; return 0; } - /* user_ctx_params is also not set, use driver's defalut queue_num */ numa_bitmask_setall(ctx_params->bmp); - for (i = 0; i < driver->op_type_num; i++) { - ctx_params->ctx_set_num[i].sync_ctx_num = driver->queue_num; - ctx_params->ctx_set_num[i].async_ctx_num = driver->queue_num; + for (i = 0; i < max_op_type; i++) { + ctx_params->ctx_set_num[i].sync_ctx_num = 1; + ctx_params->ctx_set_num[i].async_ctx_num = 1; } } - - ctx_params->op_type_num = driver->op_type_num; - if (ctx_params->op_type_num > (__u32)max_op_type) { - WD_ERR("fail to check driver op type numbers.\n"); - numa_free_nodemask(ctx_params->bmp); - return -WD_EAGAIN; - } + ctx_params->op_type_num = max_op_type; return 0; } @@ -1949,7 +2000,7 @@ static void create_lib_to_list(const char *lib_path, struct drv_lib_list **head) node->dlhandle = dlopen(lib_path, RTLD_NODELETE | RTLD_NOW); if (!node->dlhandle) { - WD_ERR("failed to open lib file: %s, skipped\n", lib_path); + WD_ERR("failed to open lib file: %s, err: %s\n", lib_path, dlerror()); free(node); return; } @@ -2117,604 +2168,1030 @@ free_path: return (void *)head; } -struct wd_alg_driver *wd_alg_drv_bind(int task_type, const char *alg_name) +/** + * wd_ctx_unbind_drivers() - Unbind drivers from internal contexts. + * + * Decrements driver refcounts and clears all drv pointers. + * + * @config: Internal ctx config + */ +void wd_ctx_unbind_drivers(struct wd_ctx_config_internal *config) +{ + __u32 i; + + if (!config || !config->drv_array) + return; + + wd_alg_drv_ref_dec(config->drv_array, config->drv_count); + + for (i = 0; i < config->ctx_num; i++) + config->ctxs[i].drv = NULL; +} + +/** + * wd_ctx_bind_drivers() - Bind drivers to internal contexts via round-robin. + * + * This is the single write point for ctxs[i].drv in the entire lifecycle. + * Uses RR rule: ctxs[i].drv = drv_array[i % drv_count] + * + * Also: + * - Sets up soft fallback for HW drivers (once per unique HW driver) + * - Caches drv_array in config for session queries + * - Increments driver refcounts (deduplicated: each unique driver +1) + * + * @config: Internal ctx config (ctxs[] already copied by wd_init_ctx_config) + * @drv_array: Discovered unique drivers + * @drv_count: Number of unique drivers + * Return: 0 on success, negative on failure + */ +int wd_ctx_bind_drivers(struct wd_ctx_config_internal *config_api, + struct wd_ctx_config_internal *config_in, int init_type) { - struct wd_alg_driver *set_driver = NULL; struct wd_alg_driver *drv; + __u32 i; + + if (!config_api || init_type > WD_TYPE_V2) { + WD_ERR("invalid: parameters are NULL!\n"); + return -WD_EINVAL; + } - /* Get alg driver and dev name */ - switch (task_type) { - case TASK_INSTR: - drv = wd_request_drv(alg_name, true); - if (!drv) { - WD_ERR("no soft %s driver support\n", alg_name); - return NULL; + if (init_type == WD_TYPE_V1) { + if (!config_api->drv_array || config_api->drv_count != 1) { + WD_ERR("invalid: config driver number is error!\n"); + return -WD_EINVAL; } - set_driver = drv; - set_driver->fallback = 0; - break; - case TASK_HW: - case TASK_MIX: - drv = wd_request_drv(alg_name, false); - if (!drv) { - WD_ERR("no HW %s driver support\n", alg_name); - return NULL; + for (i = 0; i < config_api->ctx_num; i++) { + config_api->ctxs[i].drv = config_api->drv_array[0]; + config_api->ctxs[i].ctx_type = config_api->drv_array[0]->calc_type; } - set_driver = drv; - set_driver->fallback = 0; - if (task_type == TASK_MIX) { - drv = wd_request_drv(alg_name, true); - if (!drv) { - set_driver->fallback = 0; - WD_ERR("no soft %s driver support\n", alg_name); - } else { - set_driver->fallback = (handle_t)drv; - WD_ERR("successful to get soft driver\n"); - } + + drv = config_api->drv_array[0]; + if (!drv->fallback) { + drv->fallback = (handle_t)wd_request_drv( + config_api->alg_name, ALG_DRV_FB); } - break; - default: - WD_ERR("task type error.\n"); - return NULL; + return WD_SUCCESS; } - return set_driver; + if (!config_in || !config_in->drv_array || !config_in->drv_count) { + WD_ERR("invalid: V2 parameters, config_in=%p, drv_array=%p, ctx_num=%u!\n", + config_in, config_in ? config_in->drv_array : NULL, + config_in ? config_in->ctx_num : 0); + return -WD_EINVAL; + } + + WD_DEBUG("discovered %u drivers for ctx binding\n", config_in->drv_count); + for (i = 0; i < config_in->ctx_num; i++) { + if (!config_in->ctxs[i].drv) { + WD_ERR("failed to check ctx<%u> driver bound in internal config!\n", i); + continue; + } + /* + * The internally allocated queues have already been bound to the drivers, + * so only direct assignment processing is required here. + */ + config_api->ctxs[i].drv = config_in->ctxs[i].drv; + config_api->ctxs[i].ctx_type = config_in->ctxs[i].ctx_type; + } + + /* HW driver needs soft fallback — set once per unique driver */ + for (i = 0; i < config_in->drv_count; i++) { + drv = config_in->drv_array[i]; + if (!drv) + continue; + + if (drv->calc_type == UADK_ALG_HW && !drv->fallback) { + drv->fallback = (handle_t)wd_request_drv( + config_api->alg_name, ALG_DRV_FB); + WD_DEBUG("Set fallback for HW driver %s\n", drv->drv_name); + } + } + + /* Cache driver array for session queries */ + config_api->drv_array = config_in->drv_array; + config_api->drv_count = config_in->drv_count; + + /* Deduplicated refcount increment */ + wd_alg_drv_ref_inc(config_in->drv_array, config_in->drv_count); + + return WD_SUCCESS; } -void wd_alg_drv_unbind(struct wd_alg_driver *drv) +/** + * wd_alg_config_uninit() - Free driver discovery result. + * + * Releases the drv_array allocated by wd_alg_drv_discover(). + * Does NOT touch the drivers themselves (refcount managed separately). + * + * @attrs: Initialization attributes + */ +void wd_alg_config_uninit(struct wd_init_attrs *attrs) { - struct wd_alg_driver *fb_drv = NULL; + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; - if (!drv) + if (!internal_config || !internal_config->drv_array) return; - fb_drv = (struct wd_alg_driver *)drv->fallback; - if (fb_drv) - wd_release_drv(fb_drv); - wd_release_drv(drv); + /* Release wd_get_drv_array alloc memory */ + wd_put_drv_array(internal_config->drv_array, internal_config->drv_count); + internal_config->drv_array = NULL; + internal_config->drv_count = 0; + + /* Release ctx_config_internal */ + if (internal_config->ctxs) + free(internal_config->ctxs); + free(internal_config); + attrs->ctx_config_internal = NULL; } -int wd_alg_try_init(enum wd_status *status) +static __u32 wd_ctx_num_sum(struct wd_ctx_params *ctx_params, + struct wd_alg_driver **drv_array, + __u32 drv_count) { - enum wd_status expected; - __u32 count = 0; - bool ret; + __u32 total_ctx_num = 0; + __u32 async_num = 0; + __u32 sync_num = 0; + __u32 numa_count = 0; + __u32 per_driver_ctx; + int max_node, n; + __u32 i; - do { - expected = WD_UNINIT; - ret = __atomic_compare_exchange_n(status, &expected, WD_INITING, true, - __ATOMIC_RELAXED, __ATOMIC_RELAXED); - if (expected == WD_INIT) { - WD_ERR("The algorithm has been initialized!\n"); - return -WD_EEXIST; + for (i = 0; i < ctx_params->op_type_num; i++) { + sync_num += ctx_params->ctx_set_num[i].sync_ctx_num; + async_num += ctx_params->ctx_set_num[i].async_ctx_num; + } + per_driver_ctx = sync_num + async_num; + + max_node = numa_max_node() + 1; + if (max_node <= 0 || max_node > NUMA_NUM_NODES) + max_node = NUMA_NUM_NODES; + + for (n = 0; n < max_node; n++) { + if (numa_bitmask_isbitset(ctx_params->bmp, n)) + numa_count++; + } + if (!numa_count) + numa_count = 1; + + for (i = 0; i < drv_count; i++) { + if (drv_array[i]->calc_type == UADK_ALG_HW) + total_ctx_num += per_driver_ctx * numa_count; + else + total_ctx_num += per_driver_ctx; + } + + WD_DEBUG("total ctxs: %u (per_driver=%u, hw_numa=%u, drv_count=%u)\n", + total_ctx_num, per_driver_ctx, numa_count, drv_count); + + return total_ctx_num; +} + +/** + * wd_alg_config_init() - Discover matching drivers. + * + * Normalizes attrs->alg to alg_type ("cipher", "digest", etc.), + * then calls wd_get_drv_array() to find all unique drivers. + * Results stored in attrs->drv_array and attrs->drv_count. + * Filter drivers by sched_policy: NONE→CE, SINGLE→SVE. + * In-place removes non-matching entries, keeps at most 1 (highest priority). + * Returns filtered count via drv_count, or negative on no match. + * + * Pure query — no resource allocation, no refcount changes. + * + * @attrs: Initialization attributes (input: alg, task_type; output: drv_array, drv_count) + * Return: 0 on success, negative on failure + */ +static int wd_filter_drv_by_sched(__u32 sched_type, + struct wd_alg_driver **drv_array, + __u32 *drv_count) +{ + int primary_type, fallback_type = -1; + __u32 drv_array_cnt = *drv_count; + __u32 i, kept = 0; + + if (sched_type == SCHED_POLICY_NONE) { + primary_type = UADK_ALG_CE_INSTR; + } else if (sched_type == SCHED_POLICY_SINGLE) { + /* SINGLE: prefer SVE, fall back to CE when SVE unavailable */ + primary_type = UADK_ALG_SVE_INSTR; + fallback_type = UADK_ALG_CE_INSTR; + } else { + return 0; + } + + for (i = 0; i < drv_array_cnt; i++) { + if (drv_array[i]->calc_type == primary_type) { + if (kept != i) + drv_array[kept] = drv_array[i]; + kept++; } - usleep(WD_INIT_SLEEP_UTIME); + } - if (US2S(WD_INIT_SLEEP_UTIME * ++count) >= WD_INIT_RETRY_TIMEOUT) { - WD_ERR("The algorithm initialize wait timeout!\n"); - return -WD_ETIMEDOUT; + /* Fallback to secondary type only if primary yielded nothing */ + if (!kept && fallback_type >= 0) { + for (i = 0; i < drv_array_cnt; i++) { + if (drv_array[i]->calc_type == fallback_type) { + if (kept != i) + drv_array[kept] = drv_array[i]; + kept++; + } } - } while (!ret); + } + + *drv_count = kept; + + if (!kept) { + WD_ERR("invalid: no %s driver found for %s scheduler\n", + sched_type == SCHED_POLICY_NONE ? "CE" : "SVE/CE", + sched_type == SCHED_POLICY_NONE ? "NONE" : "SINGLE"); + return -WD_EINVAL; + } + /* Keep only the first (highest priority) */ + if (kept > 1) + *drv_count = 1; return 0; } -static __u32 wd_get_ctx_numbers(struct wd_ctx_params ctx_params, int end) +int wd_alg_config_init(struct wd_init_attrs *attrs) { - __u32 count = 0; - int i; + struct wd_ctx_config_internal *internal_config = NULL; + struct wd_alg_driver **temp_drv_array = NULL; + char alg_type[CRYPTO_MAX_ALG_NAME] = {0}; + __u32 tmp_drv_count; + __u32 tmp_ctx_num; + int ret; - for (i = 0; i < end; i++) { - count += ctx_params.ctx_set_num[i].sync_ctx_num; - count += ctx_params.ctx_set_num[i].async_ctx_num; + if (!attrs || !attrs->alg[0] || !attrs->ctx_params) + return -WD_EINVAL; + + /* Normalize alg to alg_type (e.g. "cipher", "digest") */ + ret = wd_get_alg_type(attrs->alg, alg_type); + if (ret || !alg_type[0]) { + WD_ERR("failed to get alg type for %s!\n", attrs->alg); + return -WD_EINVAL; } - return count; + /* Driver discovery */ + ret = wd_get_drv_array(alg_type, attrs->task_type, NULL, + &temp_drv_array, &tmp_drv_count); + if (ret || !tmp_drv_count) { + WD_ERR("failed to get %s's driver array!\n", attrs->alg); + goto driver_error; + } + + /* Filter drivers by sched_policy semantic constraints */ + ret = wd_filter_drv_by_sched(attrs->sched_type, temp_drv_array, &tmp_drv_count); + if (ret) + goto driver_error; + + /* Calculate total sync/async context counts */ + tmp_ctx_num = wd_ctx_num_sum(attrs->ctx_params, temp_drv_array, tmp_drv_count); + if (!tmp_ctx_num) { + WD_ERR("invalid: total_ctx_num is zero!\n"); + ret = -WD_EINVAL; + goto driver_error; + } + + /* Allocate internal ctx_config structure */ + internal_config = calloc(1, sizeof(*internal_config)); + if (!internal_config) { + ret = -WD_ENOMEM; + WD_ERR("failed to allocate ctx_config_internal!\n"); + goto driver_error; + } + + /* Allocate internal ctx array */ + internal_config->ctxs = calloc(tmp_ctx_num, sizeof(struct wd_ctx_internal)); + if (!internal_config->ctxs) { + WD_ERR("failed to allocate internal ctxs array!\n"); + ret = -WD_ENOMEM; + goto clean_config; + } + + /* Initialize configuration */ + internal_config->ctx_num = tmp_ctx_num; + internal_config->drv_array = temp_drv_array; + internal_config->drv_count = tmp_drv_count; + attrs->ctx_config_internal = internal_config; + + WD_DEBUG("Algorithm initialization started: alg=%s, task_type=%u\n", + attrs->alg, attrs->task_type); + + return WD_SUCCESS; + +clean_config: + free(internal_config); +driver_error: + wd_put_drv_array(temp_drv_array, tmp_drv_count); + attrs->ctx_config_internal = NULL; + return ret; } -static struct uacce_dev_list *wd_get_usable_list(struct uacce_dev_list *list, struct bitmask *bmp) +static int wd_parse_dev_id(handle_t h_ctx) { - struct uacce_dev_list *p, *node, *result = NULL; - struct uacce_dev *dev; - int numa_id, ret; + struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; + char *dev_path = ctx->dev_path; + char *last_str = NULL; + char *endptr; + int dev_id; - if (!bmp) { - WD_ERR("invalid: bmp is NULL!\n"); - return WD_ERR_PTR(-WD_EINVAL); - } + if (!dev_path) + return -WD_EINVAL; - p = list; - while (p) { - dev = p->dev; - numa_id = dev->numa_id; - ret = numa_bitmask_isbitset(bmp, numa_id); - if (!ret) { - p = p->next; - continue; - } + last_str = strrchr(dev_path, '-'); + if (!last_str || *(last_str + 1) == '\0') + return -WD_EINVAL; - node = calloc(1, sizeof(*node)); - if (!node) { - result = WD_ERR_PTR(-WD_ENOMEM); - goto out_free_list; - } + dev_id = strtol(last_str + 1, &endptr, DECIMAL_NUMBER); + if (*endptr != '\0' || dev_id < 0) + return -WD_EINVAL; - node->dev = wd_clone_dev(dev); - if (!node->dev) { - result = WD_ERR_PTR(-WD_ENOMEM); - goto out_free_node; - } + return dev_id; +} - if (!result) - result = node; - else - wd_add_dev_to_list(result, node); +static int wd_sched_ctx_region_key(handle_t ctx, __u8 ctx_type, + bool is_dev_policy, int *numa_id) +{ + struct wd_ctx_h *hctx; + int dev_id; + + if (is_dev_policy) { + dev_id = wd_parse_dev_id(ctx); + if (dev_id < 0) + return dev_id; - p = p->next; + *numa_id = dev_id; + return dev_id; } - return result ? result : WD_ERR_PTR(-WD_ENODEV); + *numa_id = 0; + if (ctx_type != UADK_ALG_HW) + return 0; + + hctx = (struct wd_ctx_h *)ctx; + if (hctx->dev) + *numa_id = hctx->dev->numa_id; -out_free_node: - free(node); -out_free_list: - wd_free_list_accels(result); - return result; + return 0; } -static int wd_init_ctx_set(struct wd_init_attrs *attrs, struct uacce_dev_list *list, - __u32 idx, int numa_id, __u32 op_type) +static int wd_alg_sched_instance(struct wd_sched *sched, + struct wd_ctx_config_internal *internal_config) { - struct wd_ctx_nums ctx_nums = attrs->ctx_params->ctx_set_num[op_type]; - __u32 ctx_set_num = ctx_nums.sync_ctx_num + ctx_nums.async_ctx_num; - struct wd_ctx_config *ctx_config = attrs->ctx_config; - __u32 count = idx + ctx_set_num; - struct uacce_dev *dev; - __u32 i, cnt = 0; - - /* If the ctx set number is 0, the initialization is skipped. */ - if (!ctx_set_num) - return -WD_ENOPROC; - - dev = wd_find_dev_by_numa(list, numa_id); - if (WD_IS_ERR(dev)) - return WD_PTR_ERR(dev); - - for (i = idx; i < count; i++) { - ctx_config->ctxs[i].ctx = wd_request_ctx(dev); - if (errno == WD_EBUSY) { - dev = wd_find_dev_by_numa(list, numa_id); - if (WD_IS_ERR(dev)) - return WD_PTR_ERR(dev); - - if (cnt++ > WD_INIT_RETRY_TIMES) { - WD_ERR("failed to request enough ctx due to timeout!\n"); - return -WD_ETIMEDOUT; + struct wd_ctx_internal *cur, *nxt = NULL; + struct sched_params sparams; + int cur_rgn_key, cur_numa_id; + int nxt_rgn_key, nxt_numa_id; + __u32 seg_begin, seg_end; + __u8 mode, inctx_type; + __u32 op_type, i = 0; + bool is_dev; + int ret; + + if (!sched || !internal_config) { + WD_ERR("invalid: sched, ctx_config, or ctx_params is NULL!\n"); + return -WD_EINVAL; + } + + if (!sched || !internal_config || !internal_config->ctxs) { + WD_ERR("invalid: internal_config->ctxs is NULL!\n"); + return -WD_EINVAL; + } + + is_dev = (sched->sched_policy == SCHED_POLICY_DEV); + for (i = 0; i < internal_config->ctx_num;) { + cur = &internal_config->ctxs[i]; + mode = internal_config->ctxs[i].ctx_mode; + op_type = internal_config->ctxs[i].op_type; + inctx_type = internal_config->ctxs[i].ctx_type; + + cur_rgn_key = wd_sched_ctx_region_key(cur->ctx, inctx_type, is_dev, &cur_numa_id); + if (cur_rgn_key < 0) { + WD_ERR("failed to parse region key for ctx %u!\n", i); + return -WD_EINVAL; + } + + /* Scan forward for contiguous ctxs with identical segment key */ + seg_begin = i; + for (seg_end = seg_begin; seg_end + 1 < internal_config->ctx_num; seg_end++) { + nxt = &internal_config->ctxs[seg_end + 1]; + if (nxt->ctx_mode != mode || nxt->op_type != op_type || + nxt->ctx_type != inctx_type) + break; + + nxt_rgn_key = wd_sched_ctx_region_key(nxt->ctx, nxt->ctx_type, + is_dev, &nxt_numa_id); + if (nxt_rgn_key < 0) { + WD_ERR("failed to parse region key for ctx %u!\n", seg_end + 1); + return -WD_EINVAL; } + if (nxt_rgn_key != cur_rgn_key || nxt_numa_id != cur_numa_id) + break; + } - /* self-decrease i to eliminate self-increase on next loop */ - i--; - continue; - } else if (!ctx_config->ctxs[i].ctx) { - /* - * wd_release_ctx_set will release ctx in - * caller wd_init_ctx_and_sched. - */ - return -WD_ENOMEM; + /* Register segment to scheduler */ + memset(&sparams, 0, sizeof(sparams)); + sparams.numa_id = cur_numa_id; + sparams.dev_id = cur_rgn_key; + sparams.type = op_type; + sparams.mode = mode; + sparams.begin = seg_begin; + sparams.end = seg_end; + sparams.ctx_prop = inctx_type; + + ret = wd_sched_rr_instance(sched, &sparams); + if (ret) { + WD_ERR("failed to register ctx[%u, %u] (op_type=%u, mode=%u, prop=%d)!\n", + seg_begin, seg_end, op_type, mode, inctx_type); + return ret; } - ctx_config->ctxs[i].op_type = op_type; - ctx_config->ctxs[i].ctx_mode = - ((i - idx) < ctx_nums.sync_ctx_num) ? - CTX_MODE_SYNC : CTX_MODE_ASYNC; + + i = seg_end + 1; } - return 0; + return WD_SUCCESS; } -static void wd_release_ctx_set(struct wd_ctx_config *ctx_config) +static void wd_free_ctxs_batch(struct wd_init_attrs *attrs, + __u32 allocated_count) { + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + struct wd_alg_driver *drv; __u32 i; - for (i = 0; i < ctx_config->ctx_num; i++) - if (ctx_config->ctxs[i].ctx) { - wd_release_ctx(ctx_config->ctxs[i].ctx); - ctx_config->ctxs[i].ctx = 0; - } + if (!internal_config || !internal_config->ctxs || !allocated_count) + return; + + for (i = 0; i < allocated_count; i++) { + if (!internal_config->ctxs[i].ctx) + continue; + + drv = internal_config->ctxs[i].drv; + if (drv && drv->free_ctx) + drv->free_ctx(internal_config->ctxs[i].ctx); + + internal_config->ctxs[i].ctx = 0; + } } -static int wd_instance_sched_set(struct wd_init_attrs *attrs, struct wd_ctx_nums ctx_nums, - int idx, int numa_id, int op_type) +static int wd_alloc_single_drv_ctxs(struct wd_init_attrs *attrs, + struct wd_alg_driver *drv, + __u8 ctx_mode, __u8 op_type, + __u32 *ctx_idx) { - struct wd_sched *sched = attrs->sched; - struct sched_params sparams; - int i, end, dev_id, ret = 0; - - dev_id = wd_parse_dev_id(attrs->ctx_config->ctxs[idx].ctx); - if (dev_id < 0) - return -WD_EINVAL; + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + struct wd_ctx_params *ctx_params = attrs->ctx_params; + struct wd_drv_ctx_params dparams; + int numa_nodes[UADK_MAX_NUMA_NODES]; + __u32 mode_ctx_num, numa_count = 0; + __u32 numa_idx, j; + int max_node, n; + handle_t ctx; + int ret; - for (i = 0; i < CTX_MODE_MAX; i++) { - sparams.numa_id = numa_id; - sparams.type = op_type; - sparams.dev_id = dev_id; - sparams.mode = i; - sparams.begin = idx + ctx_nums.sync_ctx_num * i; - end = idx - 1 + ctx_nums.sync_ctx_num + ctx_nums.async_ctx_num * i; - if (end < 0 || sparams.begin > (__u32)end) - continue; + if (ctx_mode == CTX_MODE_SYNC) + mode_ctx_num = ctx_params->ctx_set_num[op_type].sync_ctx_num; + else + mode_ctx_num = ctx_params->ctx_set_num[op_type].async_ctx_num; + if (!mode_ctx_num) + return WD_SUCCESS; + + if ((attrs->sched_type == SCHED_POLICY_NONE || + attrs->sched_type == SCHED_POLICY_SINGLE) && mode_ctx_num > 1) + mode_ctx_num = 1; + + if (drv->calc_type == UADK_ALG_HW) { + max_node = numa_max_node() + 1; + if (max_node <= 0 || max_node > UADK_MAX_NUMA_NODES) + max_node = UADK_MAX_NUMA_NODES; + for (n = 0; n < max_node; n++) { + if (numa_bitmask_isbitset(ctx_params->bmp, n)) + numa_nodes[numa_count++] = n; + } + } else { + numa_nodes[0] = 0; + numa_count = 1; + } + if (!numa_count) { + numa_nodes[0] = 0; + numa_count = 1; + } + + for (numa_idx = 0; numa_idx < numa_count; numa_idx++) { + for (j = 0; j < mode_ctx_num; j++) { + memset(&dparams, 0, sizeof(dparams)); + dparams.ctx_mode = ctx_mode; + dparams.op_type = op_type; + dparams.numa_id = numa_nodes[numa_idx]; + dparams.bmp = ctx_params->bmp; + dparams.epoll_en = false; + ret = drv->alloc_ctx(attrs->alg, &dparams, &ctx); + if (!ctx || ret < 0) { + if (ret == -WD_ENODEV) + break; + WD_ERR("failed to alloc ctx %u from driver %s on numa %d!\n", + *ctx_idx, drv->drv_name, numa_nodes[numa_idx]); + return ret; + } - sparams.end = end; - ret = wd_sched_rr_instance(sched, &sparams); - if (ret) - goto out; + internal_config->ctxs[*ctx_idx].ctx = ctx; + internal_config->ctxs[*ctx_idx].op_type = dparams.op_type; + internal_config->ctxs[*ctx_idx].ctx_mode = dparams.ctx_mode; + internal_config->ctxs[*ctx_idx].ctx_type = drv->calc_type; + internal_config->ctxs[*ctx_idx].drv = drv; + (*ctx_idx)++; + } } -out: - return ret; + return WD_SUCCESS; } -static int wd_init_ctx_and_sched(struct wd_init_attrs *attrs, struct bitmask *bmp, - struct uacce_dev_list *list) +static int wd_alloc_ctxs_batch(struct wd_init_attrs *attrs, + __u8 ctx_mode, __u32 *start_idx) { + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; struct wd_ctx_params *ctx_params = attrs->ctx_params; - __u32 op_type_num = ctx_params->op_type_num; - int i, ret, max_node = numa_max_node() + 1; - struct wd_ctx_nums ctx_nums; - __u32 j, idx = 0; + struct wd_alg_driver *drv; + __u8 op_type, op_type_num; + __u32 ctx_idx, drv_idx; + int ret; - for (i = 0; i < max_node; i++) { - if (!numa_bitmask_isbitset(bmp, i)) - continue; - for (j = 0; j < op_type_num; j++) { - ctx_nums = ctx_params->ctx_set_num[j]; - ret = wd_init_ctx_set(attrs, list, idx, i, j); - if (ret == -WD_ENOPROC) - continue; - else if (ret) - goto free_ctxs; - ret = wd_instance_sched_set(attrs, ctx_nums, idx, i, j); + op_type_num = ctx_params->op_type_num; + if (attrs->sched_type == SCHED_POLICY_NONE || + attrs->sched_type == SCHED_POLICY_SINGLE) + op_type_num = 1; + + ctx_idx = *start_idx; + for (drv_idx = 0; drv_idx < internal_config->drv_count; drv_idx++) { + drv = internal_config->drv_array[drv_idx]; + if (!drv || !drv->alloc_ctx) { + WD_ERR("failed to check driver %s alloc_ctx!\n", + drv ? drv->drv_name : "unknown"); + ret = -WD_EINVAL; + goto err_ctxs; + } + + for (op_type = 0; op_type < op_type_num; op_type++) { + ret = wd_alloc_single_drv_ctxs(attrs, drv, + ctx_mode, op_type, + &ctx_idx); if (ret) - goto free_ctxs; - idx += (ctx_nums.sync_ctx_num + ctx_nums.async_ctx_num); + goto err_ctxs; } } + *start_idx = ctx_idx; - return 0; - -free_ctxs: - wd_release_ctx_set(attrs->ctx_config); + return WD_SUCCESS; +err_ctxs: + wd_free_ctxs_batch(attrs, ctx_idx); return ret; } -static void wd_init_device_nodemask(struct uacce_dev_list *list, struct bitmask *bmp) +/** + * wd_alg_ctx_uninit() - Release ctxs, scheduler, ctx_config. + * + * Releases resources in reverse allocation order: + * 1. Release scheduler + * 2. Release ctxs via RR rule (drv->free_ctx) + * 3. Free ctx_config and ctxs array + * + * @attrs: Initialization attributes + */ +void wd_alg_ctx_uninit(struct wd_init_attrs *attrs) { - struct uacce_dev_list *p = list; + struct wd_ctx_config_internal *internal_config; - numa_bitmask_clearall(bmp); - while (p) { - numa_bitmask_setbit(bmp, p->dev->numa_id); - p = p->next; - } -} + if (!attrs) + return; -static int wd_alg_ctx_init(struct wd_init_attrs *attrs) -{ - struct wd_ctx_config *ctx_config = attrs->ctx_config; - struct wd_ctx_params *ctx_params = attrs->ctx_params; - struct bitmask *used_bmp = ctx_params->bmp; - struct uacce_dev_list *list, *used_list = NULL; - __u32 ctx_set_num, op_type_num; - int numa_cnt, ret; + internal_config = attrs->ctx_config_internal; - list = wd_get_accel_list(attrs->alg); - if (!list) { - WD_ERR("failed to get devices for alg: %s\n", attrs->alg); - return -WD_ENODEV; + WD_DEBUG("releasing ctxs, scheduler, and ctx_config\n"); + /* Release scheduler */ + if (attrs->sched) { + wd_sched_rr_release(attrs->sched); + attrs->sched = NULL; } - op_type_num = ctx_params->op_type_num; - ctx_set_num = wd_get_ctx_numbers(*ctx_params, op_type_num); - if (!ctx_set_num || !op_type_num) { - WD_ERR("invalid: ctx_set_num is %u, op_type_num is %u!\n", - ctx_set_num, op_type_num); - ret = -WD_EINVAL; - goto out_freelist; - } + /* Release ctxs via RR rule */ + if (internal_config) + wd_free_ctxs_batch(attrs, internal_config->ctx_num); - /* - * Not every numa has a device. Therefore, the first thing is to - * filter the devices in the selected numa node, and the second - * thing is to obtain the distribution of devices. - */ - used_list = wd_get_usable_list(list, used_bmp); - if (WD_IS_ERR(used_list)) { - ret = WD_PTR_ERR(used_list); - WD_ERR("failed to get usable devices(%d)!\n", ret); - goto out_freelist; + /* Release user-visible ctx_config */ + if (attrs->ctx_config) { + if (attrs->ctx_config->ctxs) { + free(attrs->ctx_config->ctxs); + attrs->ctx_config->ctxs = NULL; + } + free(attrs->ctx_config); + attrs->ctx_config = NULL; } - wd_init_device_nodemask(used_list, used_bmp); + WD_DEBUG("ctx uninit complete\n"); +} - numa_cnt = numa_bitmask_weight(used_bmp); - if (!numa_cnt) { - ret = numa_cnt; - WD_ERR("invalid: bmp is clear!\n"); - goto out_freeusedlist; +static int wd_init_ctx_config_sched(struct wd_init_attrs *attrs) +{ + struct wd_ctx_config_internal *internal_config = attrs->ctx_config_internal; + struct wd_ctx_params *ctx_params = attrs->ctx_params; + __u32 total_ctx_num = internal_config->ctx_num; + __u32 i; + int ret; + + /* Allocate user-visible wd_ctx_config structure */ + attrs->ctx_config = calloc(1, sizeof(struct wd_ctx_config)); + if (!attrs->ctx_config) { + WD_ERR("failed to allocate ctx_config!\n"); + return -WD_ENOMEM; } - ctx_config->ctx_num = ctx_set_num * numa_cnt; - ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); - if (!ctx_config->ctxs) { + /* Allocate user-visible wd_ctx array */ + attrs->ctx_config->ctxs = calloc(total_ctx_num, sizeof(struct wd_ctx)); + if (!attrs->ctx_config->ctxs) { + WD_ERR("failed to allocate ctxs array!\n"); ret = -WD_ENOMEM; - WD_ERR("failed to alloc ctxs!\n"); - goto out_freeusedlist; + goto cleanup_config; } - ret = wd_init_ctx_and_sched(attrs, used_bmp, used_list); - if (ret) - free(ctx_config->ctxs); + attrs->ctx_config->ctx_num = total_ctx_num; + /* Copy queue information from internal to user-visible config */ + for (i = 0; i < total_ctx_num; i++) { + attrs->ctx_config->ctxs[i].ctx = internal_config->ctxs[i].ctx; + attrs->ctx_config->ctxs[i].op_type = internal_config->ctxs[i].op_type; + attrs->ctx_config->ctxs[i].ctx_mode = internal_config->ctxs[i].ctx_mode; + } + + /* ── Call algorithm-specific init ── */ + attrs->ctx_config->cap = ctx_params->cap; + ret = attrs->alg_init(attrs->ctx_config, attrs->sched); + if (ret) { + WD_ERR("failed to initialize algorithm!\n"); + goto cleanup_ctxs; + } -out_freeusedlist: - wd_free_list_accels(used_list); -out_freelist: - wd_free_list_accels(list); + return WD_SUCCESS; +cleanup_ctxs: + free(attrs->ctx_config->ctxs); + attrs->ctx_config->ctxs = NULL; +cleanup_config: + free(attrs->ctx_config); + attrs->ctx_config = NULL; return ret; } -static int wd_alg_ce_ctx_init(struct wd_init_attrs *attrs) +/** + * wd_alg_ctx_init() - Allocate contexts, scheduler, and initialize algorithm. + * + * Uses drivers discovered by wd_alg_config_init(). + * Allocates contexts via RR: ctx[i] -> drv_array[i % drv_count]->alloc_ctx() + * Then allocates scheduler, registers context ranges, and calls alg_init + * which performs wd_init_ctx_config() (wd_ctx[] -> wd_ctx_internal[] copy). + * + * On return: + * - attrs->ctx_config: user-visible context array (populated) + * - attrs->sched: scheduler (allocated and populated) + * - attrs->ctx_config_internal: MUST be set by alg_init callback + * + * NOTE: ctxs[i].drv is still NULL after this function — set later by + * wd_ctx_bind_drivers(). + * + * @attrs: Initialization attributes (input: drv_array, ctx_params, alg_init, etc.) + * Return: 0 on success, negative on failure + */ +int wd_alg_ctx_init(struct wd_init_attrs *attrs) { - struct wd_ctx_config *ctx_config = attrs->ctx_config; - struct wd_ce_ctx *ctx; + struct wd_ctx_config_internal *internal_config; + int numa_num_int = numa_max_node(); + __u16 region_num, numa_num; + __u32 ctx_idx = 0; + int ret; - ctx_config->ctx_num = 1; - ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); - if (!ctx_config->ctxs) { - WD_ERR("failed to alloc ctxs!\n"); - return -WD_ENOMEM; - } + if (numa_num_int < 0) + numa_num = 0; + else + numa_num = (__u16)numa_num_int; - ctx = calloc(1, sizeof(struct wd_ce_ctx)); - if (!ctx) { - free(ctx_config->ctxs); - return -WD_ENOMEM; + if (!attrs || !attrs->ctx_params || !attrs->ctx_config_internal || + !attrs->ctx_config_internal->drv_array) { + WD_ERR("invalid: attrs, ctx_params, or drv_array is NULL/empty!\n"); + return -WD_EINVAL; } - ctx->fd = -1; - ctx_config->ctxs[0].ctx = (handle_t)ctx; - return WD_SUCCESS; -} + /* + * Ensure that contexts (ctx) with the same attributes are allocated first, + * thereby maintaining queue continuity within the contexts. + */ + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_SYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; -static void wd_alg_ce_ctx_uninit(struct wd_ctx_config *ctx_config) -{ - __u32 i; + /* wd_alloc_ctxs_batch already cleaned up via its internal err_ctxs. */ + ret = wd_alloc_ctxs_batch(attrs, CTX_MODE_ASYNC, &ctx_idx); + if (ret) + return -WD_EINVAL; - for (i = 0; i < ctx_config->ctx_num; i++) { - if (ctx_config->ctxs[i].ctx) { - free((struct wd_ce_ctx *)ctx_config->ctxs[i].ctx); - ctx_config->ctxs[i].ctx = 0; - } + /* Backfill actual allocated count */ + internal_config = attrs->ctx_config_internal; + internal_config->ctx_num = ctx_idx; + if (!ctx_idx) { + WD_ERR("no contexts allocated on any NUMA node!\n"); + ret = -WD_EINVAL; + goto cleanup_ctxs; } - free(ctx_config->ctxs); -} + /* ── Allocate scheduler ── */ + if (attrs->sched_type == SCHED_POLICY_DEV) + region_num = DEVICE_REGION_MAX; + else if (numa_num == 0) + region_num = 1; + else + region_num = numa_num + 1; + + attrs->sched = wd_sched_rr_alloc(attrs->sched_type, + attrs->ctx_params->op_type_num, + region_num, + attrs->alg_poll_ctx); + if (!attrs->sched) { + WD_ERR("failed to allocate scheduler!\n"); + ret = -WD_ENOMEM; + goto cleanup_ctxs; + } -static void wd_alg_ctx_uninit(struct wd_ctx_config *ctx_config) -{ - __u32 i; + /* ── Register contexts to scheduler ── */ + internal_config = attrs->ctx_config_internal; + ret = wd_alg_sched_instance(attrs->sched, internal_config); + if (ret) { + WD_ERR("failed to register contexts to scheduler!\n"); + goto cleanup_sched; + } - for (i = 0; i < ctx_config->ctx_num; i++) { - if (ctx_config->ctxs[i].ctx) { - wd_release_ctx(ctx_config->ctxs[i].ctx); - ctx_config->ctxs[i].ctx = 0; - } + /* ── Allocate user ctx_config and initialize algorithm ── */ + ret = wd_init_ctx_config_sched(attrs); + if (ret) { + WD_ERR("failed to allocate user ctx_config and initialize algorithm!\n"); + goto cleanup_user_config; } - free(ctx_config->ctxs); + WD_DEBUG("ctx init complete: %u ctxs from %u drivers\n", + internal_config->ctx_num, internal_config->drv_count); + + return WD_SUCCESS; + + /* ── Error cleanup (LIFO) ── */ +cleanup_user_config: + if (attrs->ctx_config) { + if (attrs->ctx_config->ctxs) + free(attrs->ctx_config->ctxs); + free(attrs->ctx_config); + attrs->ctx_config = NULL; + } +cleanup_sched: + wd_sched_rr_release(attrs->sched); + attrs->sched = NULL; +cleanup_ctxs: + /* Free ctxs allocated so far using RR rule */ + wd_free_ctxs_batch(attrs, ctx_idx); + return ret; } -static int wd_alg_init_sve_ctx(struct wd_ctx_config *ctx_config) +/** + * wd_alg_attrs_uninit() - Release all algorithm resources. + * + * Releases resources in reverse order of allocation: + * 1. wd_alg_ctx_uninit() — free contexts, scheduler, ctx_config + * 2. wd_alg_config_uninit() — free driver array and internal config + * + * @attrs: Initialization attributes + */ +void wd_alg_attrs_uninit(struct wd_init_attrs *attrs) { - struct wd_soft_ctx *ctx_sync, *ctx_async; + if (!attrs) + return; - ctx_config->ctx_num = WD_SOFT_CTX_NUM; - ctx_config->ctxs = calloc(ctx_config->ctx_num, sizeof(struct wd_ctx)); - if (!ctx_config->ctxs) - return -WD_ENOMEM; + WD_DEBUG("Algorithm cleanup started: alg=%s\n", attrs->alg); - ctx_sync = calloc(1, sizeof(struct wd_soft_ctx)); - if (!ctx_sync) - goto free_ctxs; + /* Release ctxs, scheduler, ctx_config */ + wd_alg_ctx_uninit(attrs); - ctx_sync->fd = -1; - ctx_config->ctxs[WD_SOFT_SYNC_CTX].op_type = 0; - ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx_mode = CTX_MODE_SYNC; - ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx = (handle_t)ctx_sync; + /* Free driver array and internal config */ + wd_alg_config_uninit(attrs); - ctx_async = calloc(1, sizeof(struct wd_soft_ctx)); - if (!ctx_async) - goto free_ctx_sync; + WD_DEBUG("Algorithm cleanup complete\n"); +} - ctx_async->fd = -1; - ctx_config->ctxs[WD_SOFT_ASYNC_CTX].op_type = 0; - ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx_mode = CTX_MODE_ASYNC; - ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx = (handle_t)ctx_async; +static bool wd_check_sva_mode(const char *alg_type) +{ + struct uacce_dev_list *dev_list; + bool is_sva = true; - return 0; + dev_list = wd_get_accel_list(alg_type); + if (!dev_list || !dev_list->dev) + return true; -free_ctx_sync: - free(ctx_sync); -free_ctxs: - free(ctx_config->ctxs); - return -WD_ENOMEM; -} + if (!(dev_list->dev->flags & UACCE_DEV_SVA)) + is_sva = false; -static void wd_alg_uninit_sve_ctx(struct wd_ctx_config *ctx_config) -{ - free((struct wd_soft_ctx *)ctx_config->ctxs[WD_SOFT_ASYNC_CTX].ctx); - free((struct wd_soft_ctx *)ctx_config->ctxs[WD_SOFT_SYNC_CTX].ctx); - free(ctx_config->ctxs); + wd_free_list_accels(dev_list); + return is_sva; } -int wd_alg_attrs_init(struct wd_init_attrs *attrs) +static int wd_check_nosva_type(struct wd_init_attrs *attrs) { - wd_alg_poll_ctx alg_poll_func = attrs->alg_poll_ctx; - wd_alg_init alg_init_func = attrs->alg_init; - __u32 sched_type = attrs->sched_type; - struct wd_ctx_config *ctx_config = NULL; - struct wd_sched *alg_sched = NULL; - char alg_type[CRYPTO_MAX_ALG_NAME]; - int driver_type = UADK_ALG_HW; - const char *alg = attrs->alg; - int ret = -WD_EINVAL; + char alg_type_buf[CRYPTO_MAX_ALG_NAME] = {0}; + bool is_sva; + int ret; - if (!attrs->ctx_params) + ret = wd_get_alg_type(attrs->alg, alg_type_buf); + if (ret) { + WD_ERR("failed to get alg type for No-SVA check!\n"); return -WD_EINVAL; + } - if (attrs->driver) - driver_type = attrs->driver->calc_type; + /* These two special types require the use of the original algorithm names. */ + if (strcmp(alg_type_buf, "comp") == 0 || + strcmp(alg_type_buf, "ecc") == 0) + (void)strcpy(alg_type_buf, attrs->alg); - switch (driver_type) { - case UADK_ALG_SOFT: - case UADK_ALG_CE_INSTR: - ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - return -WD_ENOMEM; - } - attrs->ctx_config = ctx_config; - - /* Use default sched_type to alloc scheduler */ - alg_sched = wd_sched_rr_alloc(SCHED_POLICY_NONE, 1, 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to alloc scheduler\n"); - goto out_ctx_config; + is_sva = wd_check_sva_mode(alg_type_buf); + if (is_sva) { + if (attrs->sched_type == SCHED_POLICY_DEV) { + WD_ERR("invalid: SVA mode does not support SCHED_POLICY_DEV!\n"); + return -WD_EINVAL; } + return WD_SUCCESS; + } - attrs->sched = alg_sched; + /* + * No-SVA with TASK_INSTR: CE/SVE drivers execute on CPU and + * do not perform DMA, no dependency on hardware SVA mode. + */ + if (attrs->task_type == TASK_INSTR) + return WD_SUCCESS; - ret = wd_alg_ce_ctx_init(attrs); - if (ret) { - WD_ERR("fail to init ce ctx\n"); - goto out_freesched; - } + /* No-SVA only allows TASK_HW */ + if (attrs->task_type != TASK_HW) { + WD_ERR("invalid: No-SVA mode only supports TASK_HW, got %u!\n", + attrs->task_type); + return -WD_EINVAL; + } - ret = alg_init_func(ctx_config, alg_sched); - if (ret) - goto out_pre_init; + /* TASK_HW + DEV: correct combination */ + if (attrs->sched_type == SCHED_POLICY_DEV) + return WD_SUCCESS; - break; - case UADK_ALG_SVE_INSTR: - /* Use default sched_type to alloc scheduler */ - alg_sched = wd_sched_rr_alloc(SCHED_POLICY_SINGLE, 1, 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to alloc scheduler\n"); - return -WD_EINVAL; - } - attrs->sched = alg_sched; + /* TASK_HW + RR/LOOP/HUNGRY: auto-switch to DEV */ + if (attrs->sched_type == SCHED_POLICY_RR || + attrs->sched_type == SCHED_POLICY_LOOP || + attrs->sched_type == SCHED_POLICY_HUNGRY) { + WD_INFO("info: No-SVA mode auto-switching sched %u to SCHED_POLICY_DEV!\n", + attrs->sched_type); + attrs->sched_type = SCHED_POLICY_DEV; + return WD_SUCCESS; + } - ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - goto out_freesched; - } - attrs->ctx_config = ctx_config; + /* TASK_HW + NONE/SINGLE/INSTR: reject */ + WD_ERR("invalid: No-SVA mode requires SCHED_POLICY_DEV, got %u!\n", + attrs->sched_type); + return -WD_EINVAL; +} - ret = wd_alg_init_sve_ctx(ctx_config); - if (ret) { - WD_ERR("fail to init sve ctx!\n"); - goto out_freesched; - } +/** + * wd_task_sched_check() - Validate task_type + sched_type combination. + * + * After driver discovery, check that the requested scheduling + * policy is compatible with the discovered drivers and task type. + * + * Invalid combinations: + * TASK_HW + SCHED_POLICY_INSTR - instr poll only polls ctx[0], + * losing HW async completions. + * TASK_INSTR + SCHED_POLICY_DEV - CE/SVE/SOFT drivers have no + * dev_id for device-level domains. + * SCHED_POLICY_NONE + >1 driver - NONE always picks ctx[0]; + * different sessions may route to + * a driver that doesn\'t support + * their algorithm. + * + * @attrs: Initialization attributes (drv_count must be populated) + * Return: 0 on success, -WD_EINVAL on invalid combination + */ +static int wd_task_sched_check(struct wd_init_attrs *attrs) +{ + struct wd_alg_driver **drv_arr; - ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); - if (ret) { - wd_alg_uninit_sve_ctx(ctx_config); - goto out_freesched; - } - break; - case UADK_ALG_HW: - if (wd_get_alg_type(alg, alg_type)) - return -WD_EINVAL; - (void)strcpy(attrs->alg, alg_type); + if (attrs->task_type == TASK_HW && + attrs->sched_type == SCHED_POLICY_INSTR) { + WD_ERR("invalid: HW tasks must not use INSTR scheduler\n"); + return -WD_EINVAL; + } - ctx_config = calloc(1, sizeof(*ctx_config)); - if (!ctx_config) { - WD_ERR("fail to alloc ctx config\n"); - return -WD_ENOMEM; - } - attrs->ctx_config = ctx_config; + if (attrs->task_type == TASK_MIX && + attrs->sched_type == SCHED_POLICY_INSTR) { + WD_ERR("invalid: MIX tasks must not use INSTR scheduler\n"); + return -WD_EINVAL; + } - if (sched_type == SCHED_POLICY_DEV) - alg_sched = wd_sched_rr_alloc(sched_type, attrs->ctx_params->op_type_num, - DEVICE_REGION_MAX, alg_poll_func); - else - alg_sched = wd_sched_rr_alloc(sched_type, attrs->ctx_params->op_type_num, - numa_max_node() + 1, alg_poll_func); - if (!alg_sched) { - WD_ERR("fail to instance scheduler\n"); - goto out_ctx_config; - } - attrs->sched = alg_sched; + if (attrs->sched_type == SCHED_POLICY_NONE && + attrs->ctx_config_internal->drv_count > 1) { + WD_ERR("invalid: NONE scheduler requires single driver\n"); + return -WD_EINVAL; + } - ret = wd_alg_ctx_init(attrs); - if (ret) { - WD_ERR("fail to init ctx\n"); - goto out_freesched; - } + /* NONE scheduler: only TASK_INSTR (CE driver) is valid */ + if (attrs->sched_type == SCHED_POLICY_NONE && + (attrs->task_type == TASK_HW || attrs->task_type == TASK_MIX)) { + WD_ERR("invalid: NONE scheduler only supports TASK_INSTR!\n"); + return -WD_EINVAL; + } - ctx_config->cap = attrs->ctx_params->cap; - ret = alg_init_func(ctx_config, alg_sched); - if (ret) - goto out_pre_init; - break; - default: - WD_ERR("driver type error: %d\n", driver_type); + /* SINGLE scheduler: only TASK_INSTR (SVE driver) is valid */ + if (attrs->sched_type == SCHED_POLICY_SINGLE && + (attrs->task_type == TASK_HW || attrs->task_type == TASK_MIX)) { + WD_ERR("invalid: SINGLE scheduler only supports TASK_INSTR!\n"); return -WD_EINVAL; } - return 0; + /* NONE with TASK_INSTR: must be CE driver */ + if (attrs->sched_type == SCHED_POLICY_NONE && + attrs->task_type == TASK_INSTR) { + drv_arr = attrs->ctx_config_internal->drv_array; + if (drv_arr && drv_arr[0] && + drv_arr[0]->calc_type != UADK_ALG_CE_INSTR) { + WD_ERR("invalid: NONE scheduler requires CE driver, got type %d!\n", + drv_arr[0]->calc_type); + return -WD_EINVAL; + } + } -out_pre_init: - if (driver_type == UADK_ALG_CE_INSTR || driver_type == UADK_ALG_SOFT) - wd_alg_ce_ctx_uninit(ctx_config); - else - wd_alg_ctx_uninit(ctx_config); -out_freesched: - wd_sched_rr_release(alg_sched); -out_ctx_config: - if (ctx_config) - free(ctx_config); - return ret; + /* SINGLE with TASK_INSTR: must be SVE driver */ + if (attrs->sched_type == SCHED_POLICY_SINGLE && + attrs->task_type == TASK_INSTR) { + drv_arr = attrs->ctx_config_internal->drv_array; + if (drv_arr && drv_arr[0] && + (drv_arr[0]->calc_type != UADK_ALG_SVE_INSTR && + drv_arr[0]->calc_type != UADK_ALG_CE_INSTR)) { + WD_ERR("invalid: SINGLE scheduler requires CE/SVE driver, got type %d!\n", + drv_arr[0]->calc_type); + return -WD_EINVAL; + } + } + + /* No-SVA mode: only TASK_HW + SCHED_POLICY_DEV is allowed */ + return wd_check_nosva_type(attrs); } -void wd_alg_attrs_uninit(struct wd_init_attrs *attrs) +/** + * wd_alg_attrs_init() - Initialize algorithm with auto-discovered drivers. + * + * Initialization sequence: + * 1. wd_alg_config_init() — discover matching drivers, allocate internal config + * 2. wd_task_sched_check() — validate task_type + sched_type compatibility + * 3. wd_alg_ctx_init() — allocate contexts, scheduler, and initialize algorithm + * + * After this, driver init is done by the caller via wd_alg_init_driver(). + * + * @attrs: Initialization attributes (input/output) + * Return: 0 on success, negative on failure + */ +int wd_alg_attrs_init(struct wd_init_attrs *attrs) { - struct wd_ctx_config *ctx_config = attrs->ctx_config; - struct wd_sched *alg_sched = attrs->sched; - int driver_type = attrs->driver->calc_type; + int ret; - if (!ctx_config) { - wd_sched_rr_release(alg_sched); - return; + if (!attrs) { + WD_ERR("invalid: attrs is NULL!\n"); + return -WD_EINVAL; } - switch (driver_type) { - case UADK_ALG_SOFT: - case UADK_ALG_CE_INSTR: - wd_alg_ce_ctx_uninit(ctx_config); - break; - case UADK_ALG_SVE_INSTR: - wd_alg_uninit_sve_ctx(ctx_config); - break; - case UADK_ALG_HW: - wd_alg_ctx_uninit(ctx_config); - break; - default: - break; + /* Driver discovery and configuration setup */ + ret = wd_alg_config_init(attrs); + if (ret) { + WD_ERR("failed to discover drivers!\n"); + goto out_undiscover; } + WD_DEBUG("discovered %u unique drivers\n", attrs->ctx_config_internal->drv_count); - free(ctx_config); - wd_sched_rr_release(alg_sched); + /* Scheduler compatibility check */ + ret = wd_task_sched_check(attrs); + if (ret) { + WD_ERR("failed to match task type with sched type!\n"); + goto out_undiscover; + } + + /* Allocate ctxs, init scheduler and algorithm */ + ret = wd_alg_ctx_init(attrs); + if (ret) { + WD_ERR("failed to init ctx!\n"); + goto out_undiscover; + } + + WD_DEBUG("Algorithm initialization complete: %u contexts from %u drivers\n", + attrs->ctx_config_internal->ctx_num, attrs->ctx_config_internal->drv_count); + + return WD_SUCCESS; + +out_undiscover: + wd_alg_config_uninit(attrs); + return ret; } -- 2.43.0
From: Longfang Liu <liulongfang@huawei.com> This represents the core modification of the new framework. It implements unified abstract processing for all device queues across the entire UADK, and unifies the channel for service requests from UADK to user devices into queues. These queues simultaneously support hardware device queues, software computation queues, and instruction queues, thereby achieving unified queue management. Additionally, the binding relationship between the algorithm layer and the driver has been decoupled, allowing a single algorithm layer to map and utilize multiple types of devices simultaneously for algorithm acceleration, thus enabling heterogeneous fusion accelerated computing. Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> Signed-off-by: lizhi <lizhi206@huawei.com> Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> --- Makefile.am | 15 ++- conf.sh | 19 ++-- configure.ac | 10 ++ drv/hisi_udma.c | 65 +++++++++---- drv/isa_ce_sm3.c | 13 ++- drv/wd_drv.c | 4 + include/drv/wd_aead_drv.h | 1 - include/wd.h | 5 - include/wd_alg_common.h | 35 ++++++- include/wd_bmm.h | 2 +- include/wd_internal.h | 15 +-- include/wd_util.h | 60 +++++------- wd.c | 78 ++++++++------- wd_aead.c | 146 +++++++++++++++++++--------- wd_bmm.c | 72 +++++++++++--- wd_cipher.c | 2 +- wd_comp.c | 26 ++--- wd_mempool.c | 3 +- wd_sched.c | 197 ++++++++++++++++++++++++++++++++++---- 19 files changed, 550 insertions(+), 218 deletions(-) diff --git a/Makefile.am b/Makefile.am index c08c825..d2f6cfe 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,8 +1,15 @@ ACLOCAL_AMFLAGS = -I m4 -I./include AUTOMAKE_OPTIONS = foreign subdir-objects AM_CFLAGS=-Wall -Werror -fno-strict-aliasing -I$(top_srcdir)/include -AM_CFLAGS+=-fPIC -fPIE -pie -fstack-protector-strong -D_FORTIFY_SOURCE=2 \ - -O2 -ftrapv -Wl,-z,relro,-z,now -Wl,-s +AM_CFLAGS+=-fPIC -fPIE -pie -fstack-protector-strong -ftrapv -Wl,-z,relro,-z,now + +if WD_DEBUG +# Debug build: keep symbols, disable optimization and stripping for full stack traces +AM_CFLAGS+=-g -O0 -fno-omit-frame-pointer -DDEBUG +else +# Release build: optimize and strip symbols +AM_CFLAGS+=-O2 -D_FORTIFY_SOURCE=2 -Wl,-s +endif CLEANFILES = if WITH_LOG_FILE @@ -84,13 +91,13 @@ libhisi_zip_la_SOURCES=drv/hisi_comp.c hisi_comp.h drv/hisi_qm_udrv.c \ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ wd_aead.c wd_aead.h wd_aead_drv.h \ - wd.c wd.h wd_alg.h \ wd_rsa.c wd_rsa.h wd_rsa_drv.h \ wd_dh.c wd_dh.h wd_dh_drv.h \ wd_ecc.c wd_ecc.h wd_ecc_drv.h \ wd_digest.c wd_digest.h wd_digest_drv.h \ wd_util.c wd_util.h \ - wd_sched.c wd_sched.h + wd_sched.c wd_sched.h \ + wd.c wd.h libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ lib/crypto/aes.c lib/crypto/sm4.c lib/crypto/galois.c \ diff --git a/conf.sh b/conf.sh index af5f787..d30912c 100755 --- a/conf.sh +++ b/conf.sh @@ -4,12 +4,18 @@ # Build UADK into static library COMPILE_TYPE="--disable-static --enable-shared" +DEBUG_TYPE="" -# These two parameters could be in arbitary sequence -if [[ $1 && $1 = "--static" ]] || [[ $2 && $2 = "--static" ]]; then - echo "Configure to static compile!" - COMPILE_TYPE="--enable-static --disable-shared --with-static_drv" -fi +# These parameters could be in arbitary sequence +for arg in "$1" "$2" "$3"; do + if [[ $arg = "--static" ]]; then + echo "Configure to static compile!" + COMPILE_TYPE="--enable-static --disable-shared --with-static_drv" + elif [[ $arg = "--debug" ]]; then + echo "Configure to debug compile (with symbols, no optimization)!" + DEBUG_TYPE="--enable-debug=yes" + fi +done export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH ac_cv_func_malloc_0_nonnull=yes ac_cv_func_realloc_0_nonnull=yes ./configure -v \ @@ -17,4 +23,5 @@ ac_cv_func_malloc_0_nonnull=yes ac_cv_func_realloc_0_nonnull=yes ./configure -v --host aarch64-linux-gnu \ --target aarch64-linux-gnu \ --includedir=/usr/local/include/ \ - $COMPILE_TYPE + $COMPILE_TYPE \ + $DEBUG_TYPE diff --git a/configure.ac b/configure.ac index 7261b5e..7e5d723 100644 --- a/configure.ac +++ b/configure.ac @@ -34,6 +34,16 @@ AC_ARG_ENABLE([debug-log], [debug_log=false] ) +AC_ARG_ENABLE([debug], + AS_HELP_STRING([--enable-debug], [build with -g -O0 and no symbol stripping for full stack traces]), + [ AS_IF([test "x$enable_debug" = "xyes"], + [wd_debug=true], + [wd_debug=false]) + ], + [wd_debug=false] +) +AM_CONDITIONAL([WD_DEBUG], [test "x$wd_debug" = "xtrue"]) + AC_ARG_ENABLE([perf], AS_HELP_STRING([--disble-perf], [enable measuring performance]), [ AS_IF([test "x$enable_perf" = "xyes"], diff --git a/drv/hisi_udma.c b/drv/hisi_udma.c index a9f5607..2e5aa13 100644 --- a/drv/hisi_udma.c +++ b/drv/hisi_udma.c @@ -12,6 +12,7 @@ #include <sys/types.h> #include "hisi_qm_udrv.h" #include "../include/drv/wd_udma_drv.h" +#include "wd_drv.h" #define BIT(nr) (1UL << (nr)) #define UDMA_CTX_Q_NUM_DEF 1 @@ -100,7 +101,8 @@ struct udma_internal_addr { }; struct hisi_udma_ctx { - struct wd_ctx_config_internal config; + struct wd_ctx_internal **ctxs; + __u32 ctx_num; }; static int get_free_inter_addr(struct udma_internal_addr *inter_addr) @@ -290,7 +292,7 @@ static void fill_init_value(struct udma_sqe *sqe, struct wd_udma_msg *msg) memset(&sqe->init_val, msg->value, sizeof(__u64)); } -static int udma_send(struct wd_alg_driver *drv, handle_t ctx, void *udma_msg) +static int udma_send(handle_t ctx, void *udma_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -342,7 +344,7 @@ static void dump_udma_msg(struct udma_sqe *sqe, struct wd_udma_msg *msg) "op_type:%u addr_num:%d.\n", msg->op_type, msg->addr_num); } -static int udma_recv(struct wd_alg_driver *drv, handle_t ctx, void *udma_msg) +static int udma_recv(handle_t ctx, void *udma_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -442,36 +444,61 @@ free_inter_addr: return ret; } -static int udma_init(struct wd_alg_driver *drv, void *conf) +static int udma_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; + struct hisi_udma_ctx *uctx = priv; struct hisi_qm_priv qm_priv; - struct hisi_udma_ctx *priv; - handle_t h_qp = 0; - handle_t h_ctx; - __u32 i, j; + __u32 i, j, count = 0; + bool *is_match; + handle_t h_qp; int ret; if (!config || !config->ctx_num) { - WD_ERR("invalid: udma init config is null or ctx num is 0!\n"); + WD_ERR("invalid: input config or ctx num is null!\n"); return -WD_EINVAL; } - priv = malloc(sizeof(struct hisi_udma_ctx)); - if (!priv) + is_match = malloc(config->ctx_num * sizeof(bool)); + if (!is_match) + return -WD_ENOMEM; + + for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx && config->ctxs[i].drv && + strcmp(config->ctxs[i].drv->drv_name, "hisi_udma") == 0) { + is_match[i] = true; + count++; + } else { + is_match[i] = false; + } + } + + if (!count) { + WD_ERR("invalid: valid driver number is zero!\n"); + free(is_match); + return -WD_EINVAL; + } + + uctx->ctxs = calloc(count, sizeof(struct wd_ctx_internal *)); + if (!uctx->ctxs) { + free(is_match); return -WD_ENOMEM; + } + uctx->ctx_num = count; qm_priv.op_type = UDMA_ALG_TYPE; qm_priv.sqe_size = sizeof(struct udma_sqe); + count = 0; /* Allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { - h_ctx = config->ctxs[i].ctx; + if (!is_match[i]) + continue; qm_priv.qp_mode = config->ctxs[i].ctx_mode; /* Setting the epoll en to 0 for ASYNC ctx */ qm_priv.epoll_en = (qm_priv.qp_mode == CTX_MODE_SYNC) ? config->epoll_en : 0; qm_priv.idx = i; - h_qp = hisi_qm_alloc_qp(&qm_priv, h_ctx); + h_qp = hisi_qm_alloc_qp(&qm_priv, config->ctxs[i].ctx); if (!h_qp) { ret = -WD_ENOMEM; goto out; @@ -480,20 +507,22 @@ static int udma_init(struct wd_alg_driver *drv, void *conf) ret = udma_init_qp_priv(h_qp); if (ret) goto free_h_qp; + uctx->ctxs[count++] = &config->ctxs[i]; } - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + free(is_match); return WD_SUCCESS; + free_h_qp: hisi_qm_free_qp(h_qp); out: - for (j = 0; j < i; j++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); + for (j = 0; j < count; j++) { + h_qp = (handle_t)wd_ctx_get_priv(uctx->ctxs[j]->ctx); udma_uninit_qp_priv(h_qp); hisi_qm_free_qp(h_qp); } - free(priv); + free(uctx->ctxs); + free(is_match); return ret; } diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c index 8d23061..627ab68 100644 --- a/drv/isa_ce_sm3.c +++ b/drv/isa_ce_sm3.c @@ -17,17 +17,17 @@ #include "drv/isa_ce_sm3.h" #include "drv/wd_digest_drv.h" #include "wd_digest.h" -#include "wd_util.h" +#include "wd_drv.h" #define SM3_ALIGN_MASK 63U typedef void (sm3_ce_block_fn)(__u32 word_reg[SM3_STATE_WORDS], const unsigned char *src, size_t blocks); -static int sm3_ce_drv_init(struct wd_alg_driver *drv, void *conf); -static void sm3_ce_drv_exit(struct wd_alg_driver *drv); -static int sm3_ce_drv_send(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg); -static int sm3_ce_drv_recv(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg); +static int sm3_ce_drv_init(void *conf, void *priv); +static void sm3_ce_drv_exit(void *priv); +static int sm3_ce_drv_send(handle_t ctx, void *digest_msg); +static int sm3_ce_drv_recv(handle_t ctx, void *digest_msg); static int sm3_ce_get_usage(void *param); static struct wd_alg_driver sm3_ce_alg_driver = { @@ -35,6 +35,7 @@ static struct wd_alg_driver sm3_ce_alg_driver = { .alg_name = "sm3", .calc_type = UADK_ALG_CE_INSTR, .priority = 200, + .priv_size = sizeof(struct sm3_ce_drv_ctx), .queue_num = 1, .op_type_num = 1, .fallback = 0, @@ -43,6 +44,8 @@ static struct wd_alg_driver sm3_ce_alg_driver = { .send = sm3_ce_drv_send, .recv = sm3_ce_drv_recv, .get_usage = sm3_ce_get_usage, + .alloc_ctx = wd_soft_alloc_ctx, + .free_ctx = wd_soft_free_ctx, }; static void __attribute__((constructor)) sm3_ce_probe(void) diff --git a/drv/wd_drv.c b/drv/wd_drv.c index 83dcbb6..16dc24c 100644 --- a/drv/wd_drv.c +++ b/drv/wd_drv.c @@ -201,6 +201,10 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) WD_ERR("invalid: alg_name is NULL!\n"); return -WD_EINVAL; } + if (!strcmp(alg_type, "ecc")) + (void)strcpy(alg_type, "sm2"); + if (!strcmp(alg_type, "comp")) + (void)strcpy(alg_type, "zlib"); dev_list = wd_get_accel_list(alg_type); if (!dev_list) { diff --git a/include/drv/wd_aead_drv.h b/include/drv/wd_aead_drv.h index 2c53217..612a8f2 100644 --- a/include/drv/wd_aead_drv.h +++ b/include/drv/wd_aead_drv.h @@ -78,7 +78,6 @@ struct wd_aead_aiv_addr { }; struct wd_aead_extend_ops { - void *params; int (*eops_aiv_init)(struct wd_alg_driver *drv, struct wd_mm_ops *mm_ops, void **params); diff --git a/include/wd.h b/include/wd.h index a468047..6329528 100644 --- a/include/wd.h +++ b/include/wd.h @@ -104,11 +104,6 @@ typedef void (*wd_log)(const char *format, ...); #define WD_IS_ERR(h) ((uintptr_t)(h) > \ (uintptr_t)(-1000)) -enum wd_buff_type { - WD_FLAT_BUF, - WD_SGL_BUF, -}; - enum wd_alg_type { WD_CIPHER, WD_DIGEST, diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index ebba7f8..be4e3b5 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -56,6 +56,11 @@ enum wd_ctx_mode { CTX_MODE_MAX, }; +enum wd_buff_type { + WD_FLAT_BUF, + WD_SGL_BUF, +}; + enum wd_init_type { WD_TYPE_V1, WD_TYPE_V2, @@ -70,7 +75,8 @@ enum wd_mem_type { /* * struct wd_ctx - Define one ctx and related type. - * @ctx: The ctx itself. + * @ctx: The ctx itself, the hardware queue is wd_ctx_h. + * The soft computing queue is wd_soft_ctx. * @op_type: Define the operation type of this specific ctx. * e.g. 0: compression; 1: decompression. * @ctx_mode: Define this ctx is used for synchronization of asynchronization @@ -82,7 +88,7 @@ struct wd_ctx { __u8 ctx_mode; }; -/* +/** * struct wd_cap_config - Capabilities. * @ctx_msg_num: number of asynchronous msg pools that the user wants to allocate. * Optional, user can set ctx_msg_num based on the number of requests @@ -95,7 +101,7 @@ struct wd_cap_config { __u32 resv; }; -/* +/** * struct wd_ctx_config - Define a ctx set and its related attributes, which * will be used in the scope of current process. * @ctx_num: The ctx number in below ctx array. @@ -111,7 +117,7 @@ struct wd_ctx_config { struct wd_cap_config *cap; }; -/* +/** * struct wd_ctx_nums - Define the ctx sets numbers. * @sync_ctx_num: The ctx numbers which are used for sync mode for each * ctx sets. @@ -123,7 +129,7 @@ struct wd_ctx_nums { __u32 async_ctx_num; }; -/* +/** * struct wd_ctx_params - Define the ctx sets params which are used for init * algorithms. * @op_type_num: Used for index of ctx_set_num, the order is the same as @@ -170,6 +176,25 @@ struct wd_sched { typedef int (*wd_alg_init)(struct wd_ctx_config *config, struct wd_sched *sched); typedef int (*wd_alg_poll_ctx)(__u32 idx, __u32 expt, __u32 *count); +/** + * struct wd_init_attrs - Algorithm initialization attributes. + * + * Updated: No longer contains driver field. + * Initialization path determined solely by task_type. + */ +struct wd_init_attrs { + __u32 sched_type; + __u32 task_type; + char alg[CRYPTO_MAX_ALG_NAME]; + struct wd_sched *sched; + struct wd_ctx_params *ctx_params; + struct wd_ctx_config *ctx_config; + wd_alg_init alg_init; + wd_alg_poll_ctx alg_poll_ctx; + + struct wd_ctx_config_internal *ctx_config_internal; +}; + #ifdef __cplusplus } #endif diff --git a/include/wd_bmm.h b/include/wd_bmm.h index 76b56a0..4abaa4e 100644 --- a/include/wd_bmm.h +++ b/include/wd_bmm.h @@ -33,7 +33,7 @@ int wd_get_fail_num(void *pool, __u32 *fail_num); __u32 wd_get_bufsize(void *pool); handle_t wd_find_ctx(const char *alg_name); -void wd_remove_ctx_list(void); +void wd_remove_ctx_list(handle_t h_ctx); int wd_insert_ctx_list(handle_t h_ctx, char *alg_name); __u32 wd_get_dev_id(void *pool); diff --git a/include/wd_internal.h b/include/wd_internal.h index 3e8622c..95ceaa2 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -20,6 +20,7 @@ extern "C" { #define MAX_FD_NUM 65535 struct wd_ctx_h { + __u8 ctx_type; int fd; char dev_path[MAX_DEV_NAME_LEN]; char *dev_name; @@ -30,17 +31,6 @@ struct wd_ctx_h { void *priv; }; -struct wd_soft_ctx { - int fd; - void *priv; -}; - -struct wd_ce_ctx { - int fd; - char *drv_name; - void *priv; -}; - struct wd_ctx_internal { __u8 op_type; __u8 ctx_mode; @@ -60,6 +50,9 @@ struct wd_ctx_config_internal { bool epoll_en; unsigned long *msg_cnt; char *alg_name; + + struct wd_alg_driver **drv_array; + __u32 drv_count; }; struct wd_datalist { diff --git a/include/wd_util.h b/include/wd_util.h index 56dd9ba..43b89d1 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -118,19 +118,8 @@ struct wd_ctx_attr { }; struct wd_msg_handle { - int (*send)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); - int (*recv)(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg); -}; - -struct wd_init_attrs { - __u32 sched_type; - char alg[CRYPTO_MAX_ALG_NAME]; - struct wd_alg_driver *driver; - struct wd_sched *sched; - struct wd_ctx_params *ctx_params; - struct wd_ctx_config *ctx_config; - wd_alg_init alg_init; - wd_alg_poll_ctx alg_poll_ctx; + int (*send)(handle_t sess, void *msg); + int (*recv)(handle_t sess, void *msg); }; /* @@ -345,7 +334,6 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en); /** * wd_handle_msg_sync() - recv msg from hardware - * @drv: the driver to handle msg. * @msg_handle: callback of msg handle ops. * @ctx: the handle of context. * @msg: the msg of task. @@ -354,8 +342,8 @@ int wd_set_epoll_en(const char *var_name, bool *epoll_en); * * Return 0 if successful or less than 0 otherwise. */ -int wd_handle_msg_sync(struct wd_alg_driver *drv, struct wd_msg_handle *msg_handle, - handle_t ctx, void *msg, __u64 *balance, bool epoll_en); +int wd_handle_msg_sync(struct wd_msg_handle *msg_handle, handle_t ctx, + void *msg, __u64 *balance, bool epoll_en); /** * wd_init_check() - Check input parameters for wd_<alg>_init. @@ -414,7 +402,7 @@ static inline void wd_alg_clear_init(enum wd_status *status) * to the obtained queue resource and the applied driver. * @ctx_params: wd_ctx_params to be initialized. * @user_ctx_params: user input wd_ctx_params. - * @driver: device driver for the current algorithm application. + * @alg: Name of the algorithm. * @type: algorithm type. * @max_op_type: algorithm max operation type. * @@ -422,8 +410,8 @@ static inline void wd_alg_clear_init(enum wd_status *status) */ int wd_ctx_param_init(struct wd_ctx_params *ctx_params, struct wd_ctx_params *user_ctx_params, - struct wd_alg_driver *driver, - enum wd_type type, int max_op_type); + char *alg, enum wd_type type, + int max_op_type); void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params); @@ -437,29 +425,17 @@ void wd_ctx_param_uninit(struct wd_ctx_params *ctx_params); int wd_alg_attrs_init(struct wd_init_attrs *attrs); void wd_alg_attrs_uninit(struct wd_init_attrs *attrs); -/** - * wd_alg_drv_bind() - Request the ctxs and initialize the sched_domain - * with the given devices list, ctxs number and numa mask. - * @task_type: the type of task specified by the current algorithm. - * @alg_name: the name of the algorithm specified by the task. - * - * Return device driver if succeed and other NULL if fail. - */ -struct wd_alg_driver *wd_alg_drv_bind(int task_type, const char *alg_name); -void wd_alg_drv_unbind(struct wd_alg_driver *drv); - /** * wd_alg_init_driver() - Initialize the current device driver according * to the obtained queue resource and the applied driver. * @config: device resources requested by the current algorithm. * @driver: device driver for the current algorithm application. + * @drv_priv: the parameter pointer of the current device driver. * * Return 0 if succeed and other error number if fail. */ -int wd_alg_init_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver); -void wd_alg_uninit_driver(struct wd_ctx_config_internal *config, - struct wd_alg_driver *driver); +int wd_alg_init_driver(struct wd_ctx_config_internal *config); +void wd_alg_uninit_driver(struct wd_ctx_config_internal *config); /** * wd_dlopen_drv() - Open the dynamic library file of the device driver. @@ -490,10 +466,16 @@ static inline void wd_dfx_msg_cnt(struct wd_ctx_config_internal *config, bool ret; ret = wd_need_info(); - if (idx > numsize || !ret) + if (idx >= numsize || !ret) + return; + + if (!config->msg_cnt) return; sqn = config->ctxs[idx].sqn; + if (sqn >= numsize) + return; + config->msg_cnt[sqn]++; } @@ -522,6 +504,14 @@ static inline void wd_ctx_spin_unlock(struct wd_ctx_internal *ctx, int type) int wd_mem_ops_init(handle_t h_ctx, struct wd_mm_ops *mm_ops, int mem_type); +int wd_alg_config_init(struct wd_init_attrs *attrs); +void wd_alg_config_uninit(struct wd_init_attrs *attrs); +int wd_alg_ctx_init(struct wd_init_attrs *attrs); +void wd_alg_ctx_uninit(struct wd_init_attrs *attrs); +int wd_ctx_bind_drivers(struct wd_ctx_config_internal *config_api, + struct wd_ctx_config_internal *config_in, int init_type); +void wd_ctx_unbind_drivers(struct wd_ctx_config_internal *config); + #ifdef __cplusplus } #endif diff --git a/wd.c b/wd.c index 9cdc70f..ede0bf2 100644 --- a/wd.c +++ b/wd.c @@ -21,9 +21,11 @@ #include "wd.h" #include "wd_alg.h" #include "wd_internal.h" +#include "wd_bmm.h" #define SYS_CLASS_DIR "/sys/class/uacce" #define FILE_MAX_SIZE (8 << 20) #define WD_DEV_USAGE_SIZE 256 +#define WD_DEFAULT_NUMA_DISTANCE 1024 enum UADK_LOG_LEVEL { WD_LOG_NONE = 0, @@ -77,15 +79,10 @@ int memcmp_consttime(const void *s1, const void *s2, size_t n) static int wd_check_ctx_type(handle_t h_ctx) { - struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; - - /* A simple and efficient method to check the queue type */ - if (ctx->fd < 0 || ctx->fd > MAX_FD_NUM) { - WD_INFO("Invalid: this ctx not HW ctx.\n"); - return -WD_HW_EACCESS; - } + if (h_ctx && (*(__u8 *)h_ctx == UADK_ALG_HW)) + return 0; - return 0; + return -WD_HW_EACCESS; } static void wd_parse_log_level(void) @@ -193,6 +190,7 @@ static int get_int_attr(struct uacce_dev *dev, const char *attr, int *val) if (ret < 0) return ret; + errno = 0; *val = strtol(buf, NULL, 10); if (errno == ERANGE) { WD_ERR("failed to strtol %s, out of range!\n", buf); @@ -205,7 +203,7 @@ static int get_int_attr(struct uacce_dev *dev, const char *attr, int *val) static int get_str_attr(struct uacce_dev *dev, const char *attr, char *buf, size_t buf_sz) { - __u32 ret; + int ret; int size; size = get_raw_attr(dev->dev_root, attr, buf, buf_sz); @@ -489,6 +487,7 @@ void wd_release_ctx(handle_t h_ctx) if (!ctx || wd_check_ctx_type(h_ctx)) return; + wd_remove_ctx_list(h_ctx); close(ctx->fd); free(ctx->dev); free(ctx->drv_name); @@ -847,9 +846,9 @@ struct uacce_dev *wd_get_accel_dev(const char *alg_name) { struct uacce_dev_list *list, *head; struct uacce_dev *dev = NULL, *target = NULL; + int dis = WD_DEFAULT_NUMA_DISTANCE; unsigned int node; int ctx_num, tmp; - int dis = 1024; int max = 0; /* Under default conditions in a VM, the node value is 0 */ @@ -915,7 +914,8 @@ struct bitmask *wd_create_device_nodemask(struct uacce_dev_list *list) p = list; while (p) { - numa_bitmask_setbit(bmp, p->dev->numa_id); + if (p->dev) + numa_bitmask_setbit(bmp, p->dev->numa_id); p = p->next; } @@ -997,33 +997,43 @@ void wd_release_alg_cap(struct wd_capability *head) struct wd_capability *wd_get_alg_cap(void) { - struct wd_alg_list *head = wd_get_alg_head(); - struct wd_alg_list *pnext = head->next; + struct wd_drv_node *head = wd_get_alg_head(); + struct wd_drv_node *drv_node = head->next; struct wd_capability *cap_head = NULL; struct wd_capability *cap_pnext = NULL; struct wd_capability *cap_node; - - while (pnext) { - cap_node = calloc(1, sizeof(struct wd_capability)); - if (!cap_node) { - WD_ERR("fail to alloc wd capability head\n"); - goto alloc_err; - } - - (void)strcpy(cap_node->alg_name, pnext->alg_name); - (void)strcpy(cap_node->drv_name, pnext->drv_name); - cap_node->available = pnext->available; - cap_node->priority = pnext->priority; - cap_node->calc_type = pnext->calc_type; - cap_node->next = NULL; - - pnext = pnext->next; - if (!cap_pnext) { - cap_head = cap_node; - cap_pnext = cap_node; + int i; + + while (drv_node) { + /* Traverse the static algorithm array inside each driver node */ + for (i = 0; i < drv_node->alg_count; i++) { + cap_node = calloc(1, sizeof(struct wd_capability)); + if (!cap_node) { + WD_ERR("fail to alloc wd capability head\n"); + goto alloc_err; + } + /* Flatten the secondary structure into the original binary-tuple format */ + strncpy(cap_node->alg_name, drv_node->algs[i].alg_name, + CRYPTO_MAX_ALG_NAME - 1); + cap_node->alg_name[CRYPTO_MAX_ALG_NAME - 1] = '\0'; + strncpy(cap_node->drv_name, drv_node->drv_name, + CRYPTO_MAX_ALG_NAME - 1); + cap_node->drv_name[CRYPTO_MAX_ALG_NAME - 1] = '\0'; + cap_node->available = drv_node->algs[i].available; + cap_node->priority = drv_node->priority; + cap_node->calc_type = drv_node->calc_type; + cap_node->next = NULL; + + /* Append to the capability linked list */ + if (!cap_head) { + cap_head = cap_node; + cap_pnext = cap_node; + } else { + cap_pnext->next = cap_node; + cap_pnext = cap_node; + } } - cap_pnext->next = cap_node; - cap_pnext = cap_node; + drv_node = drv_node->next; } return cap_head; diff --git a/wd_aead.c b/wd_aead.c index a9e6279..eb3597e 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -32,7 +32,6 @@ struct wd_aead_setting { enum wd_status status; struct wd_ctx_config_internal config; struct wd_sched sched; - struct wd_alg_driver *driver; struct wd_async_msg_pool pool; void *dlhandle; void *dlh_list; @@ -51,7 +50,7 @@ struct wd_aead_sess { __u16 ckey_bytes; __u16 akey_bytes; __u16 auth_bytes; - void *priv; + void **priv; void *sched_key; /* Stored the counter for gcm stream mode */ __u8 *iv; @@ -59,7 +58,6 @@ struct wd_aead_sess { __u64 long_data_len; struct wd_mm_ops mm_ops; enum wd_mem_type mm_type; - struct wd_aead_extend_ops eops; }; struct wd_env_config wd_aead_env_config; @@ -75,20 +73,16 @@ static void wd_aead_close_driver(int init_type) } if (wd_aead_setting.dlhandle) { - wd_release_drv(wd_aead_setting.driver); dlclose(wd_aead_setting.dlhandle); wd_aead_setting.dlhandle = NULL; } #else - wd_release_drv(wd_aead_setting.driver); hisi_sec2_remove(); #endif } static int wd_aead_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "gcm(aes)"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -122,14 +116,6 @@ static int wd_aead_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_aead_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_aead_setting.driver = driver; return WD_SUCCESS; } @@ -334,7 +320,7 @@ static struct wd_aead_sess *check_and_init_sess(struct wd_aead_sess_setup *setup sess->dalg = setup->dalg; sess->dmode = setup->dmode; - ret = wd_drv_alg_support(sess->alg_name, wd_aead_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, &wd_aead_setting.config); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); free(sess); @@ -368,7 +354,7 @@ static int aead_setup_memory_and_buffers(struct wd_aead_sess *sess, sess->mac_bak = aead_alloc_func(mempool, WD_AEAD_CCM_GCM_MAX); if (!sess->mac_bak) { - WD_ERR("aead failed to calloc mac_bak memory!\n"); + WD_ERR("failed to calloc aead mac_bak memory!\n"); return -WD_ENOMEM; } memset(sess->mac_bak, 0, WD_AEAD_CCM_GCM_MAX); @@ -418,35 +404,99 @@ static void cleanup_session(struct wd_aead_sess *sess) static int wd_aead_sess_eops_init(struct wd_aead_sess *sess) { - int ret; - - if (sess->eops.eops_aiv_init) { - if (!sess->eops.eops_aiv_uninit) { - WD_ERR("failed to get aead extend ops free in session!\n"); - return -WD_EINVAL; + struct wd_ctx_config_internal *config = &wd_aead_setting.config; + struct wd_aead_extend_ops *eops; + struct wd_alg_driver *drv; + int ret, prev_idx; + int valid = 0; + __u32 i, j; + + /* Allocate priv array like wd_agg does */ + sess->priv = malloc(sizeof(void *) * config->ctx_num); + if (!sess->priv) + return -WD_ENOMEM; + memset(sess->priv, 0, sizeof(void *) * config->ctx_num); + + for (i = 0; i < config->ctx_num; i++) { + drv = config->ctxs[i].drv; + /* the drv initial assignment is not NULL. */ + if (!drv->get_extend_ops) + continue; + + /* check if same driver was already initialized. */ + prev_idx = -1; + for (j = 0; j < i; j++) { + if (!strcmp(config->ctxs[j].drv->drv_name, drv->drv_name)) { + prev_idx = j; + break; + } } - ret = sess->eops.eops_aiv_init(wd_aead_setting.driver, &sess->mm_ops, - &sess->eops.params); - if (ret) { - WD_ERR("failed to init aead extend ops params in session!\n"); - return ret; + if (prev_idx >= 0) { + /* Reuse existing priv */ + sess->priv[i] = sess->priv[prev_idx]; + continue; + } + + ret = drv->get_extend_ops(config->ctxs[i].drv->extend_ops); + if (!ret && config->ctxs[i].drv->extend_ops) { + eops = config->ctxs[i].drv->extend_ops; + + if (!eops->eops_aiv_init || !eops->eops_aiv_uninit) + continue; + /* call eops_aiv_init and store in priv[i]. */ + ret = eops->eops_aiv_init(drv, &sess->mm_ops, &sess->priv[i]); + if (ret) { + WD_ERR("failed to init aead extend ops params in session!\n"); + return ret; + } + valid++; } } + if (!valid) { + WD_DEBUG("failed to get aead extend ops!\n"); + return -WD_EINVAL; + } + return WD_SUCCESS; } static void wd_aead_sess_eops_uninit(struct wd_aead_sess *sess) { - if (sess->eops.eops_aiv_uninit) { - sess->eops.eops_aiv_uninit(wd_aead_setting.driver, &sess->mm_ops, - sess->eops.params); - sess->eops.params = NULL; + struct wd_ctx_config_internal *config = &wd_aead_setting.config; + struct wd_aead_extend_ops *eops; + struct wd_alg_driver *drv; + int prev_idx; + __u32 i, j; + + if (!sess->priv) + return; + + for (i = 0; i < config->ctx_num; i++) { + drv = config->ctxs[i].drv; + /* check if this is a duplicate driver that was skipped. */ + prev_idx = -1; + for (j = 0; j < i; j++) { + if (!strcmp(config->ctxs[j].drv->drv_name, drv->drv_name)) { + prev_idx = j; + break; + } + } + if (prev_idx >= 0 || !sess->priv[i] || !drv->extend_ops) + continue; + + eops = drv->extend_ops; + if (eops->eops_aiv_uninit) + eops->eops_aiv_uninit(drv, &sess->mm_ops, sess->priv[i]); } + + free(sess->priv); + sess->priv = NULL; } handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) { + struct wd_sched_params params; struct wd_aead_sess *sess; int ret; @@ -459,32 +509,30 @@ handle_t wd_aead_alloc_sess(struct wd_aead_sess_setup *setup) return (handle_t)0; } - if (wd_aead_setting.driver->get_extend_ops) { - ret = wd_aead_setting.driver->get_extend_ops(&sess->eops); - if (ret) { - WD_ERR("failed to get aead sess extend ops!\n"); - goto sess_err; - } - } - ret = wd_aead_sess_eops_init(sess); if (ret) { WD_ERR("failed to init aead sess extend eops!\n"); - goto sess_err; + goto clean_up; } sess->sched_key = (void *)wd_aead_setting.sched.sched_init( wd_aead_setting.sched.h_sched_ctx, setup->sched_param); if (WD_IS_ERR(sess->sched_key)) { WD_ERR("failed to init session schedule key!\n"); - goto sched_key_err; + goto clean_up; } + /* Set compat filtering parameters for session-ctx matching */ + memset(¶ms, 0, sizeof(params)); + params.alg_name = sess->alg_name; + params.ctxs = wd_aead_setting.config.ctxs; + wd_aead_setting.sched.set_param(wd_aead_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); + return (handle_t)sess; -sched_key_err: +clean_up: wd_aead_sess_eops_uninit(sess); -sess_err: cleanup_session(sess); return (handle_t)0; } @@ -501,8 +549,14 @@ void wd_aead_free_sess(handle_t h_sess) wd_memset_zero(sess->ckey, sess->ckey_bytes); wd_memset_zero(sess->akey, sess->akey_bytes); - if (sess->sched_key) - free(sess->sched_key); + if (sess->sched_key) { + if (wd_aead_setting.sched.sched_uninit) + wd_aead_setting.sched.sched_uninit( + wd_aead_setting.sched.h_sched_ctx, + (handle_t)sess->sched_key); + else + free(sess->sched_key); + } wd_aead_sess_eops_uninit(sess); cleanup_session(sess); } diff --git a/wd_bmm.c b/wd_bmm.c index 462a638..811afa2 100644 --- a/wd_bmm.c +++ b/wd_bmm.c @@ -95,6 +95,46 @@ struct mem_ctx_node { static TAILQ_HEAD(, mem_ctx_node) g_mem_ctx_list = TAILQ_HEAD_INITIALIZER(g_mem_ctx_list); static pthread_mutex_t g_mem_ctx_mutex = PTHREAD_MUTEX_INITIALIZER; +/* + * Fork child handler for the global ctx pool. + * + * After fork() the child is single-threaded; the inherited mutex state is + * unreliable (the holding thread, if any, does not exist in the child). + * POSIX allows reinitializing a statically-initialized mutex in the child + * fork handler, so do that unconditionally and skip lock/unlock below. + * + * Discard all inherited ctx nodes. Each node's h_ctx points to a wd_ctx_h + * whose fd and mmap regions are shared with the parent (fd via dup, + * MAP_SHARED mmap via shared physical pages). Using these ctx in the child + * would cause parent/child to contend on the same hardware queue. Free only + * the node shells (COW copies, safe to free in the child). Do NOT call + * wd_release_ctx: closing fd or munmapping regions in the child could + * interfere with the parent's shared resources. The kernel reclaims child + * fds and VMAs on exit. + */ +static void wd_bmm_atfork_child(void) +{ + struct mem_ctx_node *node; + + pthread_mutex_init(&g_mem_ctx_mutex, NULL); + + while ((node = TAILQ_FIRST(&g_mem_ctx_list)) != NULL) { + TAILQ_REMOVE(&g_mem_ctx_list, node, list_node); + free(node); + } + TAILQ_INIT(&g_mem_ctx_list); +} + +/* + * Register the atfork handler once when libwd is loaded, before any user + * code (including fork) runs. The constructor attribute ensures single + * registration regardless of how many alg init calls happen later. + */ +static void __attribute__((constructor)) wd_bmm_ctor(void) +{ + pthread_atfork(NULL, NULL, wd_bmm_atfork_child); +} + handle_t wd_find_ctx(const char *alg_name) { struct mem_ctx_node *close_node = NULL; @@ -146,21 +186,29 @@ handle_t wd_find_ctx(const char *alg_name) return h_ctx; } -void wd_remove_ctx_list(void) +void wd_remove_ctx_list(handle_t h_ctx) { struct mem_ctx_node *node; + if (!h_ctx) + return; + pthread_mutex_lock(&g_mem_ctx_mutex); - /* Free all list node */ - while ((node = TAILQ_FIRST(&g_mem_ctx_list)) != NULL) { - /* Use TAILQ_REMOVE to remove list node */ - TAILQ_REMOVE(&g_mem_ctx_list, node, list_node); - free(node); + TAILQ_FOREACH(node, &g_mem_ctx_list, list_node) { + if (node->h_ctx == h_ctx) { + TAILQ_REMOVE(&g_mem_ctx_list, node, list_node); + free(node); + break; + } } - pthread_mutex_unlock(&g_mem_ctx_mutex); } +static inline bool wd_ctx_is_hw(handle_t h_ctx) +{ + return h_ctx && (*(__u8 *)h_ctx == UADK_ALG_HW); +} + int wd_insert_ctx_list(handle_t h_ctx, char *alg_name) { struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; @@ -173,8 +221,8 @@ int wd_insert_ctx_list(handle_t h_ctx, char *alg_name) } /* A simple and efficient method to check the queue type */ - if (ctx->fd < 0 || ctx->fd > MAX_FD_NUM) { - WD_INFO("Invalid ctx: this ctx not HW ctx.\n"); + if (!wd_ctx_is_hw(h_ctx)) { + WD_INFO("Notes: this ctx not HW ctx.\n"); return 0; } @@ -568,7 +616,7 @@ static int wd_pool_pre_layout(handle_t h_ctx, * When IOMMU is disabled, the PA refers to the kernel's physical address, which * must be physically contiguous to be allocated by the kernel. * Therefore, the PA address can be obtained from the offset of the VA. - * + * */ static void *wd_iova_map(struct ctx_info *cinfo, void *va, size_t sz) { @@ -918,7 +966,7 @@ void wd_mem_free(void *pool, void *buf) struct wd_blkpool *p = pool; struct wd_blk_hd *current_hd; struct wd_blk_hd *hd; - unsigned int current_idx; + unsigned int current_idx; unsigned int blk_idx; unsigned long offset; unsigned int i, num; @@ -946,7 +994,7 @@ void wd_mem_free(void *pool, void *buf) } /* Calculate the block index. */ - offset = (unsigned long)((uintptr_t)buf - (uintptr_t)p->act_start); + offset = (unsigned long)((uintptr_t)buf - (uintptr_t)p->act_start); blk_idx = offset / sz; /* Check if the index is valid. */ diff --git a/wd_cipher.c b/wd_cipher.c index 5258191..3bfead8 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -40,7 +40,7 @@ static const unsigned char des_weak_keys[DES_WEAK_KEY_NUM][DES_KEY_SIZE] = { static const char *wd_cipher_alg_name[WD_CIPHER_ALG_TYPE_MAX][WD_CIPHER_MODE_TYPE_MAX] = { {"ecb(sm4)", "cbc(sm4)", "ctr(sm4)", "xts(sm4)", "ofb(sm4)", "cfb(sm4)", "cbc-cs1(sm4)", "cbc-cs2(sm4)", "cbc-cs3(sm4)", - "", "", "xts(sm4)"}, + "", "", "xts-gb(sm4)"}, {"ecb(aes)", "cbc(aes)", "ctr(aes)", "xts(aes)", "ofb(aes)", "cfb(aes)", "cbc-cs1(aes)", "cbc-cs2(aes)", "cbc-cs3(aes)"}, {"ecb(des)", "cbc(des)",}, diff --git a/wd_comp.c b/wd_comp.c index 7a2b0bc..df32ea5 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -19,9 +19,10 @@ #define STREAM_CHUNK (128 * 1024) #define WD_ZLIB_HEADER_SZ 2 #define WD_GZIP_HEADER_SZ 10 +#define WD_COMP_ALG_TYPE_V1 "comp" static const char *wd_comp_alg_name[WD_COMP_ALG_MAX] = { - "zlib", "gzip", "deflate", "lz77_zstd", "lz4", "lz77_only" + "deflate", "zlib", "gzip", "lz77_zstd", "lz4", "lz77_only" }; struct wd_comp_sess { @@ -35,14 +36,17 @@ struct wd_comp_sess { void *sched_key; struct wd_mm_ops mm_ops; enum wd_mem_type mm_type; + bool strm_ctx_fixed; + __u32 strm_sync_ctx_idx; + }; struct wd_comp_setting { enum wd_status status; + enum wd_init_type init_type; struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; } wd_comp_setting; @@ -60,20 +64,16 @@ static void wd_comp_close_driver(int init_type) } if (wd_comp_setting.dlhandle) { - wd_release_drv(wd_comp_setting.driver); dlclose(wd_comp_setting.dlhandle); wd_comp_setting.dlhandle = NULL; } #else - wd_release_drv(wd_comp_setting.driver); hisi_zip_remove(); #endif } static int wd_comp_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "zlib"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -107,18 +107,11 @@ static int wd_comp_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_comp_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_comp_setting.driver = driver; - return WD_SUCCESS; } +static bool wd_comp_atfork_registered; + static void wd_comp_clear_status(void) { wd_alg_clear_init(&wd_comp_setting.status); @@ -163,8 +156,7 @@ static int wd_comp_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_comp_setting.config, - wd_comp_setting.driver); + ret = wd_alg_init_driver(&wd_comp_setting.config); if (ret) goto out_clear_pool; diff --git a/wd_mempool.c b/wd_mempool.c index 2f49764..3ce9ef0 100644 --- a/wd_mempool.c +++ b/wd_mempool.c @@ -943,7 +943,8 @@ void wd_mempool_destroy(handle_t mempool) } wd_atomic_sub(&mp->ref, 1); - while(wd_atomic_load(&mp->ref)); + while (wd_atomic_load(&mp->ref)) + sched_yield(); uninit_mempool(mp); free_hugepage_mem(mp); pthread_spin_destroy(&mp->lock); diff --git a/wd_sched.c b/wd_sched.c index 2efa2aa..f704d59 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -28,21 +28,17 @@ #include "wd_internal.h" #define MAX_POLL_TIMES 1000 -#define HUNGRY_LOAD_THRESHOLD 256 #define SKEY_CTX_MAX_NUM 16 #define SKEY_MAX_THREAD_NUM 64 #define SKEY_LOAD_UPDATE_INTERVAL 1 #define HW_QUEUE_FULL_DEPTH 1024 -#define MAX_NUMA_NODES (NUMA_NUM_NODES >> 5) - /* ============================================================================ * Hash Table Configuration * ============================================================================ */ #define WD_SCHED_MAX_BUCKETS 512 #define WD_SCHED_MIN_BUCKETS 32 -#define WD_SCHED_LOAD_FACTOR 0.75f #define HASH_PRIME1 73 #define HASH_PRIME2 13 #define HASH_PRIME3 7 @@ -239,6 +235,10 @@ struct wd_sched_ctx { __u32 skey_num; pthread_mutex_t skey_lock; struct wd_sched_key *skey[SKEY_MAX_THREAD_NUM]; + + /* First ctx index per mode, used by SINGLE/NONE. */ + __u32 sync_idx; + __u32 async_idx; }; /* ============================================================================ @@ -904,6 +904,130 @@ static void wd_sched_skey_domain_destroy(struct wd_sched_key_domain *key_domain) wd_sched_skey_cache_uninit(&key_domain->idx_cache); } +static __u32 wd_sched_find_compatible_ctx(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + int region, int sched_mode) +{ + struct wd_sched_ctx_domain *domain; + __u32 i, ctx_idx, prop; + + for (prop = 0; prop < UADK_ALG_TYPE_MAX; prop++) { + domain = wd_sched_hash_table_lookup(sched_ctx->domain_hash_table, + region, sched_mode, skey->type, prop); + if (!domain || !domain->valid) + continue; + for (i = 0; i < domain->total_ctx_count; i++) { + ctx_idx = wd_sched_domain_get_next_rr(domain); + if (ctx_idx == INVALID_POS) + continue; + if (skey->ctxs[ctx_idx].drv && + wd_alg_match_drv(skey->ctxs[ctx_idx].drv, skey->alg_name)) + return ctx_idx; + } + } + + return INVALID_POS; +} + +static __u32 wd_sched_get_new_ctx(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, + int sched_mode) +{ + int region_id = skey->region_id; + __u8 ctx_prop = skey->ctx_prop; + __u32 op_type = skey->type; + __u32 ctx_idx; + int r; + + if (sched_mode >= SCHED_MODE_BUTT || + op_type >= sched_ctx->type_num || ctx_prop >= UADK_ALG_TYPE_MAX) { + WD_ERR("invalid: region: %d, mode: %d, type: %u!, prop: %u\n", + region_id, sched_mode, op_type, ctx_prop); + return INVALID_POS; + } + + if (region_id < 0 || + (sched_ctx->policy != SCHED_POLICY_DEV && + region_id >= sched_ctx->region_num)) { + WD_ERR("invalid: region_id is %d, region_num is %u!\n", + region_id, sched_ctx->region_num); + return INVALID_POS; + } + + if (!sched_ctx->domain_hash_table) + return INVALID_POS; + + /* Try current region first */ + ctx_idx = wd_sched_find_compatible_ctx(sched_ctx, skey, region_id, sched_mode); + if (ctx_idx != INVALID_POS) + return ctx_idx; + + /* DEV policy must not cross region */ + if (sched_ctx->policy == SCHED_POLICY_DEV) + return INVALID_POS; + + /* Cross-region fallback: try all other regions */ + for (r = 0; r < sched_ctx->region_num; r++) { + if (r == region_id) + continue; + ctx_idx = wd_sched_find_compatible_ctx(sched_ctx, skey, r, sched_mode); + if (ctx_idx != INVALID_POS) + return ctx_idx; + } + + return INVALID_POS; +} + +/** + * wd_sched_skey_compat_filter - Filter and replace incompatible ctxs in domain cache + * @sched_ctx: Scheduler context + * @skey: Session key with alg_name and ctxs + * @domain: Target domain (sync or async) + * @sched_mode: SCHED_MODE_SYNC or SCHED_MODE_ASYNC + * + * For each ctx in domain cache, check if it supports alg_name. + * If not, find a compatible replacement from the global domain. + */ +static void wd_sched_skey_compat_filter(struct wd_sched_ctx *sched_ctx, + struct wd_sched_key *skey, struct wd_sched_key_domain *domain, int sched_mode) +{ + __u32 ctx_idx, new_ctx; + __u32 i; + + if (!skey || !skey->alg_name || !skey->ctxs || !domain) + return; + + /* Skip uninitialized domains (no ctxs cached) */ + if (!domain->idx_cache.valid_count) + return; + + pthread_mutex_lock(&domain->lock); + + for (i = 0; i < domain->idx_cache.valid_count; i++) { + ctx_idx = domain->idx_cache.idx_list[i]; + + /* Check if current ctx is compatible */ + if (skey->ctxs[ctx_idx].drv && + wd_alg_match_drv(skey->ctxs[ctx_idx].drv, skey->alg_name)) { + /* Compatible, keep unchanged */ + continue; + } + + /* Not compatible, find a replacement from domain */ + new_ctx = wd_sched_get_new_ctx(sched_ctx, skey, sched_mode); + if (new_ctx != INVALID_POS && new_ctx != ctx_idx) { + /* Found compatible ctx, replace */ + domain->idx_cache.idx_list[i] = new_ctx; + __atomic_store_n(&domain->idx_cache.load_values[i], 0, __ATOMIC_RELAXED); + } else { + /* No compatible ctx found, mark as invalid */ + domain->idx_cache.idx_list[i] = INVALID_POS; + WD_INFO("info: no compatible ctx found for alg %s!\n", skey->alg_name); + } + } + + pthread_mutex_unlock(&domain->lock); +} /** * wd_sched_poll_skey - Poll contexts for scheduler session @@ -1444,10 +1568,17 @@ static handle_t sched_none_init(handle_t h_sched_ctx, void *sched_param) return (handle_t)0; } -static __u32 sched_none_pick_next_ctx(handle_t sched_ctx, +static __u32 sched_none_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, const int sched_mode) { - return 0; + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + + if (!sched_ctx) { + WD_ERR("invalid: sched ctx is NULL!\n"); + return INVALID_POS; + } + + return sched_mode == SCHED_MODE_SYNC ? sched_ctx->sync_idx : sched_ctx->async_idx; } static int sched_none_poll_policy(handle_t h_sched_ctx, @@ -1455,7 +1586,7 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; __u32 loop_times = MAX_POLL_TIMES + expect; - __u32 poll_num = 0; + __u32 poll_num = 0, poll_idx; int ret; if (!sched_ctx || !sched_ctx->poll_func) { @@ -1463,9 +1594,15 @@ static int sched_none_poll_policy(handle_t h_sched_ctx, return -WD_EINVAL; } + poll_idx = sched_ctx->async_idx; + if (poll_idx == INVALID_POS) { + WD_ERR("invalid: no async ctx available to poll!\n"); + return -WD_EINVAL; + } + while (loop_times > 0) { loop_times--; - ret = sched_ctx->poll_func(0, 1, &poll_num); + ret = sched_ctx->poll_func(poll_idx, 1, &poll_num); if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; else if (ret == -WD_EAGAIN) @@ -1484,13 +1621,17 @@ static handle_t sched_single_init(handle_t h_sched_ctx, void *sched_param) return (handle_t)0; } -static __u32 sched_single_pick_next_ctx(handle_t sched_ctx, +static __u32 sched_single_pick_next_ctx(handle_t h_sched_ctx, void *sched_key, const int sched_mode) { - if (sched_mode) - return 1; - else - return 0; + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; + + if (!sched_ctx) { + WD_ERR("invalid: sched ctx is NULL!\n"); + return INVALID_POS; + } + + return sched_mode == SCHED_MODE_SYNC ? sched_ctx->sync_idx : sched_ctx->async_idx; } static int sched_single_poll_policy(handle_t h_sched_ctx, @@ -1498,7 +1639,7 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, { struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; __u32 loop_times = MAX_POLL_TIMES + expect; - __u32 poll_num = 0; + __u32 poll_num = 0, poll_idx; int ret; if (!sched_ctx || !sched_ctx->poll_func) { @@ -1506,9 +1647,15 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, return -WD_EINVAL; } + poll_idx = sched_ctx->async_idx; + if (poll_idx == INVALID_POS) { + WD_ERR("invalid: no async ctx available to poll!\n"); + return -WD_EINVAL; + } + while (loop_times > 0) { loop_times--; - ret = sched_ctx->poll_func(1, 1, &poll_num); + ret = sched_ctx->poll_func(poll_idx, 1, &poll_num); if ((ret < 0) && (ret != -WD_EAGAIN)) return ret; else if (ret == -WD_EAGAIN) @@ -1890,7 +2037,7 @@ static void wd_sched_set_param(handle_t h_sched_ctx, struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; if (unlikely(!params || !skey)) { - WD_ERR("invalid: sched parmas or skey is NULL!\n"); + WD_INFO("info: sched parmas or skey is NULL!\n"); return; } @@ -1902,6 +2049,13 @@ static void wd_sched_set_param(handle_t h_sched_ctx, skey->alg_name = params->alg_name; skey->ctxs = params->ctxs; + /* If compat info provided, fix up pre-fetched ctxs */ + if (skey->alg_name && skey->ctxs) { + wd_sched_skey_compat_filter(h_sched_ctx, skey, + &skey->sync_domain, SCHED_MODE_SYNC); + wd_sched_skey_compat_filter(sched_ctx, skey, + &skey->async_domain, SCHED_MODE_ASYNC); + } } static struct wd_sched sched_table[SCHED_POLICY_BUTT] = { @@ -2055,6 +2209,15 @@ int wd_sched_rr_instance(const struct wd_sched *sched, struct sched_params *para } domain->valid = true; + /* SINGLE/NONE: record first ctx index per mode. */ + if (sched_ctx->policy == SCHED_POLICY_SINGLE || + sched_ctx->policy == SCHED_POLICY_NONE) { + if (mode == SCHED_MODE_SYNC && sched_ctx->sync_idx == INVALID_POS) + sched_ctx->sync_idx = param->begin; + else if (mode == SCHED_MODE_ASYNC && sched_ctx->async_idx == INVALID_POS) + sched_ctx->async_idx = param->begin; + } + return WD_SUCCESS; } @@ -2172,6 +2335,8 @@ struct wd_sched *wd_sched_rr_alloc(__u8 sched_type, __u8 type_num, goto err_destroy_hash; } sched_ctx->skey_num = 0; + sched_ctx->sync_idx = INVALID_POS; + sched_ctx->async_idx = INVALID_POS; sched->h_sched_ctx = (handle_t)sched_ctx; sched->sched_init = sched_table[sched_type].sched_init; -- 2.43.0
From: Zhushuai Yin <yinzhushuai@huawei.com> This modification primarily aims to enable the UADK Cipher algorithm to support the new heterogeneous scheduling framework, while the corresponding user-space driver must also be adapted accordingly. Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- drv/hisi_sec.c | 232 +++++++++++++++++++++++++++-------------------- drv/isa_ce_sm4.c | 86 +++++++++++------- wd_cipher.c | 221 +++++++++++++++++++++++++++----------------- 3 files changed, 326 insertions(+), 213 deletions(-) diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 7d472b8..91fa4ce 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -10,6 +10,7 @@ #include "crypto/aes.h" #include "crypto/galois.h" #include "hisi_qm_udrv.h" +#include "wd_drv.h" #define BIT(nr) (1UL << (nr)) #define SEC_DIGEST_ALG_OFFSET 11 @@ -205,7 +206,8 @@ enum sec_c_width { }; struct hisi_sec_ctx { - struct wd_ctx_config_internal config; + struct wd_ctx_internal **ctxs; + __u32 ctx_num; }; struct hisi_sec_sqe_type2 { @@ -526,86 +528,84 @@ static __u32 g_sec_hmac_full_len[WD_DIGEST_TYPE_MAX] = { SEC_HMAC_SHA512_MAC_LEN, SEC_HMAC_SHA512_224_MAC_LEN, SEC_HMAC_SHA512_256_MAC_LEN }; -static int hisi_sec_init(struct wd_alg_driver *drv, void *conf); -static void hisi_sec_exit(struct wd_alg_driver *drv); +static int hisi_sec_init(void *conf, void *priv); +static void hisi_sec_exit(void *priv); -static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_cipher_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_send(handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_recv(handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_send_v3(handle_t ctx, void *wd_msg); +static int hisi_sec_cipher_recv_v3(handle_t ctx, void *wd_msg); -static int hisi_sec_digest_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_digest_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); +static int hisi_sec_digest_send(handle_t ctx, void *wd_msg); +static int hisi_sec_digest_recv(handle_t ctx, void *wd_msg); +static int hisi_sec_digest_send_v3(handle_t ctx, void *wd_msg); +static int hisi_sec_digest_recv_v3(handle_t ctx, void *wd_msg); -static int hisi_sec_aead_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); -static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg); +static int hisi_sec_aead_send(handle_t ctx, void *wd_msg); +static int hisi_sec_aead_recv(handle_t ctx, void *wd_msg); +static int hisi_sec_aead_send_v3(handle_t ctx, void *wd_msg); +static int hisi_sec_aead_recv_v3(handle_t ctx, void *wd_msg); -static int cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_send(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx); if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_cipher_send(drv, ctx, msg); - return hisi_sec_cipher_send_v3(drv, ctx, msg); + return hisi_sec_cipher_send(ctx, msg); + return hisi_sec_cipher_send_v3(ctx, msg); } -static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_recv(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx); if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_cipher_recv(drv, ctx, msg); - return hisi_sec_cipher_recv_v3(drv, ctx, msg); + return hisi_sec_cipher_recv(ctx, msg); + return hisi_sec_cipher_recv_v3(ctx, msg); } -static int digest_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int digest_send(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx); if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_digest_send(drv, ctx, msg); - return hisi_sec_digest_send_v3(drv, ctx, msg); + return hisi_sec_digest_send(ctx, msg); + return hisi_sec_digest_send_v3(ctx, msg); } -static int digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int digest_recv(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx); if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_digest_recv(drv, ctx, msg); - return hisi_sec_digest_recv_v3(drv, ctx, msg); + return hisi_sec_digest_recv(ctx, msg); + return hisi_sec_digest_recv_v3(ctx, msg); } -static int aead_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int aead_send(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx); if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_aead_send(drv, ctx, msg); - return hisi_sec_aead_send_v3(drv, ctx, msg); + return hisi_sec_aead_send(ctx, msg); + return hisi_sec_aead_send_v3(ctx, msg); } -static int aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int aead_recv(handle_t ctx, void *msg) { struct hisi_qp *qp = (struct hisi_qp *)wd_ctx_get_priv(ctx); if (qp->q_info.hw_type == HISI_QM_API_VER2_BASE) - return hisi_sec_aead_recv(drv, ctx, msg); - return hisi_sec_aead_recv_v3(drv, ctx, msg); + return hisi_sec_aead_recv(ctx, msg); + return hisi_sec_aead_recv_v3(ctx, msg); } static int hisi_sec_get_usage(void *param) { struct hisi_dev_usage *sec_usage = (struct hisi_dev_usage *)param; struct wd_alg_driver *drv = sec_usage->drv; - struct wd_ctx_config_internal *config; - struct hisi_sec_ctx *priv; + struct hisi_sec_ctx *sec_ctx; char *ctx_dev_name; - handle_t ctx = 0; handle_t qp = 0; __u32 i; @@ -614,32 +614,27 @@ static int hisi_sec_get_usage(void *param) return -WD_EINVAL; } - priv = (struct hisi_sec_ctx *)drv->priv; - if (!priv) + sec_ctx = (struct hisi_sec_ctx *)drv->drv_data; + if (!sec_ctx) return -WD_EACCES; - config = &priv->config; - for (i = 0; i < config->ctx_num; i++) { - ctx_dev_name = wd_ctx_get_dev_name(config->ctxs[i].ctx); + /* Only process the queues used by your own driver. */ + for (i = 0; i < sec_ctx->ctx_num; i++) { + ctx_dev_name = wd_ctx_get_dev_name(sec_ctx->ctxs[i]->ctx); if (!strcmp(sec_usage->dev_name, ctx_dev_name)) { - ctx = config->ctxs[i].ctx; - break; + qp = (handle_t)wd_ctx_get_priv(sec_ctx->ctxs[i]->ctx); + if (qp) + return hisi_qm_get_usage(qp, 0); } } - if (ctx) - qp = (handle_t)wd_ctx_get_priv(ctx); - - if (qp) - return hisi_qm_get_usage(qp, 0); - return -WD_EACCES; } static int eops_param_check(struct wd_alg_driver *drv, struct wd_mm_ops *mm_ops) { - if (!drv || !drv->priv) { - WD_ERR("invalid: aead drv or priv is NULL!\n"); + if (!drv || !drv->drv_data) { + WD_ERR("invalid: aead drv or data is NULL!\n"); return -WD_EINVAL; } @@ -680,8 +675,8 @@ static int aead_sess_eops_init(struct wd_alg_driver *drv, return -WD_ENOMEM; } - sec_ctx = (struct hisi_sec_ctx *)drv->priv; - qp = (struct hisi_qp *)wd_ctx_get_priv(sec_ctx->config.ctxs[0].ctx); + sec_ctx = (struct hisi_sec_ctx *)drv->drv_data; + qp = (struct hisi_qp *)wd_ctx_get_priv(sec_ctx->ctxs[0]->ctx); sq_depth = qp->q_info.sq_depth; aiv_addr->aiv = mm_ops->alloc(mm_ops->usr, (__u32)sq_depth << AEAD_AIV_OFFSET); if (!aiv_addr->aiv) { @@ -735,8 +730,8 @@ static void aead_sess_eops_uninit(struct wd_alg_driver *drv, return; } - sec_ctx = (struct hisi_sec_ctx *)drv->priv; - qp = (struct hisi_qp *)wd_ctx_get_priv(sec_ctx->config.ctxs[0].ctx); + sec_ctx = (struct hisi_sec_ctx *)drv->drv_data; + qp = (struct hisi_qp *)wd_ctx_get_priv(sec_ctx->ctxs[0]->ctx); sq_depth = qp->q_info.sq_depth; aiv_addr = (struct wd_aead_aiv_addr *)params; @@ -780,7 +775,6 @@ static int sec_aead_get_extend_ops(void *ops) if (!aead_ops) return -WD_EINVAL; - aead_ops->params = NULL; aead_ops->eops_aiv_init = aead_sess_eops_init; aead_ops->eops_aiv_uninit = aead_sess_eops_uninit; @@ -793,15 +787,21 @@ static int sec_aead_get_extend_ops(void *ops) .alg_name = (sec_alg_name),\ .calc_type = UADK_ALG_HW,\ .priority = 100,\ + .priv_size = sizeof(struct hisi_sec_ctx),\ + .ops_size = sizeof(struct wd_aead_extend_ops),\ .queue_num = SEC_CTX_Q_NUM_DEF,\ .op_type_num = 1,\ + .drv_data = NULL,\ .fallback = 0,\ + .init_state = 0,\ .init = hisi_sec_init,\ .exit = hisi_sec_exit,\ .send = alg_type##_send,\ .recv = alg_type##_recv,\ .get_usage = hisi_sec_get_usage,\ .get_extend_ops = sec_aead_get_extend_ops,\ + .alloc_ctx = wd_hw_alloc_ctx, \ + .free_ctx = wd_hw_free_ctx, \ } static struct wd_alg_driver cipher_alg_driver[] = { @@ -812,6 +812,8 @@ static struct wd_alg_driver cipher_alg_driver[] = { GEN_SEC_ALG_DRIVER("cbc(sm4)", cipher), GEN_SEC_ALG_DRIVER("ctr(sm4)", cipher), GEN_SEC_ALG_DRIVER("xts(sm4)", cipher), + GEN_SEC_ALG_DRIVER("xts-gb(sm4)", cipher), + GEN_SEC_ALG_DRIVER("ecb(des)", cipher), GEN_SEC_ALG_DRIVER("cbc(des)", cipher), GEN_SEC_ALG_DRIVER("ecb(des3_ede)", cipher), @@ -1400,7 +1402,7 @@ static int fill_cipher_bd2(struct wd_cipher_msg *msg, struct hisi_sec_sqe *sqe) return 0; } -static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_cipher_send(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *msg = wd_msg; @@ -1456,7 +1458,7 @@ static int hisi_sec_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *w return 0; } -static int hisi_sec_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_cipher_recv(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *recv_msg = wd_msg; @@ -1693,7 +1695,7 @@ static void fill_sec_prefetch(__u8 data_fmt, __u32 len, __u16 hw_type, struct hi sqe->auth_mac_key |= (__u32)SEC_ENABLE_SVA_PREFETCH << SEC_SVA_PREFETCH_OFFSET; } -static int hisi_sec_cipher_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_cipher_send_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -1800,7 +1802,7 @@ static void parse_cipher_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, dump_sec_msg(temp_msg, "cipher"); } -static int hisi_sec_cipher_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_cipher_recv_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_cipher_msg *recv_msg = wd_msg; @@ -2132,7 +2134,7 @@ static int digest_len_check(struct wd_digest_msg *msg, enum sec_bd_type type) return 0; } -static int hisi_sec_digest_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_digest_send(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *msg = wd_msg; @@ -2209,7 +2211,7 @@ put_sgl: return ret; } -static int hisi_sec_digest_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_digest_recv(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *recv_msg = wd_msg; @@ -2473,7 +2475,7 @@ map_err: return -WD_ENOMEM; } -static int hisi_sec_digest_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_digest_send_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -2587,7 +2589,7 @@ static void parse_digest_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, dump_sec_msg(temp_msg, "digest"); } -static int hisi_sec_digest_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_digest_recv_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_digest_msg *recv_msg = wd_msg; @@ -3216,7 +3218,7 @@ static int fill_aead_bd2_addr(struct wd_aead_msg *msg, struct hisi_sec_sqe *sqe, return aead_mem_nosva_map(msg, sqe, idx); } -static int hisi_sec_aead_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_aead_send(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -3347,7 +3349,7 @@ static void parse_aead_bd2(struct hisi_qp *qp, struct hisi_sec_sqe *sqe, dump_sec_msg(temp_msg, "aead"); } -static int hisi_sec_aead_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_aead_recv(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_aead_msg *recv_msg = wd_msg; @@ -3744,7 +3746,7 @@ static int fill_aead_bd3_addr(struct wd_aead_msg *msg, struct hisi_sec_sqe3 *sqe return aead_mem_nosva_map_v3(msg, sqe, idx); } -static int hisi_sec_aead_send_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int hisi_sec_aead_send_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -3863,7 +3865,7 @@ static void parse_aead_bd3(struct hisi_qp *qp, struct hisi_sec_sqe3 *sqe, dump_sec_msg(temp_msg, "aead"); } -static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +int hisi_sec_aead_recv_v3(handle_t ctx, void *wd_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_aead_msg *recv_msg = wd_msg; @@ -3888,73 +3890,105 @@ static int hisi_sec_aead_recv_v3(struct wd_alg_driver *drv, handle_t ctx, void * return 0; } -static int hisi_sec_init(struct wd_alg_driver *drv, void *conf) +static int hisi_sec_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; + struct hisi_sec_ctx *sec_ctx = priv; struct hisi_qm_priv qm_priv; - struct hisi_sec_ctx *priv; - handle_t h_qp = 0; - handle_t h_ctx; - __u32 i, j; + __u32 i, j, count = 0; + bool *is_match; + handle_t h_qp; - if (!config->ctx_num) { - WD_ERR("invalid: sec init config ctx num is 0!\n"); + if (!config || !config->ctx_num) { + WD_ERR("invalid: input config or ctx num is null!\n"); return -WD_EINVAL; } + /* traverse and count the number of contexts supported by this driver. */ + is_match = malloc(config->ctx_num * sizeof(bool)); + if (!is_match) + return -WD_ENOMEM; - priv = malloc(sizeof(struct hisi_sec_ctx)); - if (!priv) + for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx && config->ctxs[i].drv && + !strcmp(config->ctxs[i].drv->drv_name, "hisi_sec2")) { + is_match[i] = true; + count++; + } else { + is_match[i] = false; + } + } + + if (!count) { + WD_ERR("invalid: valid driver number is zero!\n"); + free(is_match); return -WD_EINVAL; + } + sec_ctx->ctxs = calloc(count, sizeof(struct wd_ctx_internal *)); + if (!sec_ctx->ctxs) { + free(is_match); + return -WD_ENOMEM; + } + sec_ctx->ctx_num = count; + + /* allocate QP and store context mirror. */ qm_priv.sqe_size = sizeof(struct hisi_sec_sqe); - /* allocate qp for each context */ + count = 0; for (i = 0; i < config->ctx_num; i++) { - h_ctx = config->ctxs[i].ctx; - /* setting the type is 0 for sqc_type */ + if (!is_match[i]) + continue; + qm_priv.op_type = 0; qm_priv.qp_mode = config->ctxs[i].ctx_mode; - /* Setting the epoll en to 0 for ASYNC ctx */ qm_priv.epoll_en = (qm_priv.qp_mode == CTX_MODE_SYNC) ? config->epoll_en : 0; qm_priv.idx = i; - h_qp = hisi_qm_alloc_qp(&qm_priv, h_ctx); + h_qp = hisi_qm_alloc_qp(&qm_priv, config->ctxs[i].ctx); if (!h_qp) goto out; + config->ctxs[i].sqn = qm_priv.sqn; + /* Store the queues allocated by your own driver. */ + sec_ctx->ctxs[count++] = &config->ctxs[i]; } - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; - return 0; + free(is_match); + + return WD_SUCCESS; out: - for (j = 0; j < i; j++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); + for (j = 0; j < count; j++) { + h_qp = (handle_t)wd_ctx_get_priv(sec_ctx->ctxs[j]->ctx); hisi_qm_free_qp(h_qp); } - free(priv); + free(sec_ctx->ctxs); + free(is_match); return -WD_EINVAL; } -static void hisi_sec_exit(struct wd_alg_driver *drv) +static void hisi_sec_exit(void *priv) { - struct wd_ctx_config_internal *config; - struct hisi_sec_ctx *priv; + struct hisi_sec_ctx *sec_ctx = priv; handle_t h_qp; __u32 i; - if (!drv || !drv->priv) + if (!priv) { + WD_ERR("invalid: input parameter is NULL!\n"); return; + } - priv = (struct hisi_sec_ctx *)drv->priv; - config = &priv->config; - - for (i = 0; i < config->ctx_num; i++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); + /* Only release the queues allocated by your own driver. */ + for (i = 0; i < sec_ctx->ctx_num; i++) { + h_qp = (handle_t)wd_ctx_get_priv(sec_ctx->ctxs[i]->ctx); + if (!h_qp) + continue; hisi_qm_free_qp(h_qp); } - free(priv); - drv->priv = NULL; + + if (sec_ctx->ctxs) { + free(sec_ctx->ctxs); + sec_ctx->ctxs = NULL; + } } #ifdef WD_STATIC_DRV diff --git a/drv/isa_ce_sm4.c b/drv/isa_ce_sm4.c index 52dca1f..863b99e 100644 --- a/drv/isa_ce_sm4.c +++ b/drv/isa_ce_sm4.c @@ -12,14 +12,17 @@ */ #include "drv/wd_cipher_drv.h" -#include "wd_cipher.h" #include "isa_ce_sm4.h" +#include "wd_cipher.h" +#include "wd_drv.h" #define SM4_ENCRYPT 1 #define SM4_DECRYPT 0 #define MSG_Q_DEPTH 1024 #define INCREASE_BYTES 12 #define SM4_BLOCK_SIZE 16 +/* CTS tail: last full block + partial block */ +#define SM4_CTS_TAIL_SIZE 32 #define MAX_BLOCK_NUM (1U << 28) #define CTR96_SHIFT_BITS 8 #define SM4_BYTES2BLKS(nbytes) ((nbytes) >> 4) @@ -31,36 +34,23 @@ ((p)[0] = (__u8)((v) >> 24), (p)[1] = (__u8)((v) >> 16), \ (p)[2] = (__u8)((v) >> 8), (p)[3] = (__u8)(v)) -static int isa_ce_init(struct wd_alg_driver *drv, void *conf) +static int isa_ce_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; - struct sm4_ce_drv_ctx *priv; + struct sm4_ce_drv_ctx *sctx = priv; /* Fallback init is NULL */ - if (!drv || !conf) + if (!conf || !priv) return 0; - priv = malloc(sizeof(struct sm4_ce_drv_ctx)); - if (!priv) - return -WD_EINVAL; - config->epoll_en = 0; - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + memcpy(&sctx->config, config, sizeof(struct wd_ctx_config_internal)); - return WD_SUCCESS; + return 0; } -static void isa_ce_exit(struct wd_alg_driver *drv) +static void isa_ce_exit(void *priv) { - struct sm4_ce_drv_ctx *sctx; - - if (!drv || !drv->priv) - return; - - sctx = (struct sm4_ce_drv_ctx *)drv->priv; - free(sctx); - drv->priv = NULL; } /* increment upper 96 bits of 128-bit counter by 1 */ @@ -179,6 +169,8 @@ static void sm4_cts_cs1_mode_adapt(__u8 *cts_in, __u8 *cts_out, static void sm4_cts_cbc_crypt(struct wd_cipher_msg *msg, const struct SM4_KEY *rkey_enc, const int enc) { + /* Stack buffer for CS1 decrypt to avoid modifying caller's msg->in */ + __u8 cts_in_buf[SM4_CTS_TAIL_SIZE] = {0}; enum wd_cipher_mode mode = msg->mode; __u32 in_bytes = msg->in_bytes; __u8 *cts_in, *cts_out; @@ -204,10 +196,21 @@ static void sm4_cts_cbc_crypt(struct wd_cipher_msg *msg, if (mode == WD_CIPHER_CBC_CS1) sm4_cts_cs1_mode_adapt(cts_in, cts_out, cts_bytes, enc); } else { - if (mode == WD_CIPHER_CBC_CS1) - sm4_cts_cs1_mode_adapt(cts_in, cts_out, cts_bytes, enc); - - sm4_v8_cbc_cts_decrypt(cts_in, cts_out, cts_bytes, rkey_enc, msg->iv); + if (mode == WD_CIPHER_CBC_CS1) { + /* + * CS1 decrypt: sm4_cts_cs1_mode_adapt swaps the CTS tail + * from CS1 layout to CS3 layout so sm4_v8_cbc_cts_decrypt + * can process it. Copy to a stack buffer first to avoid + * modifying the caller's input buffer (msg->in), which + * breaks verification tools that re-encrypt the decrypted + * output and compare against the original ciphertext. + */ + memcpy(cts_in_buf, cts_in, cts_bytes); + sm4_cts_cs1_mode_adapt(cts_in_buf, cts_out, cts_bytes, enc); + sm4_v8_cbc_cts_decrypt(cts_in_buf, cts_out, cts_bytes, rkey_enc, msg->iv); + } else { + sm4_v8_cbc_cts_decrypt(cts_in, cts_out, cts_bytes, rkey_enc, msg->iv); + } } } @@ -334,17 +337,22 @@ static int sm4_xts_decrypt(struct wd_cipher_msg *msg, const struct SM4_KEY *rkey return 0; } -static int isa_ce_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int isa_ce_cipher_send(handle_t ctx, void *wd_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; struct wd_cipher_msg *msg = wd_msg; struct SM4_KEY rkey; int ret = 0; - if (!msg) { + if (!msg || !ctx) { WD_ERR("invalid: input sm4 msg is NULL!\n"); return -WD_EINVAL; } + ret = wd_queue_is_busy(sfctx); + if (ret) + return ret; + if (msg->data_fmt == WD_SGL_BUF) { WD_ERR("invalid: SM4 CE driver do not support sgl data format!\n"); return -WD_EINVAL; @@ -397,22 +405,34 @@ static int isa_ce_cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *wd_ return -WD_EINVAL; } + ret = wd_get_sqe_from_queue(sfctx, msg->tag); + if (ret) + return ret; + return ret; } -static int isa_ce_cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *wd_msg) +static int isa_ce_cipher_recv(handle_t ctx, void *wd_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; + struct wd_cipher_msg *msg = wd_msg; + int ret; + + ret = wd_put_sqe_to_queue(sfctx, &msg->tag, &msg->result); + if (ret) + return ret; + return 0; } -static int cipher_send(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_send(handle_t ctx, void *msg) { - return isa_ce_cipher_send(drv, ctx, msg); + return isa_ce_cipher_send(ctx, msg); } -static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) +static int cipher_recv(handle_t ctx, void *msg) { - return isa_ce_cipher_recv(drv, ctx, msg); + return isa_ce_cipher_recv(ctx, msg); } #define GEN_CE_ALG_DRIVER(ce_alg_name, alg_type) \ @@ -421,12 +441,16 @@ static int cipher_recv(struct wd_alg_driver *drv, handle_t ctx, void *msg) .alg_name = (ce_alg_name),\ .calc_type = UADK_ALG_CE_INSTR,\ .priority = 200,\ + .priv_size = sizeof(struct sm4_ce_drv_ctx),\ + .queue_num = 1,\ .op_type_num = 1,\ .fallback = 0,\ .init = isa_ce_init,\ .exit = isa_ce_exit,\ .send = alg_type##_send,\ .recv = alg_type##_recv,\ + .alloc_ctx = wd_soft_alloc_ctx, \ + .free_ctx = wd_soft_free_ctx, \ } static struct wd_alg_driver cipher_alg_driver[] = { diff --git a/wd_cipher.c b/wd_cipher.c index 3bfead8..e0670f5 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -52,7 +52,6 @@ struct wd_cipher_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; } wd_cipher_setting; @@ -83,20 +82,16 @@ static void wd_cipher_close_driver(int init_type) } if (wd_cipher_setting.dlhandle) { - wd_release_drv(wd_cipher_setting.driver); dlclose(wd_cipher_setting.dlhandle); wd_cipher_setting.dlhandle = NULL; } #else - wd_release_drv(wd_cipher_setting.driver); hisi_sec2_remove(); #endif } static int wd_cipher_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "cbc(aes)"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -130,15 +125,6 @@ static int wd_cipher_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_cipher_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_cipher_setting.driver = driver; - return WD_SUCCESS; } @@ -201,7 +187,7 @@ static int cipher_key_len_check(struct wd_cipher_sess *sess, __u32 length) ret = -WD_EINVAL; break; default: - WD_ERR("cipher input alg err, alg = %u\n", sess->alg); + WD_ERR("invalid: unsupported cipher input alg, alg = %u\n", sess->alg); return -WD_EINVAL; } @@ -260,7 +246,7 @@ static int cipher_setup_memory_and_buffers(struct wd_cipher_sess *sess, ret = wd_mem_ops_init(wd_cipher_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type); if (ret) { - WD_ERR("cipher failed to init memory ops!\n"); + WD_ERR("failed to init cipher memory ops!\n"); return ret; } @@ -269,7 +255,7 @@ static int cipher_setup_memory_and_buffers(struct wd_cipher_sess *sess, sess->key = sess->mm_ops.alloc(sess->mm_ops.usr, MAX_CIPHER_KEY_SIZE); if (!sess->key) { - WD_ERR("cipher failed to alloc key memory!\n"); + WD_ERR("failed to alloc cipher key memory!\n"); return -WD_ENOMEM; } memset(sess->key, 0, MAX_CIPHER_KEY_SIZE); @@ -280,6 +266,7 @@ static int cipher_setup_memory_and_buffers(struct wd_cipher_sess *sess, handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) { struct wd_cipher_sess *sess = NULL; + struct wd_sched_params params; bool ret; if (unlikely(!setup)) { @@ -295,13 +282,13 @@ handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) memset(sess, 0, sizeof(struct wd_cipher_sess)); if (setup->alg >= WD_CIPHER_ALG_TYPE_MAX || - setup->mode >= WD_CIPHER_MODE_TYPE_MAX) { + setup->mode >= WD_CIPHER_MODE_TYPE_MAX) { WD_ERR("failed to check algorithm!\n"); goto free_sess; } sess->alg_name = wd_cipher_alg_name[setup->alg][setup->mode]; - ret = wd_drv_alg_support(sess->alg_name, wd_cipher_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, &wd_cipher_setting.config); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); goto free_sess; @@ -321,6 +308,13 @@ handle_t wd_cipher_alloc_sess(struct wd_cipher_sess_setup *setup) goto free_key; } + /* Set compat filtering parameters for session-ctx matching */ + memset(¶ms, 0, sizeof(params)); + params.alg_name = sess->alg_name; + params.ctxs = wd_cipher_setting.config.ctxs; + wd_cipher_setting.sched.set_param(wd_cipher_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); + return (handle_t)sess; free_key: @@ -342,11 +336,19 @@ void wd_cipher_free_sess(handle_t h_sess) wd_memset_zero(sess->key, sess->key_bytes); sess->mm_ops.free(sess->mm_ops.usr, sess->key); - if (sess->sched_key) - free(sess->sched_key); + if (sess->sched_key) { + if (wd_cipher_setting.sched.sched_uninit) + wd_cipher_setting.sched.sched_uninit( + wd_cipher_setting.sched.h_sched_ctx, + (handle_t)sess->sched_key); + else + free(sess->sched_key); + } free(sess); } +static bool wd_cipher_atfork_registered; + static void wd_cipher_clear_status(void) { wd_alg_clear_init(&wd_cipher_setting.status); @@ -378,15 +380,8 @@ static int wd_cipher_common_init(struct wd_ctx_config *config, if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver); - if (ret) - goto out_clear_pool; - return 0; -out_clear_pool: - wd_uninit_async_request_pool(&wd_cipher_setting.pool); out_clear_sched: wd_clear_sched(&wd_cipher_setting.sched); out_clear_ctx_config: @@ -394,31 +389,24 @@ out_clear_ctx_config: return ret; } -static int wd_cipher_common_uninit(void) +static void wd_cipher_common_uninit(void) { - enum wd_status status; - - wd_alg_get_init(&wd_cipher_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - /* uninit async request pool */ wd_uninit_async_request_pool(&wd_cipher_setting.pool); /* unset config, sched, driver */ wd_clear_sched(&wd_cipher_setting.sched); - - wd_alg_uninit_driver(&wd_cipher_setting.config, - wd_cipher_setting.driver); - - return 0; } int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) { + __u32 drv_count; int ret; - pthread_atfork(NULL, NULL, wd_cipher_clear_status); + if (!wd_cipher_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_cipher_clear_status) == 0) + wd_cipher_atfork_registered = true; + } ret = wd_alg_try_init(&wd_cipher_setting.status); if (ret) @@ -428,18 +416,58 @@ int wd_cipher_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + /* init1 path is HW-only; CE/SVE drivers require init2 */ + if (sched->sched_policy == SCHED_POLICY_NONE || + sched->sched_policy == SCHED_POLICY_SINGLE) { + WD_ERR("init1 does not support NONE/SINGLE schedulers, use init2\n"); + ret = -WD_EINVAL; + goto out_clear_init; + } + ret = wd_cipher_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; + /* Internal copy (existing common_init) */ ret = wd_cipher_common_init(config, sched); if (ret) goto out_close_driver; + /* Driver discovery */ + ret = wd_get_drv_array("cipher", TASK_HW, "hisi_sec2", + &wd_cipher_setting.config.drv_array, &drv_count); + if (ret) { + WD_ERR("failed to get driver array!\n"); + goto out_common_uninit; + } + + /* RR bind drivers to internal ctxs */ + wd_cipher_setting.config.drv_count = drv_count; + ret = wd_ctx_bind_drivers(&wd_cipher_setting.config, NULL, WD_TYPE_V1); + if (ret) { + WD_ERR("failed to bind driver!\n"); + goto out_free_drv_array; + } + + /* Driver initialization */ + ret = wd_alg_init_driver(&wd_cipher_setting.config); + if (ret) { + WD_ERR("failed to init cipher driver!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_cipher_setting.status); return 0; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_cipher_setting.config); +out_free_drv_array: + wd_put_drv_array(wd_cipher_setting.config.drv_array, drv_count); + wd_cipher_setting.config.drv_array = NULL; + wd_cipher_setting.config.drv_count = 0; +out_common_uninit: + wd_cipher_common_uninit(); out_close_driver: wd_cipher_close_driver(WD_TYPE_V1); out_clear_init: @@ -449,24 +477,37 @@ out_clear_init: void wd_cipher_uninit(void) { - int ret; + enum wd_status status; - ret = wd_cipher_common_uninit(); - if (ret) + wd_alg_get_init(&wd_cipher_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_cipher_setting.config); + wd_ctx_unbind_drivers(&wd_cipher_setting.config); + wd_put_drv_array(wd_cipher_setting.config.drv_array, + wd_cipher_setting.config.drv_count); + wd_cipher_setting.config.drv_array = NULL; + wd_cipher_setting.config.drv_count = 0; + + wd_cipher_common_uninit(); + wd_cipher_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_cipher_setting.status); } -int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params *ctx_params) +int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, + struct wd_ctx_params *ctx_params) { struct wd_ctx_nums cipher_ctx_num[WD_CIPHER_DECRYPTION + 1] = {0}; struct wd_ctx_params cipher_ctx_params = {0}; int state, ret = -WD_EINVAL; bool flag; - pthread_atfork(NULL, NULL, wd_cipher_clear_status); + if (!wd_cipher_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_cipher_clear_status) == 0) + wd_cipher_atfork_registered = true; + } state = wd_alg_try_init(&wd_cipher_setting.status); if (state) @@ -489,39 +530,30 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p goto out_uninit; while (ret != 0) { - memset(&wd_cipher_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - - /* Get alg driver and dev name */ - wd_cipher_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_cipher_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } + memset(&wd_cipher_setting.config, 0, + sizeof(struct wd_ctx_config_internal)); + /* Init ctx param and prepare for ctx request */ cipher_ctx_params.ctx_set_num = cipher_ctx_num; ret = wd_ctx_param_init(&cipher_ctx_params, ctx_params, - wd_cipher_setting.driver, - WD_CIPHER_TYPE, WD_CIPHER_DECRYPTION + 1); + alg, WD_CIPHER_TYPE, + WD_CIPHER_DECRYPTION + 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_cipher_setting.driver); - wd_alg_drv_unbind(wd_cipher_setting.driver); + if (ret == -WD_EAGAIN) continue; - } - goto out_driver; + goto out_dlclose; } (void)strcpy(wd_cipher_init_attrs.alg, alg); wd_cipher_init_attrs.sched_type = sched_type; - wd_cipher_init_attrs.driver = wd_cipher_setting.driver; + wd_cipher_init_attrs.task_type = task_type; wd_cipher_init_attrs.ctx_params = &cipher_ctx_params; wd_cipher_init_attrs.alg_init = wd_cipher_common_init; wd_cipher_init_attrs.alg_poll_ctx = wd_cipher_poll_ctx; + ret = wd_alg_attrs_init(&wd_cipher_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_cipher_setting.driver); - wd_alg_drv_unbind(wd_cipher_setting.driver); wd_ctx_param_uninit(&cipher_ctx_params); continue; } @@ -530,16 +562,35 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_p } } + /* RR bind drivers */ + ret = wd_ctx_bind_drivers(&wd_cipher_setting.config, + wd_cipher_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("failed to bind driver!\n"); + goto out_common_uninit; + } + + /* Driver initialization */ + ret = wd_alg_init_driver(&wd_cipher_setting.config); + if (ret) { + WD_ERR("failed to init driver!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_cipher_setting.status); wd_ctx_param_uninit(&cipher_ctx_params); return 0; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_cipher_setting.config); +out_common_uninit: + wd_cipher_common_uninit(); + wd_alg_attrs_uninit(&wd_cipher_init_attrs); out_params_uninit: wd_ctx_param_uninit(&cipher_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_cipher_setting.driver); -out_dlopen: +out_dlclose: wd_cipher_close_driver(WD_TYPE_V2); out_uninit: wd_alg_clear_init(&wd_cipher_setting.status); @@ -548,14 +599,20 @@ out_uninit: void wd_cipher_uninit2(void) { - int ret; + enum wd_status status; - ret = wd_cipher_common_uninit(); - if (ret) + wd_alg_get_init(&wd_cipher_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_cipher_setting.config); + wd_ctx_unbind_drivers(&wd_cipher_setting.config); + wd_cipher_setting.config.drv_array = NULL; + wd_cipher_setting.config.drv_count = 0; + wd_cipher_common_uninit(); + wd_alg_attrs_uninit(&wd_cipher_init_attrs); - wd_alg_drv_unbind(wd_cipher_setting.driver); + wd_cipher_close_driver(WD_TYPE_V2); wd_alg_clear_init(&wd_cipher_setting.status); } @@ -716,13 +773,13 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_msg_handle msg_handle; int ret; - msg_handle.send = wd_cipher_setting.driver->send; - msg_handle.recv = wd_cipher_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; - wd_ctx_spin_lock(ctx, wd_cipher_setting.driver->calc_type); - ret = wd_handle_msg_sync(wd_cipher_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_cipher_setting.config.epoll_en); - wd_ctx_spin_unlock(ctx, wd_cipher_setting.driver->calc_type); + wd_ctx_spin_lock(ctx, ctx->ctx_type); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, NULL, + wd_cipher_setting.config.epoll_en); + wd_ctx_spin_unlock(ctx, ctx->ctx_type); return ret; } @@ -788,15 +845,13 @@ int wd_do_cipher_async(handle_t h_sess, struct wd_cipher_req *req) msg_id = wd_get_msg_from_pool(&wd_cipher_setting.pool, idx, (void **)&msg); - if (unlikely(msg_id < 0)) { - WD_ERR("failed to get msg from pool!\n"); - return msg_id; - } + if (unlikely(msg_id < 0)) + return -WD_EBUSY; fill_request_msg(msg, req, sess); msg->tag = msg_id; - ret = wd_alg_driver_send(wd_cipher_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("wd cipher async send err!\n"); @@ -843,7 +898,7 @@ int wd_cipher_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_cipher_setting.driver, ctx->ctx, &resp_msg); + ret = ctx->drv->recv(ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) return ret; else if (ret < 0) { -- 2.43.0
From: Zhushuai Yin <yinzhushuai@huawei.com> This modification enables the UADK AEAD algorithm to support the new heterogeneous scheduling framework by adapting the corresponding user-space driver to the new wd_alg_driver registration framework. Specifically, wd_aead.c is updated to replace the old binding model with wd_get_drv_array/wd_ctx_bind_drivers, and the wd_aead_sess priv field is changed from void* to void** to accommodate per-context private data. Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- wd_aead.c | 166 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 109 insertions(+), 57 deletions(-) diff --git a/wd_aead.c b/wd_aead.c index eb3597e..c0983a6 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -630,6 +630,8 @@ static void wd_aead_clear_status(void) wd_alg_clear_init(&wd_aead_setting.status); } +static bool wd_aead_atfork_registered; + static int wd_aead_init_nolock(struct wd_ctx_config *config, struct wd_sched *sched) { int ret; @@ -674,9 +676,20 @@ out_clear_ctx_config: int wd_aead_init(struct wd_ctx_config *config, struct wd_sched *sched) { + __u32 drv_count; int ret; - pthread_atfork(NULL, NULL, wd_aead_clear_status); + if (!wd_aead_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_aead_clear_status) == 0) + wd_aead_atfork_registered = true; + } + + /* init1 path is HW-only; CE/SVE drivers require init2 */ + if (sched->sched_policy == SCHED_POLICY_NONE || + sched->sched_policy == SCHED_POLICY_SINGLE) { + WD_ERR("init1 does not support NONE/SINGLE schedulers, use init2\n"); + return -WD_EINVAL; + } ret = wd_alg_try_init(&wd_aead_setting.status); if (ret) @@ -690,14 +703,40 @@ int wd_aead_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; - ret = wd_aead_init_nolock(config, sched); - if (ret) - goto out_close_driver; + /* Driver discovery */ + ret = wd_get_drv_array("aead", TASK_HW, "hisi_sec2", + &wd_aead_setting.config.drv_array, &drv_count); + if (ret) { + WD_ERR("failed to get driver array!\n"); + goto out_common_uninit; + } + + /* RR bind drivers to internal ctxs */ + wd_aead_setting.config.drv_count = drv_count; + ret = wd_ctx_bind_drivers(&wd_aead_setting.config, NULL, WD_TYPE_V1); + if (ret) { + WD_ERR("failed to bind drivers!\n"); + goto out_free_drv_array; + } + + ret = wd_alg_init_driver(&wd_aead_setting.config); + if (ret) { + WD_ERR("failed to init aead driver!\n"); + goto out_unbind_drivers; + } wd_alg_set_init(&wd_aead_setting.status); - return 0; + return ret; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_aead_setting.config); +out_free_drv_array: + wd_put_drv_array(wd_aead_setting.config.drv_array, drv_count); + wd_aead_setting.config.drv_array = NULL; + wd_aead_setting.config.drv_count = 0; +out_common_uninit: + wd_aead_uninit_nolock(); out_close_driver: wd_aead_close_driver(WD_TYPE_V1); out_clear_init: @@ -705,30 +744,27 @@ out_clear_init: return ret; } -static int wd_aead_uninit_nolock(void) +static void wd_aead_uninit_nolock(void) { - enum wd_status status; - - wd_alg_get_init(&wd_aead_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - wd_uninit_async_request_pool(&wd_aead_setting.pool); wd_clear_sched(&wd_aead_setting.sched); - wd_alg_uninit_driver(&wd_aead_setting.config, - wd_aead_setting.driver); - - return 0; } void wd_aead_uninit(void) { - int ret; + enum wd_status status; - ret = wd_aead_uninit_nolock(); - if (ret) + wd_alg_get_init(&wd_aead_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_aead_setting.config); + wd_ctx_unbind_drivers(&wd_aead_setting.config); + wd_put_drv_array(wd_aead_setting.config.drv_array, + wd_aead_setting.config.drv_count); + wd_aead_setting.config.drv_array = NULL; + wd_aead_setting.config.drv_count = 0; + wd_aead_uninit_nolock(); wd_aead_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_aead_setting.status); } @@ -753,12 +789,16 @@ int wd_aead_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_nums aead_ctx_num[WD_DIGEST_CIPHER_DECRYPTION + 1] = {0}; struct wd_ctx_params aead_ctx_params = {0}; int state, ret = -WD_EINVAL; + int try_cnt = 0; - pthread_atfork(NULL, NULL, wd_aead_clear_status); + if (!wd_aead_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_aead_clear_status) == 0) + wd_aead_atfork_registered = true; + } state = wd_alg_try_init(&wd_aead_setting.status); if (state) - return state; + goto out_uninit; if (!alg || sched_type >= SCHED_POLICY_BUTT || task_type < 0 || task_type >= TASK_MAX_TYPE) { @@ -777,38 +817,26 @@ int wd_aead_init2_(char *alg, __u32 sched_type, int task_type, while (ret != 0) { memset(&wd_aead_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - - /* Get alg driver and dev name */ - wd_aead_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_aead_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } - + /* Init ctx param and prepare for ctx request */ aead_ctx_params.ctx_set_num = aead_ctx_num; ret = wd_ctx_param_init(&aead_ctx_params, ctx_params, - wd_aead_setting.driver, WD_AEAD_TYPE, + alg, WD_AEAD_TYPE, WD_DIGEST_CIPHER_DECRYPTION + 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_aead_setting.driver); - wd_alg_drv_unbind(wd_aead_setting.driver); + if (ret == -WD_EAGAIN) continue; - } - goto out_driver; + goto out_dlclose; } (void)strcpy(wd_aead_init_attrs.alg, alg); wd_aead_init_attrs.sched_type = sched_type; - wd_aead_init_attrs.driver = wd_aead_setting.driver; + wd_aead_init_attrs.task_type = task_type; wd_aead_init_attrs.ctx_params = &aead_ctx_params; wd_aead_init_attrs.alg_init = wd_aead_init_nolock; wd_aead_init_attrs.alg_poll_ctx = wd_aead_poll_ctx; ret = wd_alg_attrs_init(&wd_aead_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_aead_setting.driver); - wd_alg_drv_unbind(wd_aead_setting.driver); wd_ctx_param_uninit(&aead_ctx_params); continue; } @@ -816,16 +844,36 @@ int wd_aead_init2_(char *alg, __u32 sched_type, int task_type, goto out_params_uninit; } } + + /* RR bind drivers */ + ret = wd_ctx_bind_drivers(&wd_aead_setting.config, + wd_aead_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("failed to bind driver!\n"); + goto out_common_uninit; + } + + /* Driver initialization */ + ret = wd_alg_init_driver(&wd_aead_setting.config); + if (ret) { + WD_ERR("failed to init driver!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_aead_setting.status); wd_ctx_param_uninit(&aead_ctx_params); - return 0; + return ret; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_aead_setting.config); +out_common_uninit: + wd_aead_uninit_nolock(); + wd_alg_attrs_uninit(&wd_aead_init_attrs); out_params_uninit: wd_ctx_param_uninit(&aead_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_aead_setting.driver); -out_dlopen: +out_dlclose: wd_aead_close_driver(WD_TYPE_V2); out_uninit: wd_alg_clear_init(&wd_aead_setting.status); @@ -834,14 +882,17 @@ out_uninit: void wd_aead_uninit2(void) { - int ret; + enum wd_status status; - ret = wd_aead_uninit_nolock(); - if (ret) + wd_alg_get_init(&wd_aead_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_aead_setting.config); + wd_ctx_unbind_drivers(&wd_aead_setting.config); + wd_aead_uninit_nolock(); + wd_alg_attrs_uninit(&wd_aead_init_attrs); - wd_alg_drv_unbind(wd_aead_setting.driver); wd_aead_close_driver(WD_TYPE_V2); wd_alg_clear_init(&wd_aead_setting.status); } @@ -914,7 +965,6 @@ static void fill_request_msg(struct wd_aead_msg *msg, struct wd_aead_req *req, msg->mm_ops = &sess->mm_ops; msg->mm_type = sess->mm_type; - msg->drv_cfg = sess->eops.params; fill_stream_msg(msg, req, sess); } @@ -924,12 +974,12 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_msg_handle msg_handle; int ret; - msg_handle.send = wd_aead_setting.driver->send; - msg_handle.recv = wd_aead_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_aead_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_aead_setting.config.epoll_en); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, NULL, + wd_aead_setting.config.epoll_en); pthread_spin_unlock(&ctx->lock); return ret; @@ -959,6 +1009,9 @@ int wd_do_aead_sync(handle_t h_sess, struct wd_aead_req *req) if (unlikely(ret)) return ret; + /* Set drv_cfg from priv array like wd_agg */ + msg.drv_cfg = sess->priv[idx]; + wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); ctx = config->ctxs + idx; ret = send_recv_sync(ctx, &msg); @@ -996,15 +1049,14 @@ int wd_do_aead_async(handle_t h_sess, struct wd_aead_req *req) msg_id = wd_get_msg_from_pool(&wd_aead_setting.pool, idx, (void **)&msg); - if (unlikely(msg_id < 0)) { - WD_ERR("failed to get msg from pool!\n"); - return msg_id; - } + if (unlikely(msg_id < 0)) + return -WD_EBUSY; fill_request_msg(msg, req, sess); msg->tag = msg_id; + msg->drv_cfg = sess->priv[idx]; - ret = wd_alg_driver_send(wd_aead_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("failed to send BD, hw is err!\n"); @@ -1051,7 +1103,7 @@ int wd_aead_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_aead_setting.driver, ctx->ctx, &resp_msg); + ret = ctx->drv->recv(ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { -- 2.43.0
From: lizhi <lizhi206@huawei.com> In the updated UADK framework, for asymmetric encryption algorithms such as RSA, DH, and ECC, the API layer needs to be adapted to the new UADK heterogeneous hybrid acceleration framework to ensure these algorithms can achieve heterogeneous hybrid acceleration. Signed-off-by: lizhi <lizhi206@huawei.com> Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- drv/hisi_hpre.c | 213 ++++++++++++++----------- drv/wd_drv.c | 19 ++- include/drv/wd_ecc_drv.h | 7 +- wd_dh.c | 186 +++++++++++++--------- wd_ecc.c | 325 +++++++++++++++++++++++++++------------ wd_rsa.c | 192 ++++++++++++++--------- 6 files changed, 599 insertions(+), 343 deletions(-) diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c index fae8315..9e46d77 100644 --- a/drv/hisi_hpre.c +++ b/drv/hisi_hpre.c @@ -10,6 +10,7 @@ #include <sys/mman.h> #include <sys/types.h> #include "hisi_qm_udrv.h" +#include "wd_drv.h" #include "../include/wd_ecc_curve.h" #include "../include/drv/wd_rsa_drv.h" #include "../include/drv/wd_dh_drv.h" @@ -130,12 +131,13 @@ struct hisi_hpre_sqe { }; struct hisi_hpre_ctx { - struct wd_ctx_config_internal config; + struct wd_ctx_internal **ctxs; + __u32 ctx_num; struct wd_mm_ops *mm_ops; handle_t rsv_mem_ctx; }; -struct hpre_ecc_ctx { +struct hisi_hpre_eops_ctx { __u32 enable_hpcore; }; @@ -647,44 +649,76 @@ static int hpre_init_qm_priv(struct wd_ctx_config_internal *config, struct hisi_hpre_ctx *hpre_ctx, struct hisi_qm_priv *qm_priv) { - handle_t h_ctx, h_qp; - __u32 i, j; + __u32 i, j, count; + bool *is_match; + handle_t h_qp; - memcpy(&hpre_ctx->config, config, sizeof(*config)); + /* First pass: traverse and count the number of contexts supported by this driver. */ + is_match = malloc(config->ctx_num * sizeof(bool)); + if (!is_match) + return -WD_ENOMEM; - /* allocate qp for each context */ - qm_priv->sqe_size = sizeof(struct hisi_hpre_sqe); + count = 0; + for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx && + strcmp(config->ctxs[i].drv->drv_name, "hisi_hpre") == 0) { + is_match[i] = true; + count++; + } else { + is_match[i] = false; + } + } + if (!count) { + free(is_match); + return -WD_EINVAL; + } + + hpre_ctx->ctxs = calloc(count, sizeof(struct wd_ctx_internal *)); + if (!hpre_ctx->ctxs) { + free(is_match); + return -WD_ENOMEM; + } + hpre_ctx->ctx_num = count; + + /* Second pass: allocate QP and store context mirror. */ + qm_priv->sqe_size = sizeof(struct hisi_hpre_sqe); + count = 0; for (i = 0; i < config->ctx_num; i++) { - h_ctx = config->ctxs[i].ctx; + if (!is_match[i]) + continue; + qm_priv->qp_mode = config->ctxs[i].ctx_mode; - /* Setting the epoll en to 0 for ASYNC ctx */ qm_priv->epoll_en = (qm_priv->qp_mode == CTX_MODE_SYNC) ? - config->epoll_en : 0; + config->epoll_en : 0; qm_priv->idx = i; - h_qp = hisi_qm_alloc_qp(qm_priv, h_ctx); - if (!h_qp) { - WD_ERR("failed to alloc qp!\n"); + h_qp = hisi_qm_alloc_qp(qm_priv, config->ctxs[i].ctx); + if (!h_qp) goto out; - } + config->ctxs[i].sqn = qm_priv->sqn; + /* Store the queues allocated by your own driver. */ + hpre_ctx->ctxs[count++] = &config->ctxs[i]; } + free(is_match); return WD_SUCCESS; + out: - for (j = 0; j < i; j++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); + for (j = 0; j < count; j++) { + h_qp = (handle_t)wd_ctx_get_priv(hpre_ctx->ctxs[j]->ctx); hisi_qm_free_qp(h_qp); } - + free(hpre_ctx->ctxs); + free(is_match); return -WD_EINVAL; } -static int hpre_rsa_dh_init(struct wd_alg_driver *drv, void *conf) +static int hpre_rsa_dh_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = (struct wd_ctx_config_internal *)conf; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)priv; struct hisi_qm_priv qm_priv; - struct hisi_hpre_ctx *priv; int ret; if (!config->ctx_num) { @@ -692,27 +726,19 @@ static int hpre_rsa_dh_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; } - priv = malloc(sizeof(struct hisi_hpre_ctx)); - if (!priv) - return -WD_EINVAL; - qm_priv.op_type = HPRE_HW_V2_ALG_TYPE; - ret = hpre_init_qm_priv(config, priv, &qm_priv); - if (ret) { - free(priv); + ret = hpre_init_qm_priv(config, hpre_ctx, &qm_priv); + if (ret) return ret; - } - - drv->priv = priv; return WD_SUCCESS; } -static int hpre_ecc_init(struct wd_alg_driver *drv, void *conf) +static int hpre_ecc_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = (struct wd_ctx_config_internal *)conf; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)priv; struct hisi_qm_priv qm_priv; - struct hisi_hpre_ctx *priv; int ret; if (!config->ctx_num) { @@ -720,44 +746,33 @@ static int hpre_ecc_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; } - priv = malloc(sizeof(struct hisi_hpre_ctx)); - if (!priv) - return -WD_EINVAL; - qm_priv.op_type = HPRE_HW_V3_ECC_ALG_TYPE; - ret = hpre_init_qm_priv(config, priv, &qm_priv); - if (ret) { - free(priv); + ret = hpre_init_qm_priv(config, hpre_ctx, &qm_priv); + if (ret) return ret; - } - - drv->priv = priv; return WD_SUCCESS; } -static void hpre_exit(struct wd_alg_driver *drv) +static void hpre_exit(void *priv) { - struct wd_ctx_config_internal *config; - struct hisi_hpre_ctx *priv; + struct hisi_hpre_ctx *hpre_ctx = (struct hisi_hpre_ctx *)priv; handle_t h_qp; __u32 i; - if (!drv || !drv->priv) - return; - - priv = (struct hisi_hpre_ctx *)drv->priv; - config = &priv->config; - for (i = 0; i < config->ctx_num; i++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); + /* Only release the queues allocated by your own driver. */ + for (i = 0; i < hpre_ctx->ctx_num; i++) { + h_qp = (handle_t)wd_ctx_get_priv(hpre_ctx->ctxs[i]->ctx); hisi_qm_free_qp(h_qp); } - free(priv); - drv->priv = NULL; + if (hpre_ctx->ctxs) { + free(hpre_ctx->ctxs); + hpre_ctx->ctxs = NULL; + } } -static int rsa_send(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) +static int rsa_send(handle_t ctx, void *rsa_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_rsa_msg *msg = rsa_msg; @@ -829,7 +844,7 @@ static void hpre_result_check(struct hisi_hpre_sqe *hw_msg, } } -static int rsa_recv(struct wd_alg_driver *drv, handle_t ctx, void *rsa_msg) +static int rsa_recv(handle_t ctx, void *rsa_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -949,7 +964,7 @@ static int dh_out_transfer(struct wd_dh_msg *msg, struct hisi_hpre_sqe *hw_msg, return WD_SUCCESS; } -static int dh_send(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) +static int dh_send(handle_t ctx, void *dh_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct map_info_cache cache = {0}; @@ -1020,7 +1035,7 @@ dh_fail: return ret; } -static int dh_recv(struct wd_alg_driver *drv, handle_t ctx, void *dh_msg) +static int dh_recv(handle_t ctx, void *dh_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -1168,7 +1183,7 @@ static bool big_than_one(const char *data, __u32 data_sz) static bool less_than_latter(struct wd_dtb *d, struct wd_dtb *n) { - unsigned char *d_data, *n_data; + char *d_data, *n_data; __u32 shift, i; if (d->dsize > n->dsize) @@ -1608,7 +1623,7 @@ static int u_is_in_p(struct wd_ecc_msg *msg) static int ecc_prepare_in(struct wd_ecc_msg *msg, struct hisi_hpre_sqe *hw_msg, void **data) { - struct hpre_ecc_ctx *ecc_ctx = msg->drv_cfg; + struct hisi_hpre_eops_ctx *eops_ctx = (struct hisi_hpre_eops_ctx *)msg->priv; int ret = -WD_EINVAL; switch (msg->req.op_type) { @@ -1621,11 +1636,11 @@ static int ecc_prepare_in(struct wd_ecc_msg *msg, ret = ecc_prepare_dh_gen_in(msg, hw_msg, data); break; case WD_ECXDH_GEN_KEY: - hw_msg->bd_rsv2 = ecc_ctx->enable_hpcore; + hw_msg->bd_rsv2 = eops_ctx->enable_hpcore; ret = ecc_prepare_dh_gen_in(msg, hw_msg, data); break; case WD_ECXDH_COMPUTE_KEY: - hw_msg->bd_rsv2 = ecc_ctx->enable_hpcore; + hw_msg->bd_rsv2 = eops_ctx->enable_hpcore; ret = ecc_prepare_dh_compute_in(msg, hw_msg, data); if (!ret && (msg->curve_id == WD_X25519 || msg->curve_id == WD_X448)) @@ -2154,7 +2169,7 @@ free_dst: return ret; } -static int ecc_send(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) +static int ecc_send(handle_t ctx, void *ecc_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_ecc_msg *msg = ecc_msg; @@ -2750,7 +2765,7 @@ fail: return ret; } -static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) +static int ecc_recv(handle_t ctx, void *ecc_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct wd_ecc_msg *msg = ecc_msg; @@ -2786,22 +2801,20 @@ static int ecc_recv(struct wd_alg_driver *drv, handle_t ctx, void *ecc_msg) static handle_t hpre_find_dev_qp(struct wd_alg_driver *drv, const char *dev_name) { - struct wd_ctx_config_internal *config; struct hisi_hpre_ctx *priv; char *ctx_dev_name; handle_t ctx = 0; handle_t qp = 0; __u32 i; - priv = (struct hisi_hpre_ctx *)drv->priv; + priv = (struct hisi_hpre_ctx *)drv->drv_data; if (!priv) return 0; - config = &priv->config; - for (i = 0; i < config->ctx_num; i++) { - ctx_dev_name = wd_ctx_get_dev_name(config->ctxs[i].ctx); + for (i = 0; i < priv->ctx_num; i++) { + ctx_dev_name = wd_ctx_get_dev_name(priv->ctxs[i]->ctx); if (!strcmp(ctx_dev_name, dev_name)) { - ctx = config->ctxs[i].ctx; + ctx = priv->ctxs[i]->ctx; break; } } @@ -2848,9 +2861,24 @@ static int hpre_rsa_get_usage(void *param) return -WD_EACCES; } +static bool is_valid_hw_type(struct wd_alg_driver *drv) +{ + struct hisi_hpre_ctx *hpre_ctx; + struct hisi_qp *qp; + + if (unlikely(!drv || !drv->drv_data)) + return false; + + hpre_ctx = (struct hisi_hpre_ctx *)drv->drv_data; + qp = (struct hisi_qp *)wd_ctx_get_priv(hpre_ctx->ctxs[0]->ctx); + if (!qp || qp->q_info.hw_type < HISI_QM_API_VER3_BASE) + return false; + return true; +} + static int ecc_sess_eops_init(struct wd_alg_driver *drv, void **params) { - struct hpre_ecc_ctx *ecc_ctx; + struct hisi_hpre_eops_ctx *eops_ctx; if (!params) { WD_ERR("invalid: extend ops init params address is NULL!\n"); @@ -2862,11 +2890,11 @@ static int ecc_sess_eops_init(struct wd_alg_driver *drv, void **params) return -WD_EINVAL; } - ecc_ctx = calloc(1, sizeof(struct hpre_ecc_ctx)); - if (!ecc_ctx) + eops_ctx = calloc(1, sizeof(struct hisi_hpre_eops_ctx)); + if (!eops_ctx) return -WD_ENOMEM; - *params = ecc_ctx; + *params = eops_ctx; return WD_SUCCESS; } @@ -2881,34 +2909,19 @@ static void ecc_sess_eops_uninit(struct wd_alg_driver *drv, void *params) free(params); } -static bool is_valid_hw_type(struct wd_alg_driver *drv) -{ - struct hisi_hpre_ctx *hpre_ctx; - struct hisi_qp *qp; - - if (unlikely(!drv || !drv->priv)) - return false; - - hpre_ctx = (struct hisi_hpre_ctx *)drv->priv; - qp = (struct hisi_qp *)wd_ctx_get_priv(hpre_ctx->config.ctxs[0].ctx); - if (!qp || qp->q_info.hw_type < HISI_QM_API_VER3_BASE) - return false; - return true; -} - static void ecc_sess_eops_params_cfg(struct wd_alg_driver *drv, struct wd_ecc_sess_setup *setup, struct wd_ecc_curve *cv, void *params) { __u8 data[SECP256R1_PARAM_SIZE] = SECG_P256_R1_PARAM; - struct hpre_ecc_ctx *ecc_ctx = params; + struct hisi_hpre_eops_ctx *eops_ctx = params; __u32 key_size; int ret; if (!is_valid_hw_type(drv)) return; - if (!ecc_ctx) { + if (!eops_ctx) { WD_INFO("Info: eops config exits, but params is NULL!\n"); return; } @@ -2922,7 +2935,7 @@ static void ecc_sess_eops_params_cfg(struct wd_alg_driver *drv, ret = memcmp_consttime(data, cv->p.data, SECP256R1_PARAM_SIZE); if (!ret) - ecc_ctx->enable_hpcore = 1; + eops_ctx->enable_hpcore = 1; } static int hpre_ecc_get_extend_ops(void *ops) @@ -2932,7 +2945,6 @@ static int hpre_ecc_get_extend_ops(void *ops) if (!ecc_ops) return -WD_EINVAL; - ecc_ops->params = NULL; ecc_ops->sess_init = ecc_sess_eops_init; ecc_ops->eops_params_cfg = ecc_sess_eops_params_cfg; ecc_ops->sess_uninit = ecc_sess_eops_uninit; @@ -2945,15 +2957,22 @@ static int hpre_ecc_get_extend_ops(void *ops) .alg_name = (hpre_alg_name),\ .calc_type = UADK_ALG_HW,\ .priority = 100,\ + .priv_size = sizeof(struct hisi_hpre_ctx),\ + .ops_size = sizeof(struct wd_ecc_extend_ops),\ .queue_num = HPRE_CTX_Q_NUM_DEF,\ .op_type_num = 1,\ + .drv_data = NULL, \ + .extend_ops = NULL, \ .fallback = 0,\ + .init_state = 0,\ .init = hpre_ecc_init,\ .exit = hpre_exit,\ .send = ecc_send,\ .recv = ecc_recv,\ .get_usage = hpre_ecc_get_usage,\ .get_extend_ops = hpre_ecc_get_extend_ops,\ + .alloc_ctx = wd_hw_alloc_ctx, \ + .free_ctx = wd_hw_free_ctx, \ } static struct wd_alg_driver hpre_ecc_driver[] = { @@ -2969,14 +2988,19 @@ static struct wd_alg_driver hpre_rsa_driver = { .alg_name = "rsa", .calc_type = UADK_ALG_HW, .priority = 100, + .priv_size = sizeof(struct hisi_hpre_ctx), .queue_num = HPRE_CTX_Q_NUM_DEF, .op_type_num = 1, + .drv_data = NULL, .fallback = 0, + .init_state = 0, .init = hpre_rsa_dh_init, .exit = hpre_exit, .send = rsa_send, .recv = rsa_recv, .get_usage = hpre_rsa_get_usage, + .alloc_ctx = wd_hw_alloc_ctx, + .free_ctx = wd_hw_free_ctx, }; static struct wd_alg_driver hpre_dh_driver = { @@ -2984,14 +3008,19 @@ static struct wd_alg_driver hpre_dh_driver = { .alg_name = "dh", .calc_type = UADK_ALG_HW, .priority = 100, + .priv_size = sizeof(struct hisi_hpre_ctx), .queue_num = HPRE_CTX_Q_NUM_DEF, .op_type_num = 1, + .drv_data = NULL, .fallback = 0, + .init_state = 0, .init = hpre_rsa_dh_init, .exit = hpre_exit, .send = dh_send, .recv = dh_recv, .get_usage = hpre_rsa_get_usage, + .alloc_ctx = wd_hw_alloc_ctx, + .free_ctx = wd_hw_free_ctx, }; #ifdef WD_STATIC_DRV diff --git a/drv/wd_drv.c b/drv/wd_drv.c index 16dc24c..d6a7626 100644 --- a/drv/wd_drv.c +++ b/drv/wd_drv.c @@ -163,6 +163,21 @@ out_free_list: return ret; } +static int wd_get_alg_class_type(const char *alg_name, char *alg_type) +{ + int ret; + + ret = wd_get_alg_type(alg_name, alg_type); + if (ret) + return -WD_EINVAL; + + /* all ECC types use sm2 to find device. */ + if (!strcmp(alg_type, "ecc")) + strcpy(alg_type, "sm2"); + + return WD_SUCCESS; +} + /** * wd_hw_alloc_ctx() - HW driver's alloc_ctx callback. * @@ -195,8 +210,8 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) } target_numa = ctx_params->numa_id; - /* Get algorithm type and device list */ - ret = wd_get_alg_type(alg_name, alg_type); + /* Get algorithm class type and device list */ + ret = wd_get_alg_class_type(alg_name, alg_type); if (ret) { WD_ERR("invalid: alg_name is NULL!\n"); return -WD_EINVAL; diff --git a/include/drv/wd_ecc_drv.h b/include/drv/wd_ecc_drv.h index 48c422f..585811e 100644 --- a/include/drv/wd_ecc_drv.h +++ b/include/drv/wd_ecc_drv.h @@ -56,7 +56,7 @@ struct wd_ecc_msg { __u16 key_bytes; /* key bytes */ __u8 curve_id; /* Ec curve denoted by enum wd_ecc_curve_type */ __u8 result; /* alg op error code */ - void *drv_cfg; /* internal driver configuration */ + void *priv; /* internal driver configuration */ __u8 *rsv_out; /* reserved output data pointer */ }; @@ -180,10 +180,9 @@ struct wd_ecc_out { }; struct wd_ecc_extend_ops { - void *params; /* the params are passed to the following ops */ void (*eops_params_cfg)(struct wd_alg_driver *drv, - struct wd_ecc_sess_setup *setup, - struct wd_ecc_curve *cv, void *params); + struct wd_ecc_sess_setup *setup, struct wd_ecc_curve *cv, + void *params); int (*sess_init)(struct wd_alg_driver *drv, void **params); void (*sess_uninit)(struct wd_alg_driver *drv, void *params); }; diff --git a/wd_dh.c b/wd_dh.c index 9a0176b..612e15f 100644 --- a/wd_dh.c +++ b/wd_dh.c @@ -17,6 +17,7 @@ #define DH_MAX_KEY_SIZE 512 #define WD_DH_G2 2 +#define WD_DH_OP_TYPE 1 static __thread __u64 balance; @@ -35,7 +36,6 @@ static struct wd_dh_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; } wd_dh_setting; @@ -55,19 +55,15 @@ static void wd_dh_close_driver(int init_type) if (!wd_dh_setting.dlhandle) return; - wd_release_drv(wd_dh_setting.driver); dlclose(wd_dh_setting.dlhandle); wd_dh_setting.dlhandle = NULL; #else - wd_release_drv(wd_dh_setting.driver); hisi_hpre_remove(); #endif } static int wd_dh_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "dh"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -101,18 +97,12 @@ static int wd_dh_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_dh_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_dh_setting.driver = driver; return WD_SUCCESS; } +static bool wd_dh_atfork_registered; + static void wd_dh_clear_status(void) { wd_alg_clear_init(&wd_dh_setting.status); @@ -143,15 +133,8 @@ static int wd_dh_common_init(struct wd_ctx_config *config, struct wd_sched *sche if (ret) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_dh_setting.config, - wd_dh_setting.driver); - if (ret) - goto out_clear_pool; - return WD_SUCCESS; -out_clear_pool: - wd_uninit_async_request_pool(&wd_dh_setting.pool); out_clear_sched: wd_clear_sched(&wd_dh_setting.sched); out_clear_ctx_config: @@ -159,30 +142,24 @@ out_clear_ctx_config: return ret; } -static int wd_dh_common_uninit(void) +static void wd_dh_common_uninit(void) { - enum wd_status status; - - wd_alg_get_init(&wd_dh_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - /* uninit async request pool */ wd_uninit_async_request_pool(&wd_dh_setting.pool); /* unset config, sched, driver */ wd_clear_sched(&wd_dh_setting.sched); - wd_alg_uninit_driver(&wd_dh_setting.config, - wd_dh_setting.driver); - - return WD_SUCCESS; } int wd_dh_init(struct wd_ctx_config *config, struct wd_sched *sched) { + __u32 drv_count = 0; int ret; - pthread_atfork(NULL, NULL, wd_dh_clear_status); + if (!wd_dh_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_dh_clear_status) == 0) + wd_dh_atfork_registered = true; + } ret = wd_alg_try_init(&wd_dh_setting.status); if (ret) @@ -200,10 +177,38 @@ int wd_dh_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_close_driver; + ret = wd_get_drv_array("dh", TASK_HW, "hisi_hpre", + &wd_dh_setting.config.drv_array, &drv_count); + if (ret) { + WD_ERR("driver discovery failed!\n"); + goto out_common_uninit; + } + + wd_dh_setting.config.drv_count = drv_count; + ret = wd_ctx_bind_drivers(&wd_dh_setting.config, NULL, WD_TYPE_V1); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_free_drv_array; + } + + ret = wd_alg_init_driver(&wd_dh_setting.config); + if (ret) { + WD_ERR("dh driver init failed!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_dh_setting.status); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_dh_setting.config); +out_free_drv_array: + wd_put_drv_array(wd_dh_setting.config.drv_array, drv_count); + wd_dh_setting.config.drv_array = NULL; + wd_dh_setting.config.drv_count = 0; +out_common_uninit: + wd_dh_common_uninit(); out_close_driver: wd_dh_close_driver(WD_TYPE_V1); out_clear_init: @@ -213,12 +218,19 @@ out_clear_init: void wd_dh_uninit(void) { - int ret; + enum wd_status status; - ret = wd_dh_common_uninit(); - if (ret) + wd_alg_get_init(&wd_dh_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_dh_setting.config); + wd_ctx_unbind_drivers(&wd_dh_setting.config); + wd_put_drv_array(wd_dh_setting.config.drv_array, wd_dh_setting.config.drv_count); + wd_dh_setting.config.drv_array = NULL; + wd_dh_setting.config.drv_count = 0; + wd_dh_common_uninit(); + wd_dh_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_dh_setting.status); } @@ -229,7 +241,10 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param struct wd_ctx_params dh_ctx_params = {0}; int state, ret = -WD_EINVAL; - pthread_atfork(NULL, NULL, wd_dh_clear_status); + if (!wd_dh_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_dh_clear_status) == 0) + wd_dh_atfork_registered = true; + } state = wd_alg_try_init(&wd_dh_setting.status); if (state) @@ -252,38 +267,25 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param while (ret) { memset(&wd_dh_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - - /* Get alg driver and dev name */ - wd_dh_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_dh_setting.driver) { - WD_ERR("fail to bind a valid driver.\n"); - ret = -WD_EINVAL; - goto out_dlopen; - } - + /* Init ctx param and prepare for ctx request */ dh_ctx_params.ctx_set_num = dh_ctx_num; ret = wd_ctx_param_init(&dh_ctx_params, ctx_params, - wd_dh_setting.driver, WD_DH_TYPE, WD_DH_PHASE2); + alg, WD_DH_TYPE, WD_DH_OP_TYPE); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_dh_setting.driver); - wd_alg_drv_unbind(wd_dh_setting.driver); + if (ret == -WD_EAGAIN) continue; - } goto out_driver; } (void)strcpy(wd_dh_init_attrs.alg, alg); wd_dh_init_attrs.sched_type = sched_type; - wd_dh_init_attrs.driver = wd_dh_setting.driver; + wd_dh_init_attrs.task_type = task_type; wd_dh_init_attrs.ctx_params = &dh_ctx_params; wd_dh_init_attrs.alg_init = wd_dh_common_init; wd_dh_init_attrs.alg_poll_ctx = wd_dh_poll_ctx; ret = wd_alg_attrs_init(&wd_dh_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_dh_setting.driver); - wd_alg_drv_unbind(wd_dh_setting.driver); wd_ctx_param_uninit(&dh_ctx_params); continue; } @@ -292,16 +294,33 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param } } + ret = wd_ctx_bind_drivers(&wd_dh_setting.config, + wd_dh_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_common_uninit; + } + + ret = wd_alg_init_driver(&wd_dh_setting.config); + if (ret) { + WD_ERR("driver init failed!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_dh_setting.status); wd_ctx_param_uninit(&dh_ctx_params); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_dh_setting.config); +out_common_uninit: + wd_dh_common_uninit(); + wd_alg_attrs_uninit(&wd_dh_init_attrs); out_params_uninit: wd_ctx_param_uninit(&dh_ctx_params); out_driver: - wd_alg_drv_unbind(wd_dh_setting.driver); -out_dlopen: wd_dh_close_driver(WD_TYPE_V2); out_clear_init: wd_alg_clear_init(&wd_dh_setting.status); @@ -310,14 +329,19 @@ out_clear_init: void wd_dh_uninit2(void) { - int ret; + enum wd_status status; - ret = wd_dh_common_uninit(); - if (ret) + wd_alg_get_init(&wd_dh_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_dh_setting.config); + wd_ctx_unbind_drivers(&wd_dh_setting.config); + wd_dh_setting.config.drv_array = NULL; + wd_dh_setting.config.drv_count = 0; + wd_dh_common_uninit(); + wd_alg_attrs_uninit(&wd_dh_init_attrs); - wd_alg_drv_unbind(wd_dh_setting.driver); wd_dh_close_driver(WD_TYPE_V2); wd_alg_clear_init(&wd_dh_setting.status); } @@ -386,13 +410,13 @@ int wd_do_dh_sync(handle_t sess, struct wd_dh_req *req) if (unlikely(ret)) return ret; - msg_handle.send = wd_dh_setting.driver->send; - msg_handle.recv = wd_dh_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; - pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_dh_setting.driver, &msg_handle, ctx->ctx, - &msg, &balance, wd_dh_setting.config.epoll_en); - pthread_spin_unlock(&ctx->lock); + wd_ctx_spin_lock(ctx, ctx->ctx_type); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, &msg, &balance, + wd_dh_setting.config.epoll_en); + wd_ctx_spin_unlock(ctx, ctx->ctx_type); if (unlikely(ret)) return ret; @@ -428,8 +452,7 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) mid = wd_get_msg_from_pool(&wd_dh_setting.pool, idx, (void **)&msg); if (unlikely(mid < 0)) { - WD_ERR("failed to get msg from pool!\n"); - return mid; + return -WD_EBUSY; } ret = fill_dh_msg(msg, req, (struct wd_dh_sess *)sess); @@ -437,7 +460,7 @@ int wd_do_dh_async(handle_t sess, struct wd_dh_req *req) goto fail_with_msg; msg->tag = mid; - ret = wd_alg_driver_send(wd_dh_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send dh BD, hw is err!\n"); @@ -485,7 +508,7 @@ int wd_dh_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_dh_setting.driver, ctx->ctx, &rcv_msg); + ret = ctx->drv->recv(ctx->ctx, &rcv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (unlikely(ret)) { @@ -581,6 +604,7 @@ void wd_dh_get_g(handle_t sess, struct wd_dtb **g) handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) { + struct wd_sched_params params = {0}; struct wd_dh_sess *sess; int ret; @@ -589,6 +613,12 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) return (handle_t)0; } + ret = wd_drv_alg_support("dh", &wd_dh_setting.config); + if (!ret) { + WD_ERR("failed to support this algorithm: rsa!\n"); + return (handle_t)0; + } + /* key width check */ if (setup->key_bits != 768 && setup->key_bits != 1024 && @@ -630,6 +660,13 @@ handle_t wd_dh_alloc_sess(struct wd_dh_sess_setup *setup) goto sched_err; } + /* Set compat filtering parameters for session-ctx matching */ + params.alg_name = "dh"; + params.ctxs = wd_dh_setting.config.ctxs; + wd_dh_setting.sched.set_param( + wd_dh_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); + return (handle_t)sess; sched_err: @@ -651,8 +688,15 @@ void wd_dh_free_sess(handle_t sess) if (sess_t->g.data) sess_t->mm_ops.free(sess_t->mm_ops.usr, sess_t->g.data); - if (sess_t->sched_key) - free(sess_t->sched_key); + if (sess_t->sched_key) { + if (wd_dh_setting.sched.sched_uninit) + wd_dh_setting.sched.sched_uninit( + wd_dh_setting.sched.h_sched_ctx, + (handle_t)sess_t->sched_key); + else + free(sess_t->sched_key); + } + free(sess_t); } diff --git a/wd_ecc.c b/wd_ecc.c index dae8a05..2d1aa77 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -29,6 +29,9 @@ #define GET_NEGATIVE(val) (0 - (val)) #define ZA_PARAM_NUM 6 #define WD_SECP256R1 0x18 /* consistent with enum wd_ecc_curve_id */ +#define WD_ECC_OP_TYPE 1 +/* Default ECC algorithm for wd_ecc_init v1 */ +#define ECC_ALG_DEFAULT_V1 "ecc" static __thread __u64 balance; @@ -50,10 +53,10 @@ struct wd_ecc_sess { __u32 key_size; struct wd_ecc_key key; struct wd_ecc_sess_setup setup; - struct wd_ecc_extend_ops eops; void *sched_key; struct wd_mm_ops mm_ops; enum wd_mem_type mm_type; + void **priv; }; struct wd_ecc_curve_list { @@ -68,7 +71,7 @@ static struct wd_ecc_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; + enum wd_init_type init_type; void *dlhandle; void *dlh_list; } wd_ecc_setting; @@ -112,19 +115,15 @@ static void wd_ecc_close_driver(int init_type) if (!wd_ecc_setting.dlhandle) return; - wd_release_drv(wd_ecc_setting.driver); dlclose(wd_ecc_setting.dlhandle); wd_ecc_setting.dlhandle = NULL; #else - wd_release_drv(wd_ecc_setting.driver); hisi_hpre_remove(); #endif } static int wd_ecc_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "sm2"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -158,14 +157,6 @@ static int wd_ecc_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_ecc_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_ecc_setting.driver = driver; return WD_SUCCESS; } @@ -180,6 +171,8 @@ static bool is_alg_support(const char *alg) return true; } +static bool wd_ecc_atfork_registered; + static void wd_ecc_clear_status(void) { wd_alg_clear_init(&wd_ecc_setting.status); @@ -209,15 +202,8 @@ static int wd_ecc_common_init(struct wd_ctx_config *config, struct wd_sched *sch if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_ecc_setting.config, - wd_ecc_setting.driver); - if (ret) - goto out_clear_pool; - return WD_SUCCESS; -out_clear_pool: - wd_uninit_async_request_pool(&wd_ecc_setting.pool); out_clear_sched: wd_clear_sched(&wd_ecc_setting.sched); out_clear_ctx_config: @@ -225,30 +211,24 @@ out_clear_ctx_config: return ret; } -static int wd_ecc_common_uninit(void) +static void wd_ecc_common_uninit(void) { - enum wd_status status; - - wd_alg_get_init(&wd_ecc_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - /* uninit async request pool */ wd_uninit_async_request_pool(&wd_ecc_setting.pool); /* unset config, sched, driver */ wd_clear_sched(&wd_ecc_setting.sched); - wd_alg_uninit_driver(&wd_ecc_setting.config, - wd_ecc_setting.driver); - - return WD_SUCCESS; } int wd_ecc_init(struct wd_ctx_config *config, struct wd_sched *sched) { + __u32 drv_count = 0; int ret; - pthread_atfork(NULL, NULL, wd_ecc_clear_status); + if (!wd_ecc_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_ecc_clear_status) == 0) + wd_ecc_atfork_registered = true; + } ret = wd_alg_try_init(&wd_ecc_setting.status); if (ret) @@ -266,10 +246,39 @@ int wd_ecc_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_close_driver; + ret = wd_get_drv_array(ECC_ALG_DEFAULT_V1, TASK_HW, "hisi_hpre", + &wd_ecc_setting.config.drv_array, &drv_count); + if (ret) { + WD_ERR("driver discovery failed!\n"); + goto out_common_uninit; + } + + wd_ecc_setting.config.drv_count = drv_count; + ret = wd_ctx_bind_drivers(&wd_ecc_setting.config, NULL, WD_TYPE_V1); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_free_drv_array; + } + + ret = wd_alg_init_driver(&wd_ecc_setting.config); + if (ret) { + WD_ERR("ecc driver init failed!\n"); + goto out_unbind_drivers; + } + wd_ecc_setting.init_type = WD_TYPE_V1; wd_alg_set_init(&wd_ecc_setting.status); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_ecc_setting.config); +out_free_drv_array: + wd_put_drv_array(wd_ecc_setting.config.drv_array, + wd_ecc_setting.config.drv_count); + wd_ecc_setting.config.drv_array = NULL; + wd_ecc_setting.config.drv_count = 0; +out_common_uninit: + wd_ecc_common_uninit(); out_close_driver: wd_ecc_close_driver(WD_TYPE_V1); out_clear_init: @@ -279,12 +288,20 @@ out_clear_init: void wd_ecc_uninit(void) { - int ret; + enum wd_status status; - ret = wd_ecc_common_uninit(); - if (ret) + wd_alg_get_init(&wd_ecc_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_ecc_setting.config); + wd_ctx_unbind_drivers(&wd_ecc_setting.config); + wd_put_drv_array(wd_ecc_setting.config.drv_array, + wd_ecc_setting.config.drv_count); + wd_ecc_setting.config.drv_array = NULL; + wd_ecc_setting.config.drv_count = 0; + wd_ecc_common_uninit(); + wd_ecc_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_ecc_setting.status); } @@ -296,7 +313,10 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para int state, ret = -WD_EINVAL; bool flag; - pthread_atfork(NULL, NULL, wd_ecc_clear_status); + if (!wd_ecc_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_ecc_clear_status) == 0) + wd_ecc_atfork_registered = true; + } state = wd_alg_try_init(&wd_ecc_setting.status); if (state) @@ -321,37 +341,26 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para while (ret) { memset(&wd_ecc_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - /* Get alg driver and dev name */ - wd_ecc_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_ecc_setting.driver) { - WD_ERR("failed to bind a valid driver!\n"); - ret = -WD_EINVAL; - goto out_dlopen; - } - + /* Init ctx param and prepare for ctx request */ ecc_ctx_params.ctx_set_num = ecc_ctx_num; ret = wd_ctx_param_init(&ecc_ctx_params, ctx_params, - wd_ecc_setting.driver, WD_ECC_TYPE, WD_EC_OP_MAX); + alg, WD_ECC_TYPE, WD_ECC_OP_TYPE); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_ecc_setting.driver); - wd_alg_drv_unbind(wd_ecc_setting.driver); + if (ret == -WD_EAGAIN) continue; - } + goto out_driver; } (void)strcpy(wd_ecc_init_attrs.alg, alg); wd_ecc_init_attrs.sched_type = sched_type; - wd_ecc_init_attrs.driver = wd_ecc_setting.driver; + wd_ecc_init_attrs.task_type = task_type; wd_ecc_init_attrs.ctx_params = &ecc_ctx_params; wd_ecc_init_attrs.alg_init = wd_ecc_common_init; wd_ecc_init_attrs.alg_poll_ctx = wd_ecc_poll_ctx; ret = wd_alg_attrs_init(&wd_ecc_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_ecc_setting.driver); - wd_alg_drv_unbind(wd_ecc_setting.driver); wd_ctx_param_uninit(&ecc_ctx_params); continue; } @@ -359,17 +368,32 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para goto out_params_uninit; } } + ret = wd_ctx_bind_drivers(&wd_ecc_setting.config, + wd_ecc_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_common_uninit; + } + ret = wd_alg_init_driver(&wd_ecc_setting.config); + if (ret) + goto out_unbind_drivers; + + wd_ecc_setting.init_type = WD_TYPE_V2; wd_alg_set_init(&wd_ecc_setting.status); wd_ctx_param_uninit(&ecc_ctx_params); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_ecc_setting.config); +out_common_uninit: + wd_ecc_common_uninit(); + wd_alg_attrs_uninit(&wd_ecc_init_attrs); out_params_uninit: wd_ctx_param_uninit(&ecc_ctx_params); out_driver: - wd_alg_drv_unbind(wd_ecc_setting.driver); -out_dlopen: wd_ecc_close_driver(WD_TYPE_V2); out_clear_init: wd_alg_clear_init(&wd_ecc_setting.status); @@ -378,14 +402,20 @@ out_clear_init: void wd_ecc_uninit2(void) { - int ret; + enum wd_status status; - ret = wd_ecc_common_uninit(); - if (ret) + wd_alg_get_init(&wd_ecc_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_ecc_setting.config); + wd_ctx_unbind_drivers(&wd_ecc_setting.config); + wd_ecc_setting.config.drv_array = NULL; + wd_ecc_setting.config.drv_count = 0; + wd_ecc_common_uninit(); + wd_alg_attrs_uninit(&wd_ecc_init_attrs); - wd_alg_drv_unbind(wd_ecc_setting.driver); + wd_ecc_close_driver(WD_TYPE_V2); wd_alg_clear_init(&wd_ecc_setting.status); } @@ -1174,49 +1204,135 @@ static void del_sess_key(struct wd_ecc_sess *sess) static int wd_ecc_sess_eops_init(struct wd_ecc_sess *sess) { - int ret; - - if (sess->eops.sess_init) { - if (!sess->eops.sess_uninit) { - WD_ERR("failed to get extend ops in session!\n"); - return -WD_EINVAL; + struct wd_ctx_config_internal *config = &wd_ecc_setting.config; + struct wd_ecc_extend_ops *eops; + struct wd_alg_driver *drv; + int ret, valid = 0; + int prev_idx; + __u32 i, j; + + sess->priv = malloc(sizeof(void *) * config->ctx_num); + if (!sess->priv) + return -WD_ENOMEM; + memset(sess->priv, 0, sizeof(void *) * config->ctx_num); + + for (i = 0; i < config->ctx_num; i++) { + drv = config->ctxs[i].drv; + if (!drv->extend_ops) + continue; + + prev_idx = -1; + for (j = 0; j < i; j++) { + if (!strcmp(config->ctxs[j].drv->drv_name, drv->drv_name)) { + prev_idx = j; + break; + } } - ret = sess->eops.sess_init(wd_ecc_setting.driver, &sess->eops.params); - if (ret) { - WD_ERR("failed to init extend ops params in session!\n"); - return ret; + if (prev_idx >= 0) { + sess->priv[i] = sess->priv[prev_idx]; + continue; + } + + ret = drv->get_extend_ops(config->ctxs[i].drv->extend_ops); + if (!ret && config->ctxs[i].drv->extend_ops) { + valid++; + eops = config->ctxs[i].drv->extend_ops; + + if (eops->sess_init) { + if (!eops->sess_uninit) { + WD_ERR("failed to get session uninit ops!\n"); + return -WD_EINVAL; + } + ret = eops->sess_init(drv, &sess->priv[i]); + if (ret) { + WD_ERR("failed to init session priv!\n"); + return ret; + } + } } } + + if (!valid) + WD_DEBUG("no ecc extend ops found!\n"); + return WD_SUCCESS; } static void wd_ecc_sess_eops_uninit(struct wd_ecc_sess *sess) { - if (sess->eops.sess_uninit) { - sess->eops.sess_uninit(wd_ecc_setting.driver, sess->eops.params); - sess->eops.params = NULL; + struct wd_ctx_config_internal *config = &wd_ecc_setting.config; + struct wd_ecc_extend_ops *eops; + struct wd_alg_driver *drv; + int prev_idx; + __u32 i, j; + + for (i = 0; i < config->ctx_num; i++) { + drv = config->ctxs[i].drv; + eops = drv->extend_ops; + if (!eops) + continue; + + prev_idx = -1; + for (j = 0; j < i; j++) { + if (!strcmp(config->ctxs[j].drv->drv_name, drv->drv_name)) { + prev_idx = j; + break; + } + } + if (prev_idx >= 0) + continue; + + if (eops->sess_uninit) + eops->sess_uninit(drv, sess->priv[i]); } + + if (sess->priv) { + free(sess->priv); + sess->priv = NULL; + } + + return; } -static void wd_ecc_sess_eops_cfg(struct wd_ecc_sess_setup *setup, - struct wd_ecc_sess *sess) + +static void wd_ecc_sess_eops_cfg(struct wd_ecc_sess *sess, struct wd_ecc_sess_setup *setup) { - if (sess->eops.sess_init && sess->eops.eops_params_cfg) { - /* the config result does not impact task sucesss or failure */ - sess->eops.eops_params_cfg(wd_ecc_setting.driver, setup, sess->key.cv, - sess->eops.params); + struct wd_ctx_config_internal *config = &wd_ecc_setting.config; + struct wd_ecc_extend_ops *eops; + struct wd_alg_driver *drv; + int prev_idx; + __u32 i, j; + + for (i = 0; i < config->ctx_num; i++) { + drv = config->ctxs[i].drv; + eops = drv->extend_ops; + if (!eops || !eops->eops_params_cfg) + continue; + + prev_idx = -1; + for (j = 0; j < i; j++) { + if (!strcmp(config->ctxs[j].drv->drv_name, drv->drv_name)) { + prev_idx = j; + break; + } + } + if (prev_idx >= 0) + continue; + + eops->eops_params_cfg(drv, setup, sess->key.cv, sess->priv[i]); } } handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) { + struct wd_sched_params params = {0}; struct wd_ecc_sess *sess; int ret; if (setup_param_check(setup)) return (handle_t)0; - ret = wd_drv_alg_support(setup->alg, wd_ecc_setting.driver); + ret = wd_drv_alg_support(setup->alg, &wd_ecc_setting.config); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", setup->alg); return (handle_t)0; @@ -1238,14 +1354,6 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops)); sess->mm_type = setup->mm_type; - if (wd_ecc_setting.driver->get_extend_ops) { - ret = wd_ecc_setting.driver->get_extend_ops(&sess->eops); - if (ret) { - WD_ERR("failed to get ecc sess extend ops!\n"); - goto sess_err; - } - } - ret = wd_ecc_sess_eops_init(sess); if (ret) { WD_ERR("failed to init ecc sess extend eops!\n"); @@ -1258,7 +1366,7 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) goto eops_err; } - wd_ecc_sess_eops_cfg(setup, sess); + wd_ecc_sess_eops_cfg(sess, setup); /* Some simple scheduler don't need scheduling parameters */ sess->sched_key = (void *)wd_ecc_setting.sched.sched_init( @@ -1268,6 +1376,13 @@ handle_t wd_ecc_alloc_sess(struct wd_ecc_sess_setup *setup) goto sched_err; } + /* Set compat filtering parameters for session-ctx matching */ + params.alg_name = sess->setup.alg; + params.ctxs = wd_ecc_setting.config.ctxs; + wd_ecc_setting.sched.set_param( + wd_ecc_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); + return (handle_t)sess; sched_err: @@ -1288,8 +1403,15 @@ void wd_ecc_free_sess(handle_t sess) return; } - if (sess_t->sched_key) - free(sess_t->sched_key); + if (sess_t->sched_key) { + if (wd_ecc_setting.sched.sched_uninit) + wd_ecc_setting.sched.sched_uninit( + wd_ecc_setting.sched.h_sched_ctx, + (handle_t)sess_t->sched_key); + else + free(sess_t->sched_key); + } + del_sess_key(sess_t); wd_ecc_sess_eops_uninit(sess_t); free(sess_t); @@ -1568,7 +1690,6 @@ static int fill_ecc_msg(struct wd_ecc_msg *msg, struct wd_ecc_req *req, msg->mm_type = sess->mm_type; msg->key_bytes = sess->key_size; msg->curve_id = sess->setup.cv.cfg.id; - msg->drv_cfg = sess->eops.params; msg->result = WD_EINVAL; switch (req->op_type) { @@ -1648,13 +1769,14 @@ int wd_do_ecc_sync(handle_t h_sess, struct wd_ecc_req *req) if (unlikely(ret)) return ret; - msg_handle.send = wd_ecc_setting.driver->send; - msg_handle.recv = wd_ecc_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; + msg.priv = sess->priv[idx]; - pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_ecc_setting.driver, &msg_handle, ctx->ctx, &msg, - &balance, wd_ecc_setting.config.epoll_en); - pthread_spin_unlock(&ctx->lock); + wd_ctx_spin_lock(ctx, ctx->ctx_type); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, &msg, &balance, + wd_ecc_setting.config.epoll_en); + wd_ctx_spin_unlock(ctx, ctx->ctx_type); if (unlikely(ret)) return ret; @@ -2327,19 +2449,18 @@ int wd_do_ecc_async(handle_t sess, struct wd_ecc_req *req) return ret; ctx = config->ctxs + idx; - mid = wd_get_msg_from_pool(&wd_ecc_setting.pool, idx, (void **)&msg); if (unlikely(mid < 0)) { - WD_ERR("failed to get msg from pool!\n"); - return mid; + return -WD_EBUSY; } ret = fill_ecc_msg(msg, req, (struct wd_ecc_sess *)sess); if (ret) goto fail_with_msg; msg->tag = mid; + msg->priv = sess_t->priv[idx]; - ret = wd_alg_driver_send(wd_ecc_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send ecc BD, hw is err!\n"); @@ -2386,7 +2507,7 @@ int wd_ecc_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_ecc_setting.driver, ctx->ctx, &recv_msg); + ret = ctx->drv->recv(ctx->ctx, &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { diff --git a/wd_rsa.c b/wd_rsa.c index 93ec336..754c28a 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -16,6 +16,7 @@ #include "wd_rsa.h" #define RSA_MAX_KEY_SIZE 512 +#define WD_RSA_OP_TYPE 1 static __thread __u64 balance; @@ -76,7 +77,6 @@ static struct wd_rsa_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; } wd_rsa_setting; @@ -96,19 +96,15 @@ static void wd_rsa_close_driver(int init_type) if (!wd_rsa_setting.dlhandle) return; - wd_release_drv(wd_rsa_setting.driver); dlclose(wd_rsa_setting.dlhandle); wd_rsa_setting.dlhandle = NULL; #else - wd_release_drv(wd_rsa_setting.driver); hisi_hpre_remove(); #endif } static int wd_rsa_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "rsa"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -142,18 +138,12 @@ static int wd_rsa_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_rsa_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support!\n", alg_name); - return -WD_EINVAL; - } - - wd_rsa_setting.driver = driver; return WD_SUCCESS; } +static bool wd_rsa_atfork_registered; + static void wd_rsa_clear_status(void) { wd_alg_clear_init(&wd_rsa_setting.status); @@ -183,15 +173,8 @@ static int wd_rsa_common_init(struct wd_ctx_config *config, struct wd_sched *sch if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_rsa_setting.config, - wd_rsa_setting.driver); - if (ret) - goto out_clear_pool; - return WD_SUCCESS; -out_clear_pool: - wd_uninit_async_request_pool(&wd_rsa_setting.pool); out_clear_sched: wd_clear_sched(&wd_rsa_setting.sched); out_clear_ctx_config: @@ -199,30 +182,24 @@ out_clear_ctx_config: return ret; } -static int wd_rsa_common_uninit(void) +static void wd_rsa_common_uninit(void) { - enum wd_status status; - - wd_alg_get_init(&wd_rsa_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - /* uninit async request pool */ wd_uninit_async_request_pool(&wd_rsa_setting.pool); /* unset config, sched, driver */ wd_clear_sched(&wd_rsa_setting.sched); - wd_alg_uninit_driver(&wd_rsa_setting.config, - wd_rsa_setting.driver); - - return WD_SUCCESS; } int wd_rsa_init(struct wd_ctx_config *config, struct wd_sched *sched) { + __u32 drv_count = 0; int ret; - pthread_atfork(NULL, NULL, wd_rsa_clear_status); + if (!wd_rsa_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_rsa_clear_status) == 0) + wd_rsa_atfork_registered = true; + } ret = wd_alg_try_init(&wd_rsa_setting.status); if (ret) @@ -240,10 +217,39 @@ int wd_rsa_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_close_driver; + ret = wd_get_drv_array("rsa", TASK_HW, "hisi_hpre", + &wd_rsa_setting.config.drv_array, &drv_count); + if (ret) { + WD_ERR("driver discovery failed!\n"); + goto out_common_uninit; + } + + wd_rsa_setting.config.drv_count = drv_count; + ret = wd_ctx_bind_drivers(&wd_rsa_setting.config, + NULL, WD_TYPE_V1); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_free_drv_array; + } + + ret = wd_alg_init_driver(&wd_rsa_setting.config); + if (ret) { + WD_ERR("rsa driver init failed!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_rsa_setting.status); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_rsa_setting.config); +out_free_drv_array: + wd_put_drv_array(wd_rsa_setting.config.drv_array, drv_count); + wd_rsa_setting.config.drv_array = NULL; + wd_rsa_setting.config.drv_count = 0; +out_common_uninit: + wd_rsa_common_uninit(); out_close_driver: wd_rsa_close_driver(WD_TYPE_V1); out_clear_init: @@ -253,12 +259,20 @@ out_clear_init: void wd_rsa_uninit(void) { - int ret; + enum wd_status status; - ret = wd_rsa_common_uninit(); - if (ret) + wd_alg_get_init(&wd_rsa_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_rsa_setting.config); + wd_ctx_unbind_drivers(&wd_rsa_setting.config); + wd_put_drv_array(wd_rsa_setting.config.drv_array, + wd_rsa_setting.config.drv_count); + wd_rsa_setting.config.drv_array = NULL; + wd_rsa_setting.config.drv_count = 0; + wd_rsa_common_uninit(); + wd_rsa_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_rsa_setting.status); } @@ -269,7 +283,10 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para struct wd_ctx_params rsa_ctx_params = {0}; int state, ret = -WD_EINVAL; - pthread_atfork(NULL, NULL, wd_rsa_clear_status); + if (!wd_rsa_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_rsa_clear_status) == 0) + wd_rsa_atfork_registered = true; + } state = wd_alg_try_init(&wd_rsa_setting.status); if (state) @@ -293,37 +310,26 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para while (ret) { memset(&wd_rsa_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - /* Get alg driver and dev name */ - wd_rsa_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_rsa_setting.driver) { - WD_ERR("failed to bind a valid driver!\n"); - ret = -WD_EINVAL; - goto out_dlopen; - } - + /* Init ctx param and prepare for ctx request */ rsa_ctx_params.ctx_set_num = rsa_ctx_num; ret = wd_ctx_param_init(&rsa_ctx_params, ctx_params, - wd_rsa_setting.driver, WD_RSA_TYPE, WD_RSA_GENKEY); + alg, WD_RSA_TYPE, WD_RSA_OP_TYPE); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_rsa_setting.driver); - wd_alg_drv_unbind(wd_rsa_setting.driver); + if (ret == -WD_EAGAIN) continue; - } - goto out_driver; + + goto out_dlclose; } (void)strcpy(wd_rsa_init_attrs.alg, alg); wd_rsa_init_attrs.sched_type = sched_type; - wd_rsa_init_attrs.driver = wd_rsa_setting.driver; + wd_rsa_init_attrs.task_type = task_type; wd_rsa_init_attrs.ctx_params = &rsa_ctx_params; wd_rsa_init_attrs.alg_init = wd_rsa_common_init; wd_rsa_init_attrs.alg_poll_ctx = wd_rsa_poll_ctx; ret = wd_alg_attrs_init(&wd_rsa_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_rsa_setting.driver); - wd_alg_drv_unbind(wd_rsa_setting.driver); wd_ctx_param_uninit(&rsa_ctx_params); continue; } @@ -332,16 +338,33 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para } } + ret = wd_ctx_bind_drivers(&wd_rsa_setting.config, + wd_rsa_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_common_uninit; + } + + ret = wd_alg_init_driver(&wd_rsa_setting.config); + if (ret) { + WD_ERR("driver init failed!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_rsa_setting.status); wd_ctx_param_uninit(&rsa_ctx_params); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_rsa_setting.config); +out_common_uninit: + wd_alg_attrs_uninit(&wd_rsa_init_attrs); + wd_rsa_common_uninit(); out_params_uninit: wd_ctx_param_uninit(&rsa_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_rsa_setting.driver); -out_dlopen: +out_dlclose: wd_rsa_close_driver(WD_TYPE_V2); out_clear_init: wd_alg_clear_init(&wd_rsa_setting.status); @@ -350,14 +373,19 @@ out_clear_init: void wd_rsa_uninit2(void) { - int ret; + enum wd_status status; - ret = wd_rsa_common_uninit(); - if (ret) + wd_alg_get_init(&wd_rsa_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_rsa_setting.config); + wd_ctx_unbind_drivers(&wd_rsa_setting.config); + wd_rsa_setting.config.drv_array = NULL; + wd_rsa_setting.config.drv_count = 0; + wd_rsa_common_uninit(); + wd_alg_attrs_uninit(&wd_rsa_init_attrs); - wd_alg_drv_unbind(wd_rsa_setting.driver); wd_rsa_close_driver(WD_TYPE_V2); wd_alg_clear_init(&wd_rsa_setting.status); } @@ -447,13 +475,13 @@ int wd_do_rsa_sync(handle_t h_sess, struct wd_rsa_req *req) if (unlikely(ret)) return ret; - msg_handle.send = wd_rsa_setting.driver->send; - msg_handle.recv = wd_rsa_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; - pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_rsa_setting.driver, &msg_handle, ctx->ctx, &msg, - &balance, wd_rsa_setting.config.epoll_en); - pthread_spin_unlock(&ctx->lock); + wd_ctx_spin_lock(ctx, ctx->ctx_type); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, &msg, &balance, + wd_rsa_setting.config.epoll_en); + wd_ctx_spin_unlock(ctx, ctx->ctx_type); if (unlikely(ret)) return ret; @@ -489,8 +517,7 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) mid = wd_get_msg_from_pool(&wd_rsa_setting.pool, idx, (void **)&msg); if (unlikely(mid < 0)) { - WD_ERR("failed to get msg from pool!\n"); - return mid; + return -WD_EBUSY; } ret = fill_rsa_msg(msg, req, (struct wd_rsa_sess *)sess); @@ -498,7 +525,7 @@ int wd_do_rsa_async(handle_t sess, struct wd_rsa_req *req) goto fail_with_msg; msg->tag = mid; - ret = wd_alg_driver_send(wd_rsa_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send rsa BD, hw is err!\n"); @@ -545,7 +572,7 @@ int wd_rsa_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_rsa_setting.driver, ctx->ctx, &recv_msg); + ret = ctx->drv->recv(ctx->ctx, &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { @@ -935,6 +962,7 @@ static void del_sess(struct wd_rsa_sess *c) /* Before initiate this context, we should get a queue from WD */ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) { + struct wd_sched_params params = {0}; struct wd_rsa_sess *sess; int ret; @@ -943,6 +971,12 @@ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) return(handle_t)0; } + ret = wd_drv_alg_support("rsa", &wd_rsa_setting.config); + if (!ret) { + WD_ERR("failed to support this algorithm: rsa!\n"); + return (handle_t)0; + } + if (setup->key_bits != 1024 && setup->key_bits != 2048 && setup->key_bits != 3072 && @@ -981,6 +1015,13 @@ handle_t wd_rsa_alloc_sess(struct wd_rsa_sess_setup *setup) goto sched_err; } + /* Set compat filtering parameters for session-ctx matching */ + params.alg_name = "rsa"; + params.ctxs = wd_rsa_setting.config.ctxs; + wd_rsa_setting.sched.set_param( + wd_rsa_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); + return (handle_t)sess; sched_err: @@ -999,8 +1040,15 @@ void wd_rsa_free_sess(handle_t sess) return; } - if (sess_t->sched_key) - free(sess_t->sched_key); + if (sess_t->sched_key) { + if (wd_rsa_setting.sched.sched_uninit) + wd_rsa_setting.sched.sched_uninit( + wd_rsa_setting.sched.h_sched_ctx, + (handle_t)sess_t->sched_key); + else + free(sess_t->sched_key); + } + del_sess_key(sess_t); del_sess(sess_t); } -- 2.43.0
From: Zongyu Wu <wuzongyu1@huawei.com> For compression algorithms in the updated UADK framework, the API layer needs to be adapted to the new UADK heterogeneous hybrid acceleration framework, thereby ensuring that both compression and decompression algorithms can achieve heterogeneous hybrid acceleration. Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> --- drv/hisi_comp.c | 36 ++++----- wd_comp.c | 194 ++++++++++++++++++++++++++++++++++------------- wd_zlibwrapper.c | 12 ++- 3 files changed, 165 insertions(+), 77 deletions(-) diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index 97bcf39..c75e734 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -5,6 +5,7 @@ #include "drv/wd_comp_drv.h" #include "drv/hisi_comp_huf.h" #include "hisi_qm_udrv.h" +#include "wd_drv.h" #define ZLIB 0 #define GZIP 1 @@ -15,6 +16,7 @@ #define ZLIB_HEADER "\x78\x9c" #define ZLIB_HEADER_SZ 2 #define ZIP_CTX_Q_NUM_DEF 1 +#define WD_COMP_ALG_TYPE_V1 "deflate" /* * We use a extra field for gzip block length. So the fourth byte is \x04. * This is necessary because our software don't know the size of block when @@ -1473,11 +1475,11 @@ static void hisi_zip_sqe_ops_adapt(handle_t h_qp) } } -static int hisi_zip_init(struct wd_alg_driver *drv, void *conf) +static int hisi_zip_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; + struct hisi_zip_ctx *zip_ctx = (struct hisi_zip_ctx *)priv; struct hisi_qm_priv qm_priv; - struct hisi_zip_ctx *priv; handle_t h_qp = 0; handle_t h_ctx; __u32 i, j; @@ -1487,11 +1489,7 @@ static int hisi_zip_init(struct wd_alg_driver *drv, void *conf) return -WD_EINVAL; } - priv = malloc(sizeof(struct hisi_zip_ctx)); - if (!priv) - return -WD_EINVAL; - - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); + memcpy(&zip_ctx->config, config, sizeof(struct wd_ctx_config_internal)); /* allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { h_ctx = config->ctxs[i].ctx; @@ -1509,7 +1507,6 @@ static int hisi_zip_init(struct wd_alg_driver *drv, void *conf) } hisi_zip_sqe_ops_adapt(h_qp); - drv->priv = priv; return 0; out: @@ -1517,28 +1514,20 @@ out: h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); hisi_qm_free_qp(h_qp); } - free(priv); return -WD_EINVAL; } -static void hisi_zip_exit(struct wd_alg_driver *drv) +static void hisi_zip_exit(void *priv) { - struct wd_ctx_config_internal *config; - struct hisi_zip_ctx *priv; + struct hisi_zip_ctx *zip_ctx = (struct hisi_zip_ctx *)priv; + struct wd_ctx_config_internal *config = &zip_ctx->config; handle_t h_qp; __u32 i; - if (!drv || !drv->priv) - return; - - priv = (struct hisi_zip_ctx *)drv->priv; - config = &priv->config; for (i = 0; i < config->ctx_num; i++) { h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); hisi_qm_free_qp(h_qp); } - free(priv); - drv->priv = NULL; } static int fill_zip_comp_sqe(struct hisi_qp *qp, struct wd_comp_msg *msg, @@ -1593,7 +1582,7 @@ static int fill_zip_comp_sqe(struct hisi_qp *qp, struct wd_comp_msg *msg, return 0; } -static int hisi_zip_comp_send(struct wd_alg_driver *drv, handle_t ctx, void *comp_msg) +static int hisi_zip_comp_send(handle_t ctx, void *comp_msg) { struct hisi_qp *qp = wd_ctx_get_priv(ctx); struct wd_comp_msg *msg = comp_msg; @@ -1785,7 +1774,7 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, return 0; } -static int hisi_zip_comp_recv(struct wd_alg_driver *drv, handle_t ctx, void *comp_msg) +static int hisi_zip_comp_recv(handle_t ctx, void *comp_msg) { struct hisi_qp *qp = wd_ctx_get_priv(ctx); struct wd_comp_msg *recv_msg = comp_msg; @@ -1838,7 +1827,7 @@ static int hisi_zip_get_usage(void *param) return -WD_EINVAL; } - priv = (struct hisi_zip_ctx *)drv->priv; + priv = (struct hisi_zip_ctx *)drv->drv_data; if (!priv) return -WD_EACCES; @@ -1866,6 +1855,7 @@ static int hisi_zip_get_usage(void *param) .alg_name = (zip_alg_name),\ .calc_type = UADK_ALG_HW,\ .priority = 100,\ + .priv_size = sizeof(struct hisi_zip_ctx),\ .queue_num = ZIP_CTX_Q_NUM_DEF,\ .op_type_num = 2,\ .fallback = 0,\ @@ -1874,6 +1864,8 @@ static int hisi_zip_get_usage(void *param) .send = hisi_zip_comp_send,\ .recv = hisi_zip_comp_recv,\ .get_usage = hisi_zip_get_usage, \ + .alloc_ctx = wd_hw_alloc_ctx, \ + .free_ctx = wd_hw_free_ctx, \ } static struct wd_alg_driver zip_alg_driver[] = { diff --git a/wd_comp.c b/wd_comp.c index df32ea5..35af141 100644 --- a/wd_comp.c +++ b/wd_comp.c @@ -171,31 +171,24 @@ out_clear_ctx_config: return ret; } -static int wd_comp_uninit_nolock(void) +static void wd_comp_uninit_nolock(void) { - enum wd_status status; - - wd_alg_get_init(&wd_comp_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - /* Uninit async request pool */ wd_uninit_async_request_pool(&wd_comp_setting.pool); /* Unset config, sched, driver */ wd_clear_sched(&wd_comp_setting.sched); - - wd_alg_uninit_driver(&wd_comp_setting.config, - wd_comp_setting.driver); - - return 0; } int wd_comp_init(struct wd_ctx_config *config, struct wd_sched *sched) { + __u32 drv_count; int ret; - pthread_atfork(NULL, NULL, wd_comp_clear_status); + if (!wd_comp_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_comp_clear_status) == 0) + wd_comp_atfork_registered = true; + } ret = wd_alg_try_init(&wd_comp_setting.status); if (ret) @@ -213,10 +206,37 @@ int wd_comp_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_driver; + ret = wd_get_drv_array(WD_COMP_ALG_TYPE_V1, TASK_HW, "hisi_zip", + &wd_comp_setting.config.drv_array, &drv_count); + if (ret) { + WD_ERR("driver discovery failed!\n"); + goto out_uninit_nolock; + } + + wd_comp_setting.config.drv_count = drv_count; + wd_comp_setting.init_type = WD_TYPE_V1; + ret = wd_ctx_bind_drivers(&wd_comp_setting.config, NULL, WD_TYPE_V1); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_free_drv_array; + } + + ret = wd_alg_init_driver(&wd_comp_setting.config); + if (ret) + goto out_unbind_drivers; + wd_alg_set_init(&wd_comp_setting.status); return 0; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_comp_setting.config); +out_free_drv_array: + wd_put_drv_array(wd_comp_setting.config.drv_array, drv_count); + wd_comp_setting.config.drv_array = NULL; + wd_comp_setting.config.drv_count = 0; +out_uninit_nolock: + wd_comp_uninit_nolock(); out_clear_driver: wd_comp_close_driver(WD_TYPE_V1); out_clear_init: @@ -226,12 +246,21 @@ out_clear_init: void wd_comp_uninit(void) { - int ret; + enum wd_status status; - ret = wd_comp_uninit_nolock(); - if (ret) + wd_alg_get_init(&wd_comp_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_comp_setting.config); + wd_ctx_unbind_drivers(&wd_comp_setting.config); + + wd_put_drv_array(wd_comp_setting.config.drv_array, + wd_comp_setting.config.drv_count); + wd_comp_setting.config.drv_array = NULL; + wd_comp_setting.config.drv_count = 0; + + wd_comp_uninit_nolock(); wd_comp_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_comp_setting.status); } @@ -241,9 +270,13 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par struct wd_ctx_nums comp_ctx_num[WD_DIR_MAX] = {0}; struct wd_ctx_params comp_ctx_params = {0}; int state, ret = -WD_EINVAL; + int try_cnt = 0; bool flag; - pthread_atfork(NULL, NULL, wd_comp_clear_status); + if (!wd_comp_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_comp_clear_status) == 0) + wd_comp_atfork_registered = true; + } state = wd_alg_try_init(&wd_comp_setting.status); if (state) @@ -266,38 +299,33 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par goto out_uninit; while (ret != 0) { - memset(&wd_comp_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - - /* Get alg driver and dev name */ - wd_comp_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_comp_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); + if (try_cnt++ >= WD_INIT2_MAX_RETRY) { + WD_ERR("failed to init2 after %d retries.\n", + WD_INIT2_MAX_RETRY); goto out_dlclose; } + memset(&wd_comp_setting.config, 0, sizeof(struct wd_ctx_config_internal)); + /* Init ctx param and prepare for ctx request */ comp_ctx_params.ctx_set_num = comp_ctx_num; ret = wd_ctx_param_init(&comp_ctx_params, ctx_params, - wd_comp_setting.driver, WD_COMP_TYPE, WD_DIR_MAX); + alg, WD_COMP_TYPE, WD_DIR_MAX); if (ret) { if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_comp_setting.driver); - wd_alg_drv_unbind(wd_comp_setting.driver); continue; } - goto out_unbind_drv; + goto out_dlclose; } (void)strcpy(wd_comp_init_attrs.alg, alg); wd_comp_init_attrs.sched_type = sched_type; - wd_comp_init_attrs.driver = wd_comp_setting.driver; + wd_comp_init_attrs.task_type = task_type; wd_comp_init_attrs.ctx_params = &comp_ctx_params; wd_comp_init_attrs.alg_init = wd_comp_init_nolock; wd_comp_init_attrs.alg_poll_ctx = wd_comp_poll_ctx; ret = wd_alg_attrs_init(&wd_comp_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_comp_setting.driver); - wd_alg_drv_unbind(wd_comp_setting.driver); wd_ctx_param_uninit(&comp_ctx_params); continue; } @@ -306,15 +334,31 @@ int wd_comp_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_par } } + wd_comp_setting.init_type = WD_TYPE_V2; + ret = wd_ctx_bind_drivers(&wd_comp_setting.config, + wd_comp_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("driver binding failed!\n"); + goto out_uninit_nolock; + } + + ret = wd_alg_init_driver(&wd_comp_setting.config); + if (ret) + goto out_unbind_drivers; + wd_alg_set_init(&wd_comp_setting.status); wd_ctx_param_uninit(&comp_ctx_params); - return 0; + return ret; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_comp_setting.config); +out_uninit_nolock: + wd_comp_uninit_nolock(); + wd_alg_attrs_uninit(&wd_comp_init_attrs); out_params_uninit: wd_ctx_param_uninit(&comp_ctx_params); -out_unbind_drv: - wd_alg_drv_unbind(wd_comp_setting.driver); out_dlclose: wd_comp_close_driver(WD_TYPE_V2); out_uninit: @@ -324,14 +368,19 @@ out_uninit: void wd_comp_uninit2(void) { - int ret; + enum wd_status status; - ret = wd_comp_uninit_nolock(); - if (ret) + wd_alg_get_init(&wd_comp_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_comp_setting.config); + wd_ctx_unbind_drivers(&wd_comp_setting.config); + wd_comp_setting.config.drv_array = NULL; + wd_comp_setting.config.drv_count = 0; + wd_comp_uninit_nolock(); + wd_alg_attrs_uninit(&wd_comp_init_attrs); - wd_alg_drv_unbind(wd_comp_setting.driver); wd_comp_close_driver(WD_TYPE_V2); wd_alg_clear_init(&wd_comp_setting.status); } @@ -366,7 +415,7 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_comp_setting.driver, ctx->ctx, &resp_msg); + ret = ctx->drv->recv(ctx->ctx, &resp_msg); if (unlikely(ret < 0)) { if (ret == -WD_HW_EACCESS) WD_ERR("wd comp recv hw error!\n"); @@ -397,6 +446,11 @@ int wd_comp_poll_ctx(__u32 idx, __u32 expt, __u32 *count) static int wd_comp_check_sess_params(struct wd_comp_sess_setup *setup) { + if (!setup) { + WD_ERR("invalid: setup is NULL!\n"); + return -WD_EINVAL; + } + if (setup->alg_type >= WD_COMP_ALG_MAX) { WD_ERR("invalid: alg_type is %u!\n", setup->alg_type); return -WD_EINVAL; @@ -443,12 +497,11 @@ static void wd_free_ctx_buf(struct wd_mm_ops *mm_ops, struct wd_comp_sess *sess) handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) { + struct wd_sched_params params = {0}; + const char *target_alg_name; struct wd_comp_sess *sess; int ret; - if (!setup) - return (handle_t)0; - ret = wd_comp_check_sess_params(setup); if (ret) return (handle_t)0; @@ -457,6 +510,14 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) if (!sess) return (handle_t)0; + target_alg_name = wd_comp_alg_name[setup->alg_type]; + + ret = wd_drv_alg_support(target_alg_name, &wd_comp_setting.config); + if (!ret) { + WD_ERR("failed to support algorithm: %s!\n", target_alg_name); + goto sess_err; + } + /* Memory type set */ ret = wd_mem_ops_init(wd_comp_setting.config.ctxs[0].ctx, &setup->mm_ops, setup->mm_type); if (ret) { @@ -472,6 +533,8 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) sess->comp_lv = setup->comp_lv; sess->win_sz = setup->win_sz; sess->stream_pos = WD_COMP_STREAM_NEW; + sess->strm_ctx_fixed = false; + sess->strm_sync_ctx_idx = 0; sess->mm_type = setup->mm_type; memcpy(&sess->mm_ops, &setup->mm_ops, sizeof(struct wd_mm_ops)); @@ -484,6 +547,12 @@ handle_t wd_comp_alloc_sess(struct wd_comp_sess_setup *setup) goto sched_err; } + /* Set compat filtering parameters for session-ctx matching */ + params.alg_name = target_alg_name; + params.ctxs = wd_comp_setting.config.ctxs; + wd_comp_setting.sched.set_param(wd_comp_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); + return (handle_t)sess; sched_err: @@ -503,8 +572,14 @@ void wd_comp_free_sess(handle_t h_sess) if (sess->ctx_buf) wd_free_ctx_buf(&sess->mm_ops, sess); - if (sess->sched_key) - free(sess->sched_key); + if (sess->sched_key) { + if (wd_comp_setting.sched.sched_uninit) + wd_comp_setting.sched.sched_uninit( + wd_comp_setting.sched.h_sched_ctx, + (handle_t)sess->sched_key); + else + free(sess->sched_key); + } free(sess); } @@ -655,9 +730,23 @@ static int wd_comp_sync_job(struct wd_comp_sess *sess, __u32 idx; int ret; - idx = wd_comp_setting.sched.pick_next_ctx(h_sched_ctx, - sess->sched_key, - CTX_MODE_SYNC); + if (msg->stream_mode == WD_COMP_STATEFUL) { + if (!sess->strm_ctx_fixed) { + idx = wd_comp_setting.sched.pick_next_ctx(h_sched_ctx, + sess->sched_key, CTX_MODE_SYNC); + if (unlikely(idx >= config->ctx_num)) + return -WD_EINVAL; + + sess->strm_sync_ctx_idx = idx; + sess->strm_ctx_fixed = true; + } else { + idx = sess->strm_sync_ctx_idx; + } + } else { + idx = wd_comp_setting.sched.pick_next_ctx(h_sched_ctx, + sess->sched_key, CTX_MODE_SYNC); + } + ret = wd_check_ctx(config, CTX_MODE_SYNC, idx); if (unlikely(ret)) return ret; @@ -665,12 +754,12 @@ static int wd_comp_sync_job(struct wd_comp_sess *sess, wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); ctx = config->ctxs + idx; - msg_handle.send = wd_comp_setting.driver->send; - msg_handle.recv = wd_comp_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_comp_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, config->epoll_en); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, + NULL, config->epoll_en); pthread_spin_unlock(&ctx->lock); return ret; @@ -811,7 +900,6 @@ int wd_do_comp_strm(handle_t h_sess, struct wd_comp_req *req) msg.stream_mode = WD_COMP_STATEFUL; src_len = req->src_len; - ret = wd_comp_sync_job(sess, req, &msg); if (unlikely(ret)) return ret; @@ -860,13 +948,13 @@ int wd_do_comp_async(handle_t h_sess, struct wd_comp_req *req) tag = wd_get_msg_from_pool(&wd_comp_setting.pool, idx, (void **)&msg); if (unlikely(tag < 0)) { WD_ERR("failed to get msg from pool!\n"); - return tag; + return -WD_EBUSY; } fill_comp_msg(sess, msg, req); msg->tag = tag; msg->stream_mode = WD_COMP_STATELESS; - ret = wd_alg_driver_send(wd_comp_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("wd comp send error, ret = %d!\n", ret); diff --git a/wd_zlibwrapper.c b/wd_zlibwrapper.c index 8f32f0f..92e1f1a 100644 --- a/wd_zlibwrapper.c +++ b/wd_zlibwrapper.c @@ -39,6 +39,8 @@ enum alg_win_bits { static pthread_mutex_t wd_zlib_mutex = PTHREAD_MUTEX_INITIALIZER; static int zlib_status; +static bool wd_zlib_atfork_registered; + static void wd_zlib_unlock(void) { zlib_status = WD_ZLIB_UNINIT; @@ -236,7 +238,10 @@ static int wd_zlib_do_request(z_streamp strm, int flush, enum wd_comp_op_type ty /* === Compression === */ int wd_deflate_init(z_streamp strm, int level, int windowbits) { - pthread_atfork(NULL, NULL, wd_zlib_unlock); + if (!wd_zlib_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_zlib_unlock) == 0) + wd_zlib_atfork_registered = true; + } return wd_zlib_init(strm, level, windowbits, WD_DIR_COMPRESS); } @@ -270,7 +275,10 @@ int wd_deflate_end(z_streamp strm) /* === Decompression === */ int wd_inflate_init(z_streamp strm, int windowbits) { - pthread_atfork(NULL, NULL, wd_zlib_unlock); + if (!wd_zlib_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_zlib_unlock) == 0) + wd_zlib_atfork_registered = true; + } return wd_zlib_init(strm, 0, windowbits, WD_DIR_DECOMPRESS); } -- 2.43.0
From: Wenkai Lin <linwenkai6@hisilicon.com> For the newly added UDMA algorithm type, the API layer needs to be adapted to the new UADK heterogeneous hybrid acceleration framework, thereby ensuring that the data movement functionality can achieve hybrid acceleration combining hardware acceleration and instruction acceleration capabilities. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- drv/hisi_udma.c | 55 ++++++++++++-------------- wd_udma.c | 101 ++++++++++++++++++++++++++++++------------------ 2 files changed, 88 insertions(+), 68 deletions(-) diff --git a/drv/hisi_udma.c b/drv/hisi_udma.c index 2e5aa13..2d665e8 100644 --- a/drv/hisi_udma.c +++ b/drv/hisi_udma.c @@ -465,7 +465,7 @@ static int udma_init(void *conf, void *priv) for (i = 0; i < config->ctx_num; i++) { if (config->ctxs[i].ctx && config->ctxs[i].drv && - strcmp(config->ctxs[i].drv->drv_name, "hisi_udma") == 0) { + strcmp(config->ctxs[i].drv->drv_name, "hisi_zip") == 0) { is_match[i] = true; count++; } else { @@ -526,36 +526,34 @@ out: return ret; } -static void udma_exit(struct wd_alg_driver *drv) +static void udma_exit(void *priv) { - struct wd_ctx_config_internal *config; - struct hisi_udma_ctx *priv; + struct hisi_udma_ctx *uctx = priv; handle_t h_qp; __u32 i; - if (!drv || !drv->priv) + if (!priv) return; - priv = (struct hisi_udma_ctx *)drv->priv; - config = &priv->config; - for (i = 0; i < config->ctx_num; i++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); - udma_uninit_qp_priv(h_qp); - hisi_qm_free_qp(h_qp); + for (i = 0; i < uctx->ctx_num; i++) { + h_qp = (handle_t)wd_ctx_get_priv(uctx->ctxs[i]->ctx); + if (h_qp) { + udma_uninit_qp_priv(h_qp); + hisi_qm_free_qp(h_qp); + } + } + if (uctx->ctxs) { + free(uctx->ctxs); + uctx->ctxs = NULL; } - - free(priv); - drv->priv = NULL; } static int udma_get_usage(void *param) { struct hisi_dev_usage *udma_usage = (struct hisi_dev_usage *)param; struct wd_alg_driver *drv = udma_usage->drv; - struct wd_ctx_config_internal *config; - struct hisi_udma_ctx *priv; + struct hisi_udma_ctx *uctx; char *ctx_dev_name; - handle_t ctx = 0; handle_t qp = 0; __u32 i; @@ -564,25 +562,19 @@ static int udma_get_usage(void *param) return -WD_EINVAL; } - priv = (struct hisi_udma_ctx *)drv->priv; - if (!priv) + uctx = (struct hisi_udma_ctx *)drv->drv_data; + if (!uctx) return -WD_EACCES; - config = &priv->config; - for (i = 0; i < config->ctx_num; i++) { - ctx_dev_name = wd_ctx_get_dev_name(config->ctxs[i].ctx); + for (i = 0; i < uctx->ctx_num; i++) { + ctx_dev_name = wd_ctx_get_dev_name(uctx->ctxs[i]->ctx); if (!strcmp(udma_usage->dev_name, ctx_dev_name)) { - ctx = config->ctxs[i].ctx; - break; + qp = (handle_t)wd_ctx_get_priv(uctx->ctxs[i]->ctx); + if (qp) + return hisi_qm_get_usage(qp, 0); } } - if (ctx) - qp = (handle_t)wd_ctx_get_priv(ctx); - - if (qp) - return hisi_qm_get_usage(qp, UDMA_ALG_TYPE); - return -WD_EACCES; } @@ -591,6 +583,7 @@ static struct wd_alg_driver udma_driver = { .alg_name = "udma", .calc_type = UADK_ALG_HW, .priority = 100, + .priv_size = sizeof(struct hisi_udma_ctx), .queue_num = UDMA_CTX_Q_NUM_DEF, .op_type_num = 1, .fallback = 0, @@ -599,6 +592,8 @@ static struct wd_alg_driver udma_driver = { .send = udma_send, .recv = udma_recv, .get_usage = udma_get_usage, + .alloc_ctx = wd_hw_alloc_ctx, + .free_ctx = wd_hw_free_ctx, }; #ifdef WD_STATIC_DRV diff --git a/wd_udma.c b/wd_udma.c index eebe495..582c8d1 100644 --- a/wd_udma.c +++ b/wd_udma.c @@ -21,7 +21,6 @@ static struct wd_udma_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; void *dlhandle; void *dlh_list; } wd_udma_setting; @@ -34,7 +33,6 @@ static void wd_udma_close_driver(void) wd_dlclose_drv(wd_udma_setting.dlh_list); wd_udma_setting.dlh_list = NULL; #else - wd_release_drv(wd_udma_setting.driver); hisi_udma_remove(); #endif } @@ -67,14 +65,22 @@ void wd_udma_free_sess(handle_t sess) return; } - if (sess_t->sched_key) - free(sess_t->sched_key); + if (sess_t->sched_key) { + if (wd_udma_setting.sched.sched_uninit) + wd_udma_setting.sched.sched_uninit( + wd_udma_setting.sched.h_sched_ctx, + (handle_t)sess_t->sched_key); + else + free(sess_t->sched_key); + } free(sess_t); } handle_t wd_udma_alloc_sess(struct wd_udma_sess_setup *setup) { + struct wd_sched_params params; struct wd_udma_sess *sess; + int ret; if (!setup) { WD_ERR("invalid: alloc udma sess setup NULL!\n"); @@ -86,6 +92,12 @@ handle_t wd_udma_alloc_sess(struct wd_udma_sess_setup *setup) return (handle_t)0; sess->alg_name = "udma"; + ret = wd_drv_alg_support(sess->alg_name, &wd_udma_setting.config); + if (!ret) { + WD_ERR("failed to support udma algorithm: %s!\n", sess->alg_name); + goto free_sess; + } + /* Some simple scheduler don't need scheduling parameters */ sess->sched_key = (void *)wd_udma_setting.sched.sched_init( wd_udma_setting.sched.h_sched_ctx, setup->sched_param); @@ -93,6 +105,11 @@ handle_t wd_udma_alloc_sess(struct wd_udma_sess_setup *setup) WD_ERR("failed to init session schedule key!\n"); goto free_sess; } + params.alg_name = sess->alg_name; + params.ctxs = wd_udma_setting.config.ctxs; + wd_udma_setting.sched.set_param( + wd_udma_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); return (handle_t)sess; @@ -226,10 +243,10 @@ int wd_do_udma_sync(handle_t h_sess, struct wd_udma_req *req) fill_udma_msg(&msg, req); - msg_handle.send = wd_udma_setting.driver->send; - msg_handle.recv = wd_udma_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_udma_setting.driver, &msg_handle, ctx->ctx, + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, &msg, NULL, wd_udma_setting.config.epoll_en); pthread_spin_unlock(&ctx->lock); if (unlikely(ret)) @@ -276,7 +293,7 @@ int wd_do_udma_async(handle_t sess, struct wd_udma_req *req) fill_udma_msg(msg, req); msg->tag = mid; - ret = wd_alg_driver_send(wd_udma_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret)) { if (ret != -WD_EBUSY) WD_ERR("failed to send udma BD, hw is err!\n"); @@ -314,7 +331,7 @@ static int wd_udma_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_udma_setting.driver, ctx->ctx, &rcv_msg); + ret = ctx->drv->recv(ctx->ctx, &rcv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (unlikely(ret)) { @@ -353,6 +370,8 @@ int wd_udma_poll(__u32 expt, __u32 *count) return wd_udma_setting.sched.poll_policy(h_sched_ctx, expt, count); } +static bool wd_udma_atfork_registered = false; + static void wd_udma_clear_status(void) { wd_alg_clear_init(&wd_udma_setting.status); @@ -364,7 +383,6 @@ static void wd_udma_alg_uninit(void) wd_uninit_async_request_pool(&wd_udma_setting.pool); /* Unset config, sched, driver */ wd_clear_sched(&wd_udma_setting.sched); - wd_alg_uninit_driver(&wd_udma_setting.config, wd_udma_setting.driver); } void wd_udma_uninit(void) @@ -372,12 +390,16 @@ void wd_udma_uninit(void) enum wd_status status; wd_alg_get_init(&wd_udma_setting.status, &status); - if (status == WD_UNINIT) + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_udma_setting.config); + wd_ctx_unbind_drivers(&wd_udma_setting.config); + wd_udma_setting.config.drv_array = NULL; + wd_udma_setting.config.drv_count = 0; wd_udma_alg_uninit(); + wd_alg_attrs_uninit(&wd_udma_init_attrs); - wd_alg_drv_unbind(wd_udma_setting.driver); wd_udma_close_driver(); wd_alg_clear_init(&wd_udma_setting.status); } @@ -405,14 +427,8 @@ static int wd_udma_alg_init(struct wd_ctx_config *config, struct wd_sched *sched if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_udma_setting.config, wd_udma_setting.driver); - if (ret) - goto out_clear_pool; - return WD_SUCCESS; -out_clear_pool: - wd_uninit_async_request_pool(&wd_udma_setting.pool); out_clear_sched: wd_clear_sched(&wd_udma_setting.sched); out_clear_ctx_config: @@ -426,8 +442,12 @@ int wd_udma_init(const char *alg, __u32 sched_type, int task_type, struct wd_ctx_nums udma_ctx_num[WD_UDMA_OP_MAX] = {0}; struct wd_ctx_params udma_ctx_params = {0}; int state, ret = -WD_EINVAL; + int try_cnt = 0; - pthread_atfork(NULL, NULL, wd_udma_clear_status); + if (!wd_udma_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_udma_clear_status) == 0) + wd_udma_atfork_registered = true; + } state = wd_alg_try_init(&wd_udma_setting.status); if (state) @@ -449,39 +469,30 @@ int wd_udma_init(const char *alg, __u32 sched_type, int task_type, goto out_clear_init; while (ret) { - memset(&wd_udma_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - - /* Get alg driver and dev name */ - wd_udma_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_udma_setting.driver) { - WD_ERR("fail to bind a valid driver.\n"); - ret = -WD_EINVAL; - goto out_dlopen; + if (try_cnt++ >= WD_INIT2_MAX_RETRY) { + WD_ERR("failed to init2 after %d retries.\n", + WD_INIT2_MAX_RETRY); + goto out_driver; } - + memset(&wd_udma_setting.config, 0, sizeof(struct wd_ctx_config_internal)); udma_ctx_params.ctx_set_num = udma_ctx_num; ret = wd_ctx_param_init(&udma_ctx_params, ctx_params, - wd_udma_setting.driver, WD_UDMA_TYPE, WD_UDMA_OP_MAX); + alg, WD_UDMA_TYPE, WD_UDMA_OP_MAX); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_udma_setting.driver); - wd_alg_drv_unbind(wd_udma_setting.driver); + if (ret == -WD_EAGAIN) continue; - } goto out_driver; } (void)strcpy(wd_udma_init_attrs.alg, alg); wd_udma_init_attrs.sched_type = sched_type; - wd_udma_init_attrs.driver = wd_udma_setting.driver; + wd_udma_init_attrs.task_type = task_type; wd_udma_init_attrs.ctx_params = &udma_ctx_params; wd_udma_init_attrs.alg_init = wd_udma_alg_init; wd_udma_init_attrs.alg_poll_ctx = wd_udma_poll_ctx; ret = wd_alg_attrs_init(&wd_udma_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_udma_setting.driver); - wd_alg_drv_unbind(wd_udma_setting.driver); wd_ctx_param_uninit(&udma_ctx_params); continue; } @@ -489,17 +500,31 @@ int wd_udma_init(const char *alg, __u32 sched_type, int task_type, goto out_params_uninit; } } + ret = wd_ctx_bind_drivers(&wd_udma_setting.config, + wd_udma_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("failed to bind driver for udma!\n"); + goto out_common_uninit; + } + + ret = wd_alg_init_driver(&wd_udma_setting.config); + if (ret) + goto out_unbind_drivers; wd_alg_set_init(&wd_udma_setting.status); wd_ctx_param_uninit(&udma_ctx_params); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_udma_setting.config); +out_common_uninit: + wd_udma_alg_uninit(); + wd_alg_attrs_uninit(&wd_udma_init_attrs); out_params_uninit: wd_ctx_param_uninit(&udma_ctx_params); out_driver: - wd_alg_drv_unbind(wd_udma_setting.driver); -out_dlopen: wd_udma_close_driver(); out_clear_init: wd_alg_clear_init(&wd_udma_setting.status); -- 2.43.0
From: Wenkai Lin <linwenkai6@hisilicon.com> For big data acceleration algorithms, the API layer needs to be adapted to the new UADK heterogeneous hybrid acceleration framework, thereby ensuring that big data acceleration functionality can be adapted to and support the new hybrid acceleration framework. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> --- drv/hisi_dae.c | 50 ++++-- drv/hisi_dae.h | 8 +- drv/hisi_dae_common.c | 101 ++++++----- drv/hisi_dae_join_gather.c | 36 ++-- include/drv/wd_agg_drv.h | 6 +- include/drv/wd_join_gather_drv.h | 12 +- wd_agg.c | 239 +++++++++++++++++--------- wd_join_gather.c | 286 +++++++++++++++++++------------ 8 files changed, 459 insertions(+), 279 deletions(-) diff --git a/drv/hisi_dae.c b/drv/hisi_dae.c index d86b9f6..d495541 100644 --- a/drv/hisi_dae.c +++ b/drv/hisi_dae.c @@ -126,6 +126,8 @@ static void fill_hashagg_task_type(struct wd_agg_msg *msg, struct dae_sqe *sqe, else sqe->task_type_ext = DAE_HASHAGG_OUTPUT; break; + default: + break; } } @@ -200,6 +202,8 @@ static void fill_hashagg_table_data(struct dae_sqe *sqe, struct dae_addr_list *a hw_table = &addr_list->src_table; table_data = &agg_ctx->rehash_table; break; + default: + break; } sqe->table_row_size = agg_ctx->row_size; @@ -282,7 +286,6 @@ static void fill_hashagg_key_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext_s } } } - static void fill_hashagg_merge_key_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext_sqe, struct dae_addr_list *addr_list, struct wd_agg_msg *msg) { @@ -350,6 +353,8 @@ static void fill_hashagg_input_data(struct dae_sqe *sqe, struct dae_ext_sqe *ext agg_col_num = cols_data->output_num; fill_hashagg_data_info(sqe, ext_sqe, cols_data->input_data, cols_data->input_num); break; + default: + break; } for (i = 0; i < agg_col_num; i++) { @@ -426,7 +431,7 @@ static int check_hashagg_param(struct wd_agg_msg *msg) return WD_SUCCESS; } -static int hashagg_send(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_msg) +static int hashagg_send(handle_t ctx, void *hashagg_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -448,6 +453,10 @@ static int hashagg_send(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_m fill_hashagg_task_type(msg, &sqe, qp->q_info.hw_type); sqe.data_row_num = msg->row_count; + if (qp->q_info.hw_type >= HISI_QM_API_VER5_BASE && msg->pos == WD_AGG_STREAM_INPUT) { + sqe.init_row_num = msg->in_row_count; + sqe.break_point_en = sqe.init_row_num ? 1 : 0; + } idx = get_free_ext_addr(ext_addr); if (idx < 0) @@ -504,7 +513,7 @@ static void fill_sum_overflow_cols(struct dae_sqe *sqe, struct wd_agg_msg *msg, } static void fill_hashagg_msg_task_done(struct dae_sqe *sqe, struct wd_agg_msg *msg, - struct wd_agg_msg *temp_msg, struct hashagg_ctx *agg_ctx) + struct wd_agg_msg *temp_msg, __u16 hw_type) { if (sqe->task_type_ext == DAE_HASHAGG_OUTPUT) { msg->out_row_count = sqe->out_raw_num; @@ -515,11 +524,15 @@ static void fill_hashagg_msg_task_done(struct dae_sqe *sqe, struct wd_agg_msg *m msg->out_row_count = temp_msg->row_count; } else { msg->in_row_count = temp_msg->row_count; + if (hw_type >= HISI_QM_API_VER5_BASE) { + if (!sqe->output_end) + msg->in_row_count = sqe->data_row_offset; + } } } static void fill_hashagg_msg_task_err(struct dae_sqe *sqe, struct wd_agg_msg *msg, - struct wd_agg_msg *temp_msg, struct hashagg_ctx *agg_ctx) + struct wd_agg_msg *temp_msg, __u16 hw_type) { switch (sqe->err_type) { case DAE_TASK_BD_ERROR_MIN ... DAE_TASK_BD_ERROR_MAX: @@ -541,6 +554,11 @@ static void fill_hashagg_msg_task_err(struct dae_sqe *sqe, struct wd_agg_msg *ms case DAE_HASHAGG_RESULT_OVERFLOW: msg->in_row_count = temp_msg->row_count; msg->result = WD_AGG_SUM_OVERFLOW; + if (hw_type >= HISI_QM_API_VER5_BASE && + sqe->task_type_ext == DAE_HASH_AGGREGATE) { + if (!sqe->output_end) + msg->in_row_count = sqe->data_row_offset; + } break; case DAE_TASK_BUS_ERROR: WD_ERR("failed to do hashagg task, bus error! etype %u!\n", sqe->err_type); @@ -557,13 +575,14 @@ static void fill_hashagg_msg_task_err(struct dae_sqe *sqe, struct wd_agg_msg *ms break; } - if (sqe->task_type_ext == DAE_HASHAGG_OUTPUT) { + if (sqe->task_type_ext == DAE_HASHAGG_OUTPUT || + sqe->task_type_ext == DAE_HASHAGG_MERGE) { msg->out_row_count = sqe->out_raw_num; msg->output_done = sqe->output_end; } } -static int hashagg_recv(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_msg) +static int hashagg_recv(handle_t ctx, void *hashagg_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -606,9 +625,9 @@ static int hashagg_recv(struct wd_alg_driver *drv, handle_t ctx, void *hashagg_m msg->in_row_count = 0; if (likely(sqe.done_flag == DAE_HW_TASK_DONE)) { - fill_hashagg_msg_task_done(&sqe, msg, temp_msg, agg_ctx); + fill_hashagg_msg_task_done(&sqe, msg, temp_msg, qp->q_info.hw_type); } else if (sqe.done_flag == DAE_HW_TASK_ERR) { - fill_hashagg_msg_task_err(&sqe, msg, temp_msg, agg_ctx); + fill_hashagg_msg_task_err(&sqe, msg, temp_msg, qp->q_info.hw_type); } else { msg->result = WD_AGG_PARSE_ERROR; WD_ERR("failed to do hashagg task, hardware does not process the task!\n"); @@ -1214,17 +1233,11 @@ static void hashagg_sess_priv_uninit(struct wd_alg_driver *drv, void *priv) free(agg_ctx); } -static int hashagg_sess_priv_init(struct wd_alg_driver *drv, - struct wd_agg_sess_setup *setup, void **priv) +static int hashagg_sess_priv_init(struct wd_agg_sess_setup *setup, void **priv) { struct hashagg_ctx *agg_ctx; int ret; - if (!drv || !drv->priv) { - WD_ERR("invalid: dae drv is NULL!\n"); - return -WD_EINVAL; - } - if (!setup || !priv) { WD_ERR("invalid: dae sess priv is NULL!\n"); return -WD_EINVAL; @@ -1265,8 +1278,7 @@ static int agg_get_row_size(struct wd_alg_driver *drv, void *param) return agg_ctx->row_size; } -static int agg_hash_table_init(struct wd_alg_driver *drv, - struct wd_dae_hash_table *hash_table, void *priv) +static int agg_hash_table_init(struct wd_dae_hash_table *hash_table, void *priv) { struct hashagg_ctx *agg_ctx = priv; @@ -1297,6 +1309,8 @@ static struct wd_alg_driver hashagg_driver = { .alg_name = "hashagg", .calc_type = UADK_ALG_HW, .priority = 100, + .priv_size = sizeof(struct hisi_dae_ctx), + .ops_size = sizeof(struct wd_agg_ops), .queue_num = DAE_CTX_Q_NUM_DEF, .op_type_num = 1, .fallback = 0, @@ -1306,6 +1320,8 @@ static struct wd_alg_driver hashagg_driver = { .recv = hashagg_recv, .get_usage = dae_get_usage, .get_extend_ops = dae_get_extend_ops, + .alloc_ctx = wd_hw_alloc_ctx, + .free_ctx = wd_hw_free_ctx, }; #ifdef WD_STATIC_DRV diff --git a/drv/hisi_dae.h b/drv/hisi_dae.h index f82b13c..3ac50eb 100644 --- a/drv/hisi_dae.h +++ b/drv/hisi_dae.h @@ -14,6 +14,7 @@ #include "config.h" #include "wd_alg.h" #include "wd_dae.h" +#include "wd_drv.h" #ifdef __cplusplus extern "C" { @@ -207,11 +208,12 @@ struct hash_table_data { }; struct hisi_dae_ctx { - struct wd_ctx_config_internal config; + struct wd_ctx_internal **ctxs; + __u32 ctx_num; }; -void dae_exit(struct wd_alg_driver *drv); -int dae_init(struct wd_alg_driver *drv, void *conf); +void dae_exit(void *priv); +int dae_init(void *conf, void *priv); int dae_hash_table_init(struct hash_table_data *hw_table, struct hash_table_data *rehash_table, struct wd_dae_hash_table *hash_table, diff --git a/drv/hisi_dae_common.c b/drv/hisi_dae_common.c index c077d1d..24f6481 100644 --- a/drv/hisi_dae_common.c +++ b/drv/hisi_dae_common.c @@ -308,36 +308,60 @@ update_table: return ret; } -int dae_init(struct wd_alg_driver *drv, void *conf) +int dae_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; + struct hisi_dae_ctx *dae_ctx = priv; struct hisi_qm_priv qm_priv; - struct hisi_dae_ctx *priv; - handle_t h_qp = 0; - handle_t h_ctx; - __u32 i, j; + __u32 i, j, count = 0; + bool *is_match; + handle_t h_qp; int ret; if (!config || !config->ctx_num) { - WD_ERR("invalid: dae init config is null or ctx num is 0!\n"); + WD_ERR("invalid: input config or ctx num is null!\n"); + return -WD_EINVAL; + } + + is_match = malloc(config->ctx_num * sizeof(bool)); + if (!is_match) + return -WD_ENOMEM; + + for (i = 0; i < config->ctx_num; i++) { + if (config->ctxs[i].ctx && config->ctxs[i].drv && + !strcmp(config->ctxs[i].drv->drv_name, "hisi_zip")) { + is_match[i] = true; + count++; + } else { + is_match[i] = false; + } + } + if (!count) { + WD_ERR("invalid: valid driver number is zero!\n"); + free(is_match); return -WD_EINVAL; } - priv = malloc(sizeof(struct hisi_dae_ctx)); - if (!priv) + dae_ctx->ctxs = calloc(count, sizeof(struct wd_ctx_internal *)); + if (!dae_ctx->ctxs) { + free(is_match); return -WD_ENOMEM; + } + dae_ctx->ctx_num = count; qm_priv.op_type = DAE_SQC_ALG_TYPE; qm_priv.sqe_size = sizeof(struct dae_sqe); + count = 0; /* Allocate qp for each context */ for (i = 0; i < config->ctx_num; i++) { - h_ctx = config->ctxs[i].ctx; + if (!is_match[i]) + continue; qm_priv.qp_mode = config->ctxs[i].ctx_mode; /* Setting the epoll en to 0 for ASYNC ctx */ qm_priv.epoll_en = (qm_priv.qp_mode == CTX_MODE_SYNC) ? config->epoll_en : 0; qm_priv.idx = i; - h_qp = hisi_qm_alloc_qp(&qm_priv, h_ctx); + h_qp = hisi_qm_alloc_qp(&qm_priv, config->ctxs[i].ctx); if (!h_qp) { ret = -WD_ENOMEM; goto out; @@ -346,58 +370,57 @@ int dae_init(struct wd_alg_driver *drv, void *conf) ret = dae_init_qp_priv(h_qp); if (ret) goto free_h_qp; + dae_ctx->ctxs[count++] = &config->ctxs[i]; } - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + free(is_match); return WD_SUCCESS; free_h_qp: hisi_qm_free_qp(h_qp); out: - for (j = 0; j < i; j++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[j].ctx); + for (j = 0; j < count; j++) { + h_qp = (handle_t)wd_ctx_get_priv(dae_ctx->ctxs[j]->ctx); if (h_qp) { dae_uninit_qp_priv(h_qp); hisi_qm_free_qp(h_qp); } } - free(priv); + free(dae_ctx->ctxs); + free(is_match); return ret; } -void dae_exit(struct wd_alg_driver *drv) +void dae_exit(void *priv) { - struct wd_ctx_config_internal *config; - struct hisi_dae_ctx *priv; + struct hisi_dae_ctx *dae_ctx = priv; handle_t h_qp; __u32 i; - if (!drv || !drv->priv) + if (!priv) { + WD_ERR("invalid: input parameter is NULL!\n"); return; + } - priv = (struct hisi_dae_ctx *)drv->priv; - config = &priv->config; - for (i = 0; i < config->ctx_num; i++) { - h_qp = (handle_t)wd_ctx_get_priv(config->ctxs[i].ctx); + for (i = 0; i < dae_ctx->ctx_num; i++) { + h_qp = (handle_t)wd_ctx_get_priv(dae_ctx->ctxs[i]->ctx); if (h_qp) { dae_uninit_qp_priv(h_qp); hisi_qm_free_qp(h_qp); } } - - free(priv); - drv->priv = NULL; + if (dae_ctx->ctxs) { + free(dae_ctx->ctxs); + dae_ctx->ctxs = NULL; + } } int dae_get_usage(void *param) { struct hisi_dev_usage *dae_usage = (struct hisi_dev_usage *)param; struct wd_alg_driver *drv = dae_usage->drv; - struct wd_ctx_config_internal *config; - struct hisi_dae_ctx *priv; + struct hisi_dae_ctx *dae_ctx; char *ctx_dev_name; - handle_t ctx = 0; handle_t qp = 0; __u32 i; @@ -406,24 +429,18 @@ int dae_get_usage(void *param) return -WD_EINVAL; } - priv = (struct hisi_dae_ctx *)drv->priv; - if (!priv) + dae_ctx = (struct hisi_dae_ctx *)drv->drv_data; + if (!dae_ctx) return -WD_EACCES; - config = &priv->config; - for (i = 0; i < config->ctx_num; i++) { - ctx_dev_name = wd_ctx_get_dev_name(config->ctxs[i].ctx); + for (i = 0; i < dae_ctx->ctx_num; i++) { + ctx_dev_name = wd_ctx_get_dev_name(dae_ctx->ctxs[i]->ctx); if (!strcmp(dae_usage->dev_name, ctx_dev_name)) { - ctx = config->ctxs[i].ctx; - break; + qp = (handle_t)wd_ctx_get_priv(dae_ctx->ctxs[i]->ctx); + if (qp) + return hisi_qm_get_usage(qp, 0); } } - if (ctx) - qp = (handle_t)wd_ctx_get_priv(ctx); - - if (qp) - return hisi_qm_get_usage(qp, DAE_SQC_ALG_TYPE); - return -WD_EACCES; } diff --git a/drv/hisi_dae_join_gather.c b/drv/hisi_dae_join_gather.c index 5a96d76..98c998f 100644 --- a/drv/hisi_dae_join_gather.c +++ b/drv/hisi_dae_join_gather.c @@ -455,7 +455,7 @@ static int check_join_gather_param(struct wd_join_gather_msg *msg) return WD_SUCCESS; } -static int join_gather_send(struct wd_alg_driver *drv, handle_t ctx, void *send_msg) +static int join_gather_send(handle_t ctx, void *send_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(ctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -540,7 +540,7 @@ static void fill_join_gather_task_err(struct dae_sqe *sqe, struct wd_join_gather } } -static int join_gather_recv(struct wd_alg_driver *drv, handle_t hctx, void *recv_msg) +static int join_gather_recv(handle_t hctx, void *recv_msg) { handle_t h_qp = (handle_t)wd_ctx_get_priv(hctx); struct hisi_qp *qp = (struct hisi_qp *)h_qp; @@ -861,6 +861,17 @@ static void gather_get_batch_rowsize(struct join_gather_col_data *cols_data, gather_data[i].data_info); batch_row_size[n] = row_count_size + GATHER_ROW_BATCH_EMPTY_SIZE; + /* Align batch_row_size for better gather performance: + * 1-63B -> 32B aligned + * 64-127B -> 64B aligned + * >=128B -> 128B aligned + */ + if (batch_row_size[n] < ROW_SIZE64) + batch_row_size[n] = ALIGN(batch_row_size[n], ROW_SIZE32); + else if (batch_row_size[n] < ROW_SIZE128) + batch_row_size[n] = ALIGN(batch_row_size[n], ROW_SIZE64); + else + batch_row_size[n] = ALIGN(batch_row_size[n], ROW_SIZE128); } } @@ -892,7 +903,7 @@ static int join_gather_fill_ctx(struct join_gather_ctx *ctx, return WD_SUCCESS; } -static void join_gather_sess_priv_uninit(struct wd_alg_driver *drv, void *priv) +static void join_gather_sess_priv_uninit(void *priv) { struct join_gather_ctx *ctx = priv; @@ -904,17 +915,11 @@ static void join_gather_sess_priv_uninit(struct wd_alg_driver *drv, void *priv) free(ctx); } -static int join_gather_sess_priv_init(struct wd_alg_driver *drv, - struct wd_join_gather_sess_setup *setup, void **priv) +static int join_gather_sess_priv_init(struct wd_join_gather_sess_setup *setup, void **priv) { struct join_gather_ctx *ctx; int ret; - if (!drv || !drv->priv) { - WD_ERR("invalid: dae drv is NULL!\n"); - return -WD_EINVAL; - } - if (!setup || !priv) { WD_ERR("invalid: dae sess priv is NULL!\n"); return -WD_EINVAL; @@ -941,7 +946,7 @@ free_ctx: return ret; } -static int join_get_table_row_size(struct wd_alg_driver *drv, void *param) +static int join_get_table_row_size(void *param) { struct join_gather_ctx *ctx = param; @@ -951,7 +956,7 @@ static int join_get_table_row_size(struct wd_alg_driver *drv, void *param) return ctx->hash_table_row_size; } -static int gather_get_batch_row_size(struct wd_alg_driver *drv, void *param, +static int gather_get_batch_row_size(void *param, __u32 *row_size, __u32 size) { struct join_gather_ctx *ctx = param; @@ -967,8 +972,7 @@ static int gather_get_batch_row_size(struct wd_alg_driver *drv, void *param, return 0; } -static int join_hash_table_init(struct wd_alg_driver *drv, - struct wd_dae_hash_table *table, void *priv) +static int join_hash_table_init(struct wd_dae_hash_table *table, void *priv) { struct join_gather_ctx *ctx = priv; @@ -1002,6 +1006,8 @@ static int join_gather_get_extend_ops(void *ops) .alg_name = (dae_alg_name),\ .calc_type = UADK_ALG_HW,\ .priority = 100,\ + .priv_size = sizeof(struct hisi_dae_ctx), \ + .ops_size = sizeof(struct wd_join_gather_ops), \ .queue_num = DAE_CTX_Q_NUM_DEF,\ .op_type_num = 1,\ .fallback = 0,\ @@ -1011,6 +1017,8 @@ static int join_gather_get_extend_ops(void *ops) .recv = join_gather_recv,\ .get_extend_ops = join_gather_get_extend_ops,\ .get_usage = dae_get_usage,\ + .alloc_ctx = wd_hw_alloc_ctx,\ + .free_ctx = wd_hw_free_ctx, \ } static struct wd_alg_driver join_gather_driver[] = { diff --git a/include/drv/wd_agg_drv.h b/include/drv/wd_agg_drv.h index b26b25d..400fc09 100644 --- a/include/drv/wd_agg_drv.h +++ b/include/drv/wd_agg_drv.h @@ -43,11 +43,9 @@ struct wd_agg_msg { struct wd_agg_ops { int (*get_row_size)(struct wd_alg_driver *drv, void *priv); - int (*sess_init)(struct wd_alg_driver *drv, - struct wd_agg_sess_setup *setup, void **priv); + int (*sess_init)(struct wd_agg_sess_setup *setup, void **priv); void (*sess_uninit)(struct wd_alg_driver *drv, void *priv); - int (*hash_table_init)(struct wd_alg_driver *drv, - struct wd_dae_hash_table *hash_table, void *priv); + int (*hash_table_init)(struct wd_dae_hash_table *hash_table, void *priv); }; struct wd_agg_msg *wd_agg_get_msg(__u32 idx, __u32 tag); diff --git a/include/drv/wd_join_gather_drv.h b/include/drv/wd_join_gather_drv.h index dbf4ee7..d7f136d 100644 --- a/include/drv/wd_join_gather_drv.h +++ b/include/drv/wd_join_gather_drv.h @@ -33,14 +33,12 @@ struct wd_join_gather_msg { }; struct wd_join_gather_ops { - int (*get_table_row_size)(struct wd_alg_driver *drv, void *priv); - int (*get_batch_row_size)(struct wd_alg_driver *drv, void *priv, + int (*get_table_row_size)(void *priv); + int (*get_batch_row_size)(void *priv, __u32 *batch_row_size, __u32 size); - int (*sess_init)(struct wd_alg_driver *drv, - struct wd_join_gather_sess_setup *setup, void **priv); - void (*sess_uninit)(struct wd_alg_driver *drv, void *priv); - int (*hash_table_init)(struct wd_alg_driver *drv, - struct wd_dae_hash_table *hash_table, void *priv); + int (*sess_init)(struct wd_join_gather_sess_setup *setup, void **priv); + void (*sess_uninit)(void *priv); + int (*hash_table_init)(struct wd_dae_hash_table *hash_table, void *priv); }; struct wd_join_gather_msg *wd_join_gather_get_msg(__u32 idx, __u32 tag); diff --git a/wd_agg.c b/wd_agg.c index bb834cb..95cb869 100644 --- a/wd_agg.c +++ b/wd_agg.c @@ -32,8 +32,6 @@ struct wd_agg_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; - void *priv; void *dlhandle; void *dlh_list; } wd_agg_setting; @@ -78,7 +76,6 @@ static void wd_agg_close_driver(void) wd_dlclose_drv(wd_agg_setting.dlh_list); wd_agg_setting.dlh_list = NULL; #else - wd_release_drv(wd_agg_setting.driver); hisi_dae_remove(); #endif } @@ -353,27 +350,41 @@ out_key: return -WD_ENOMEM; } -static int wd_agg_init_sess_priv(struct wd_agg_sess *sess, struct wd_agg_sess_setup *setup) +static int init_single_driver_session(struct wd_alg_driver *drv, + struct wd_agg_sess *sess, + struct wd_agg_sess_setup *setup) { + struct wd_agg_ops *eops; int ret; - if (sess->ops.sess_init) { - if (!sess->ops.sess_uninit) { + if (!drv->get_extend_ops || !drv->extend_ops) + return WD_SUCCESS; + + ret = drv->get_extend_ops(drv->extend_ops); + if (ret) { + WD_ERR("failed to set session extend ops!\n"); + return -WD_EINVAL; + } + + eops = drv->extend_ops; + + if (eops->sess_init) { + if (!eops->sess_uninit) { WD_ERR("failed to get session uninit ops!\n"); return -WD_EINVAL; } - ret = sess->ops.sess_init(wd_agg_setting.driver, setup, &sess->priv); + ret = eops->sess_init(setup, &sess->priv); if (ret) { WD_ERR("failed to init session priv!\n"); return ret; } } - if (sess->ops.get_row_size) { - ret = sess->ops.get_row_size(wd_agg_setting.driver, sess->priv); + if (eops->get_row_size && sess->priv) { + ret = eops->get_row_size(drv, sess->priv); if (ret <= 0) { - if (sess->ops.sess_uninit) - sess->ops.sess_uninit(wd_agg_setting.driver, sess->priv); + if (eops->sess_uninit) + eops->sess_uninit(drv, sess->priv); WD_ERR("failed to get hash table row size: %d!\n", ret); return -WD_EINVAL; } @@ -383,10 +394,61 @@ static int wd_agg_init_sess_priv(struct wd_agg_sess *sess, struct wd_agg_sess_se return WD_SUCCESS; } +static void cleanup_partial_init(struct wd_agg_sess *sess) +{ + struct wd_ctx_config_internal *config = &wd_agg_setting.config; + struct wd_alg_driver *drv; + struct wd_agg_ops *eops; + + if (!sess->priv) + return; + + drv = config->ctxs[0].drv; + if (drv->extend_ops) { + eops = drv->extend_ops; + if (eops->sess_uninit) + eops->sess_uninit(drv, sess->priv); + } +} + +static int wd_agg_init_sess_priv(struct wd_agg_sess *sess, struct wd_agg_sess_setup *setup) +{ + struct wd_ctx_config_internal *config = &wd_agg_setting.config; + struct wd_alg_driver *first_drv = NULL; + __u32 i; + + /* + * The algorithm always works in stream mode, so it supports only + * the switchover between ctxs of the same driver type. + */ + for (i = 0; i < config->ctx_num; i++) { + if (!first_drv) { + first_drv = config->ctxs[i].drv; + } else if (config->ctxs[i].drv != first_drv) { + WD_ERR("incompatible driver: %s vs %s, only single driver type allowed!\n", + config->ctxs[i].drv->drv_name, first_drv->drv_name); + return -WD_EINVAL; + } + } + + return init_single_driver_session(first_drv, sess, setup); +} + +static int wd_agg_uninit_sess_priv(struct wd_agg_sess *sess) +{ + if (!sess || !sess->priv) + return WD_SUCCESS; + + cleanup_partial_init(sess); + sess->priv = NULL; + return WD_SUCCESS; +} + handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) { - __u32 out_agg_cols_num = 0; + struct wd_sched_params params; struct wd_agg_sess *sess; + __u32 out_agg_cols_num = 0; int ret; ret = wd_agg_check_sess_params(setup, &out_agg_cols_num); @@ -402,7 +464,7 @@ handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) sess->agg_conf.out_cols_num = out_agg_cols_num; sess->alg_name = wd_agg_alg_name; - ret = wd_drv_alg_support(sess->alg_name, wd_agg_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, &wd_agg_setting.config); if (!ret) { WD_ERR("failed to support agg algorithm: %s!\n", sess->alg_name); goto free_sess; @@ -415,14 +477,12 @@ handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) WD_ERR("failed to init agg session schedule key!\n"); goto free_sess; } + params.alg_name = sess->alg_name; + params.ctxs = wd_agg_setting.config.ctxs; + wd_agg_setting.sched.set_param( + wd_agg_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); - if (wd_agg_setting.driver->get_extend_ops) { - ret = wd_agg_setting.driver->get_extend_ops(&sess->ops); - if (ret) { - WD_ERR("failed to get agg extend ops!\n"); - goto free_key; - } - } ret = wd_agg_init_sess_priv(sess, setup); if (ret) @@ -437,10 +497,16 @@ handle_t wd_agg_alloc_sess(struct wd_agg_sess_setup *setup) return (handle_t)sess; uninit_priv: - if (sess->ops.sess_uninit) - sess->ops.sess_uninit(wd_agg_setting.driver, sess->priv); + wd_agg_uninit_sess_priv(sess); free_key: - free(sess->sched_key); + if (sess->sched_key) { + if (wd_agg_setting.sched.sched_uninit) + wd_agg_setting.sched.sched_uninit( + wd_agg_setting.sched.h_sched_ctx, + (handle_t)sess->sched_key); + else + free(sess->sched_key); + } free_sess: free(sess); return (handle_t)0; @@ -459,10 +525,15 @@ void wd_agg_free_sess(handle_t h_sess) free(sess->agg_conf.cols_info); free(sess->key_conf.data_size); - if (sess->ops.sess_uninit) - sess->ops.sess_uninit(wd_agg_setting.driver, sess->priv); - if (sess->sched_key) - free(sess->sched_key); + wd_agg_uninit_sess_priv(sess); + if (sess->sched_key) { + if (wd_agg_setting.sched.sched_uninit) + wd_agg_setting.sched.sched_uninit( + wd_agg_setting.sched.h_sched_ctx, + (handle_t)sess->sched_key); + else + free(sess->sched_key); + } free(sess); } @@ -510,8 +581,10 @@ static int wd_agg_check_sess_state(struct wd_agg_sess *sess, enum wd_agg_sess_st int wd_agg_set_hash_table(handle_t h_sess, struct wd_dae_hash_table *info) { struct wd_agg_sess *sess = (struct wd_agg_sess *)h_sess; + struct wd_ctx_config_internal *config = &wd_agg_setting.config; struct wd_dae_hash_table *hash_table, *rehash_table; enum wd_agg_sess_state expected; + struct wd_agg_ops *eops; int ret; if (!sess || !info) { @@ -548,12 +621,12 @@ int wd_agg_set_hash_table(handle_t h_sess, struct wd_dae_hash_table *info) hash_table = &sess->hash_table; rehash_table = &sess->rehash_table; - memcpy(rehash_table, hash_table, sizeof(struct wd_dae_hash_table)); memcpy(hash_table, info, sizeof(struct wd_dae_hash_table)); - if (sess->ops.hash_table_init) { - ret = sess->ops.hash_table_init(wd_agg_setting.driver, hash_table, sess->priv); + eops = config->ctxs[0].drv->extend_ops; + if (eops && eops->hash_table_init) { + ret = eops->hash_table_init(hash_table, sess->priv); if (ret) { memcpy(hash_table, rehash_table, sizeof(struct wd_dae_hash_table)); memset(rehash_table, 0, sizeof(struct wd_dae_hash_table)); @@ -562,12 +635,13 @@ int wd_agg_set_hash_table(handle_t h_sess, struct wd_dae_hash_table *info) } return WD_SUCCESS; - out: __atomic_store_n(&sess->state, expected, __ATOMIC_RELEASE); return ret; } +static bool wd_agg_atfork_registered; + static void wd_agg_clear_status(void) { wd_alg_clear_init(&wd_agg_setting.status); @@ -596,14 +670,8 @@ static int wd_agg_alg_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_agg_setting.config, wd_agg_setting.driver); - if (ret) - goto out_clear_pool; - return WD_SUCCESS; -out_clear_pool: - wd_uninit_async_request_pool(&wd_agg_setting.pool); out_clear_sched: wd_clear_sched(&wd_agg_setting.sched); out_clear_ctx_config: @@ -611,23 +679,12 @@ out_clear_ctx_config: return ret; } -static int wd_agg_alg_uninit(void) +static void wd_agg_alg_uninit(void) { - enum wd_status status; - - wd_alg_get_init(&wd_agg_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - /* Uninit async request pool */ wd_uninit_async_request_pool(&wd_agg_setting.pool); - /* Unset config, sched, driver */ wd_clear_sched(&wd_agg_setting.sched); - - wd_alg_uninit_driver(&wd_agg_setting.config, wd_agg_setting.driver); - - return WD_SUCCESS; } int wd_agg_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params *ctx_params) @@ -635,16 +692,20 @@ int wd_agg_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params struct wd_ctx_params agg_ctx_params = {0}; struct wd_ctx_nums agg_ctx_num = {0}; int ret = -WD_EINVAL; + int try_cnt = 0; int state; bool flag; - pthread_atfork(NULL, NULL, wd_agg_clear_status); + if (!wd_agg_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_agg_clear_status) == 0) + wd_agg_atfork_registered = true; + } state = wd_alg_try_init(&wd_agg_setting.status); if (state) return state; - if (!alg || sched_type >= SCHED_POLICY_BUTT || + if (!alg || sched_type >= SCHED_POLICY_BUTT || task_type == TASK_MIX || task_type < 0 || task_type >= TASK_MAX_TYPE) { WD_ERR("invalid: agg init input param is wrong!\n"); goto out_uninit; @@ -661,38 +722,30 @@ int wd_agg_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params goto out_uninit; while (ret != 0) { - memset(&wd_agg_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - - /* Get alg driver and dev name */ - wd_agg_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_agg_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; + if (try_cnt++ >= WD_INIT2_MAX_RETRY) { + WD_ERR("failed to init2 after %d retries.\n", + WD_INIT2_MAX_RETRY); + goto out_driver; } - + memset(&wd_agg_setting.config, 0, sizeof(struct wd_ctx_config_internal)); agg_ctx_params.ctx_set_num = &agg_ctx_num; - ret = wd_ctx_param_init(&agg_ctx_params, ctx_params, wd_agg_setting.driver, + ret = wd_ctx_param_init(&agg_ctx_params, ctx_params, alg, WD_AGG_TYPE, 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_agg_setting.driver); - wd_alg_drv_unbind(wd_agg_setting.driver); + if (ret == -WD_EAGAIN) continue; - } goto out_driver; } (void)strcpy(wd_agg_init_attrs.alg, alg); wd_agg_init_attrs.sched_type = sched_type; - wd_agg_init_attrs.driver = wd_agg_setting.driver; + wd_agg_init_attrs.task_type = task_type; wd_agg_init_attrs.ctx_params = &agg_ctx_params; wd_agg_init_attrs.alg_init = wd_agg_alg_init; wd_agg_init_attrs.alg_poll_ctx = wd_agg_poll_ctx; ret = wd_alg_attrs_init(&wd_agg_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_agg_setting.driver); - wd_alg_drv_unbind(wd_agg_setting.driver); wd_ctx_param_uninit(&agg_ctx_params); continue; } @@ -700,17 +753,31 @@ int wd_agg_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params goto out_params_uninit; } } + ret = wd_ctx_bind_drivers(&wd_agg_setting.config, + wd_agg_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("failed to bind driver for hashagg!\n"); + goto out_common_uninit; + } + + ret = wd_alg_init_driver(&wd_agg_setting.config); + if (ret) + goto out_unbind_drivers; wd_alg_set_init(&wd_agg_setting.status); wd_ctx_param_uninit(&agg_ctx_params); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_agg_setting.config); +out_common_uninit: + wd_agg_alg_uninit(); + wd_alg_attrs_uninit(&wd_agg_init_attrs); out_params_uninit: wd_ctx_param_uninit(&agg_ctx_params); out_driver: - wd_alg_drv_unbind(wd_agg_setting.driver); -out_dlopen: wd_agg_close_driver(); out_uninit: wd_alg_clear_init(&wd_agg_setting.status); @@ -719,14 +786,19 @@ out_uninit: void wd_agg_uninit(void) { - int ret; + enum wd_status status; - ret = wd_agg_alg_uninit(); - if (ret) + wd_alg_get_init(&wd_agg_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_agg_setting.config); + wd_ctx_unbind_drivers(&wd_agg_setting.config); + wd_agg_setting.config.drv_array = NULL; + wd_agg_setting.config.drv_count = 0; + wd_agg_alg_uninit(); + wd_alg_attrs_uninit(&wd_agg_init_attrs); - wd_alg_drv_unbind(wd_agg_setting.driver); wd_agg_close_driver(); wd_alg_clear_init(&wd_agg_setting.status); } @@ -740,7 +812,6 @@ static void fill_request_msg_input(struct wd_agg_msg *msg, struct wd_agg_req *re msg->agg_cols_num = sess->agg_conf.cols_num; memcpy(&msg->hash_table, &sess->hash_table, sizeof(struct wd_dae_hash_table)); msg->row_count = req->in_row_count; - msg->priv = sess->priv; if (!is_rehash) { msg->pos = WD_AGG_STREAM_INPUT; msg->agg_cols_info = sess->agg_conf.cols_info; @@ -759,7 +830,6 @@ static void fill_request_msg_output(struct wd_agg_msg *msg, struct wd_agg_req *r msg->key_cols_num = sess->key_conf.cols_num; msg->agg_cols_num = sess->agg_conf.cols_num; - msg->priv = sess->priv; if (!is_rehash) { msg->pos = WD_AGG_STREAM_OUTPUT; msg->is_count_all = sess->agg_conf.is_count_all; @@ -1097,13 +1167,13 @@ static int wd_agg_sync_job(struct wd_agg_sess *sess, struct wd_agg_req *req, wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); ctx = config->ctxs + idx; + msg->priv = sess->priv; - msg_handle.send = wd_agg_setting.driver->send; - msg_handle.recv = wd_agg_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(wd_agg_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, config->epoll_en); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, NULL, config->epoll_en); pthread_spin_unlock(&ctx->lock); return ret; @@ -1194,17 +1264,16 @@ static int wd_agg_async_job(struct wd_agg_sess *sess, struct wd_agg_req *req, bo ctx = config->ctxs + idx; msg_id = wd_get_msg_from_pool(&wd_agg_setting.pool, idx, (void **)&msg); - if (unlikely(msg_id < 0)) { - WD_ERR("failed to get agg msg from pool!\n"); - return msg_id; - } + if (unlikely(msg_id < 0)) + return -WD_EBUSY; if (is_input) fill_request_msg_input(msg, req, sess, false); else fill_request_msg_output(msg, req, sess, false); + msg->priv = sess->priv; msg->tag = msg_id; - ret = wd_alg_driver_send(wd_agg_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("wd agg async send err!\n"); @@ -1544,7 +1613,7 @@ static int wd_agg_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_agg_setting.driver, ctx->ctx, &resp_msg); + ret = ctx->drv->recv(ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) { return ret; } else if (unlikely(ret < 0)) { diff --git a/wd_join_gather.c b/wd_join_gather.c index 37e0022..8857c8f 100644 --- a/wd_join_gather.c +++ b/wd_join_gather.c @@ -32,8 +32,6 @@ struct wd_join_gather_setting { struct wd_ctx_config_internal config; struct wd_sched sched; struct wd_async_msg_pool pool; - struct wd_alg_driver *driver; - void *priv; void *dlhandle; void *dlh_list; }; @@ -56,7 +54,6 @@ struct wd_join_gather_sess { enum multi_batch_index_type index_type; enum wd_join_sess_state state; enum wd_join_gather_alg alg; - struct wd_join_gather_ops ops; struct wd_join_cols_conf join_conf; struct wd_gather_tables_conf gather_conf; struct wd_dae_hash_table hash_table; @@ -79,7 +76,6 @@ static void wd_join_gather_close_driver(void) wd_dlclose_drv(wd_join_gather_setting.dlh_list); wd_join_gather_setting.dlh_list = NULL; #else - wd_release_drv(wd_join_gather_setting.driver); hisi_dae_join_gather_remove(); #endif } @@ -404,70 +400,140 @@ free_join: return -WD_ENOMEM; } +static void cleanup_partial_init(struct wd_join_gather_sess *sess) +{ + struct wd_ctx_config_internal *config = &wd_join_gather_setting.config; + struct wd_alg_driver *drv; + struct wd_join_gather_ops *eops; + + if (!sess->priv) + return; + + drv = config->ctxs[0].drv; + if (drv->extend_ops) { + eops = drv->extend_ops; + if (eops->sess_uninit) + eops->sess_uninit(sess->priv); + } +} + static void wd_join_gather_uninit_sess(struct wd_join_gather_sess *sess) { - if (sess->gather_conf.batch_row_size) - free(sess->gather_conf.batch_row_size); + if (!sess) + return; - if (sess->ops.sess_uninit) - sess->ops.sess_uninit(wd_join_gather_setting.driver, sess->priv); + cleanup_partial_init(sess); + free(sess->gather_conf.batch_row_size); + sess->gather_conf.batch_row_size = NULL; } -static int wd_join_gather_init_sess(struct wd_join_gather_sess *sess, - struct wd_join_gather_sess_setup *setup) +static int handle_algorithm_specific_init(struct wd_join_gather_ops *eops, + struct wd_join_gather_sess *sess, + struct wd_join_gather_sess_setup *setup) { - struct wd_alg_driver *drv = wd_join_gather_setting.driver; __u32 array_size; int ret; - if (sess->ops.sess_init) { - if (!sess->ops.sess_uninit) { - WD_ERR("failed to get session uninit ops!\n"); - return -WD_EINVAL; - } - ret = sess->ops.sess_init(drv, setup, &sess->priv); - if (ret) { - WD_ERR("failed to init session priv!\n"); - return ret; - } - } + if (!sess->priv) + return WD_SUCCESS; - if (sess->ops.get_table_row_size && setup->alg != WD_GATHER) { - ret = sess->ops.get_table_row_size(drv, sess->priv); + if (eops->get_table_row_size && setup->alg != WD_GATHER) { + ret = eops->get_table_row_size(sess->priv); if (ret <= 0) { WD_ERR("failed to get hash table row size: %d!\n", ret); - goto uninit; + return -WD_EINVAL; } sess->hash_table.table_row_size = ret; } - if (sess->ops.get_batch_row_size && setup->alg != WD_JOIN) { + if (eops->get_batch_row_size && setup->alg != WD_JOIN) { array_size = setup->gather_table_num * sizeof(__u32); - sess->gather_conf.batch_row_size = malloc(array_size); - if (!sess->gather_conf.batch_row_size) - goto uninit; + if (!sess->gather_conf.batch_row_size) { + sess->gather_conf.batch_row_size = malloc(array_size); + if (!sess->gather_conf.batch_row_size) + return -WD_ENOMEM; + ret = eops->get_batch_row_size(sess->priv, + sess->gather_conf.batch_row_size, + array_size); + if (ret) { + free(sess->gather_conf.batch_row_size); + sess->gather_conf.batch_row_size = NULL; + WD_ERR("failed to get batch table row size!\n"); + return -WD_EINVAL; + } + } + } + + return WD_SUCCESS; +} + +static int init_single_driver_session(struct wd_alg_driver *drv, + struct wd_join_gather_sess *sess, + struct wd_join_gather_sess_setup *setup) +{ + struct wd_join_gather_ops *eops; + int ret; + + if (!drv->get_extend_ops || !drv->extend_ops) + return WD_SUCCESS; + + ret = drv->get_extend_ops(drv->extend_ops); + if (ret) { + WD_ERR("failed to set session extend ops!\n"); + return -WD_EINVAL; + } - ret = sess->ops.get_batch_row_size(drv, sess->priv, - sess->gather_conf.batch_row_size, - array_size); + eops = drv->extend_ops; + + if (eops->sess_init) { + if (!eops->sess_uninit) { + WD_ERR("failed to get session uninit ops!\n"); + return -WD_EINVAL; + } + ret = eops->sess_init(setup, &sess->priv); if (ret) { - WD_ERR("failed to get batch table row size!\n"); - goto free_batch; + WD_ERR("failed to init session priv!\n"); + return ret; } } + ret = handle_algorithm_specific_init(eops, sess, setup); + if (ret != WD_SUCCESS) { + if (eops->sess_uninit) + eops->sess_uninit(sess->priv); + return ret; + } + return WD_SUCCESS; +} -free_batch: - free(sess->gather_conf.batch_row_size); -uninit: - if (sess->ops.sess_uninit) - sess->ops.sess_uninit(drv, sess->priv); - return -WD_EINVAL; +static int wd_join_gather_init_sess(struct wd_join_gather_sess *sess, + struct wd_join_gather_sess_setup *setup) +{ + struct wd_ctx_config_internal *config = &wd_join_gather_setting.config; + struct wd_alg_driver *first_drv = NULL; + __u32 i; + + /* + * The algorithm always works in stream mode, so it supports only + * the switchover between ctxs of the same driver type. + */ + for (i = 0; i < config->ctx_num; i++) { + if (!first_drv) { + first_drv = config->ctxs[i].drv; + } else if (config->ctxs[i].drv != first_drv) { + WD_ERR("incompatible driver: %s vs %s, only single driver type allowed!\n", + config->ctxs[i].drv->drv_name, first_drv->drv_name); + return -WD_EINVAL; + } + } + + return init_single_driver_session(first_drv, sess, setup); } handle_t wd_join_gather_alloc_sess(struct wd_join_gather_sess_setup *setup) { + struct wd_sched_params params; struct wd_join_gather_sess *sess; int ret; @@ -486,7 +552,7 @@ handle_t wd_join_gather_alloc_sess(struct wd_join_gather_sess_setup *setup) sess->index_type = setup->index_type; sess->join_conf.key_output_enable = setup->join_table.key_output_enable; - ret = wd_drv_alg_support(wd_join_gather_alg[sess->alg], wd_join_gather_setting.driver); + ret = wd_drv_alg_support(wd_join_gather_alg[sess->alg], &wd_join_gather_setting.config); if (!ret) { WD_ERR("failed to check driver alg: %s!\n", wd_join_gather_alg[sess->alg]); goto free_sess; @@ -499,14 +565,11 @@ handle_t wd_join_gather_alloc_sess(struct wd_join_gather_sess_setup *setup) WD_ERR("failed to init join_gather session schedule key!\n"); goto free_sess; } - - if (wd_join_gather_setting.driver->get_extend_ops) { - ret = wd_join_gather_setting.driver->get_extend_ops(&sess->ops); - if (ret) { - WD_ERR("failed to get join gather extend ops!\n"); - goto free_key; - } - } + params.alg_name = wd_join_gather_alg[sess->alg]; + params.ctxs = wd_join_gather_setting.config.ctxs; + wd_join_gather_setting.sched.set_param( + wd_join_gather_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); ret = wd_join_gather_init_sess(sess, setup); if (ret) @@ -523,7 +586,14 @@ handle_t wd_join_gather_alloc_sess(struct wd_join_gather_sess_setup *setup) uninit_sess: wd_join_gather_uninit_sess(sess); free_key: - free(sess->sched_key); + if (sess->sched_key) { + if (wd_join_gather_setting.sched.sched_uninit) + wd_join_gather_setting.sched.sched_uninit( + wd_join_gather_setting.sched.h_sched_ctx, + (handle_t)sess->sched_key); + else + free(sess->sched_key); + } free_sess: free(sess); return (handle_t)0; @@ -542,8 +612,14 @@ void wd_join_gather_free_sess(handle_t h_sess) wd_join_gather_uninit_sess(sess); - if (sess->sched_key) - free(sess->sched_key); + if (sess->sched_key) { + if (wd_join_gather_setting.sched.sched_uninit) + wd_join_gather_setting.sched.sched_uninit( + wd_join_gather_setting.sched.h_sched_ctx, + (handle_t)sess->sched_key); + else + free(sess->sched_key); + } free(sess); } @@ -616,6 +692,8 @@ int wd_join_set_hash_table(handle_t h_sess, struct wd_dae_hash_table *info) { struct wd_join_gather_sess *sess = (struct wd_join_gather_sess *)h_sess; enum wd_join_sess_state expected; + struct wd_ctx_config_internal *config = &wd_join_gather_setting.config; + struct wd_join_gather_ops *eops; int ret; if (!sess || !info) { @@ -655,9 +733,9 @@ int wd_join_set_hash_table(handle_t h_sess, struct wd_dae_hash_table *info) if (!info->ext_table_row_num || !info->ext_table) WD_INFO("info: extern hash table is NULL!\n"); - if (sess->ops.hash_table_init) { - ret = sess->ops.hash_table_init(wd_join_gather_setting.driver, - info, sess->priv); + eops = config->ctxs[0].drv->extend_ops; + if (eops && eops->hash_table_init) { + ret = eops->hash_table_init(info, sess->priv); if (ret) goto out; } @@ -671,6 +749,8 @@ out: return ret; } +static bool wd_join_gather_atfork_registered = false; + static void wd_join_gather_clear_status(void) { wd_alg_clear_init(&wd_join_gather_setting.status); @@ -699,14 +779,8 @@ static int wd_join_gather_alg_init(struct wd_ctx_config *config, struct wd_sched if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_join_gather_setting.config, wd_join_gather_setting.driver); - if (ret) - goto out_clear_pool; - return WD_SUCCESS; -out_clear_pool: - wd_uninit_async_request_pool(&wd_join_gather_setting.pool); out_clear_sched: wd_clear_sched(&wd_join_gather_setting.sched); out_clear_ctx_config: @@ -714,23 +788,13 @@ out_clear_ctx_config: return ret; } -static int wd_join_gather_alg_uninit(void) +static void wd_join_gather_alg_uninit(void) { - enum wd_status status; - - wd_alg_get_init(&wd_join_gather_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - /* Uninit async request pool */ wd_uninit_async_request_pool(&wd_join_gather_setting.pool); /* Unset config, sched, driver */ wd_clear_sched(&wd_join_gather_setting.sched); - - wd_alg_uninit_driver(&wd_join_gather_setting.config, wd_join_gather_setting.driver); - - return WD_SUCCESS; } int wd_join_gather_init(char *alg, __u32 sched_type, int task_type, @@ -742,13 +806,16 @@ int wd_join_gather_init(char *alg, __u32 sched_type, int task_type, int state; bool flag; - pthread_atfork(NULL, NULL, wd_join_gather_clear_status); + if (!wd_join_gather_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_join_gather_clear_status) == 0) + wd_join_gather_atfork_registered = true; + } state = wd_alg_try_init(&wd_join_gather_setting.status); if (state) return state; - if (!alg || sched_type >= SCHED_POLICY_BUTT || + if (!alg || sched_type >= SCHED_POLICY_BUTT || task_type == TASK_MIX || task_type < 0 || task_type >= TASK_MAX_TYPE) { WD_ERR("invalid: join_gathe init input param is wrong!\n"); goto out_uninit; @@ -766,38 +833,24 @@ int wd_join_gather_init(char *alg, __u32 sched_type, int task_type, while (ret != 0) { memset(&wd_join_gather_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - - /* Get alg driver and dev name */ - wd_join_gather_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_join_gather_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; - } - join_gather_ctx_params.ctx_set_num = &join_gather_ctx_num; - ret = wd_ctx_param_init(&join_gather_ctx_params, ctx_params, - wd_join_gather_setting.driver, + ret = wd_ctx_param_init(&join_gather_ctx_params, ctx_params, alg, WD_JOIN_GATHER_TYPE, 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_join_gather_setting.driver); - wd_alg_drv_unbind(wd_join_gather_setting.driver); + if (ret == -WD_EAGAIN) continue; - } goto out_driver; } (void)strcpy(wd_join_gather_init_attrs.alg, alg); wd_join_gather_init_attrs.sched_type = sched_type; - wd_join_gather_init_attrs.driver = wd_join_gather_setting.driver; + wd_join_gather_init_attrs.task_type = task_type; wd_join_gather_init_attrs.ctx_params = &join_gather_ctx_params; wd_join_gather_init_attrs.alg_init = wd_join_gather_alg_init; wd_join_gather_init_attrs.alg_poll_ctx = wd_join_gather_poll_ctx; ret = wd_alg_attrs_init(&wd_join_gather_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_join_gather_setting.driver); - wd_alg_drv_unbind(wd_join_gather_setting.driver); wd_ctx_param_uninit(&join_gather_ctx_params); continue; } @@ -806,16 +859,31 @@ int wd_join_gather_init(char *alg, __u32 sched_type, int task_type, } } + ret = wd_ctx_bind_drivers(&wd_join_gather_setting.config, + wd_join_gather_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("failed to bind driver for hashjoin!\n"); + goto out_common_uninit; + } + + ret = wd_alg_init_driver(&wd_join_gather_setting.config); + if (ret) + goto out_unbind_drivers; + wd_alg_set_init(&wd_join_gather_setting.status); wd_ctx_param_uninit(&join_gather_ctx_params); return WD_SUCCESS; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_join_gather_setting.config); +out_common_uninit: + wd_join_gather_alg_uninit(); + wd_alg_attrs_uninit(&wd_join_gather_init_attrs); out_params_uninit: wd_ctx_param_uninit(&join_gather_ctx_params); out_driver: - wd_alg_drv_unbind(wd_join_gather_setting.driver); -out_dlopen: wd_join_gather_close_driver(); out_uninit: wd_alg_clear_init(&wd_join_gather_setting.status); @@ -824,14 +892,19 @@ out_uninit: void wd_join_gather_uninit(void) { - int ret; + enum wd_status status; - ret = wd_join_gather_alg_uninit(); - if (ret) + wd_alg_get_init(&wd_join_gather_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_join_gather_setting.config); + wd_ctx_unbind_drivers(&wd_join_gather_setting.config); + wd_join_gather_setting.config.drv_array = NULL; + wd_join_gather_setting.config.drv_count = 0; + wd_join_gather_alg_uninit(); + wd_alg_attrs_uninit(&wd_join_gather_init_attrs); - wd_alg_drv_unbind(wd_join_gather_setting.driver); wd_join_gather_close_driver(); wd_alg_clear_init(&wd_join_gather_setting.status); } @@ -871,7 +944,6 @@ static void fill_join_gather_msg(struct wd_join_gather_msg *msg, struct wd_join_ struct wd_join_gather_sess *sess) { memcpy(&msg->req, req, sizeof(struct wd_join_gather_req)); - msg->priv = sess->priv; msg->op_type = req->op_type; switch (req->op_type) { @@ -1211,11 +1283,12 @@ static int wd_join_gather_sync_job(struct wd_join_gather_sess *sess, wd_dfx_msg_cnt(config, WD_CTX_CNT_NUM, idx); ctx = config->ctxs + idx; - msg_handle.send = setting->driver->send; - msg_handle.recv = setting->driver->recv; + msg->priv = sess->priv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; pthread_spin_lock(&ctx->lock); - ret = wd_handle_msg_sync(setting->driver, &msg_handle, ctx->ctx, + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, NULL, config->epoll_en); pthread_spin_unlock(&ctx->lock); @@ -1304,14 +1377,13 @@ static int wd_join_gather_async_job(struct wd_join_gather_sess *sess, ctx = config->ctxs + idx; msg_id = wd_get_msg_from_pool(&setting->pool, idx, (void **)&msg); - if (msg_id < 0) { - WD_ERR("failed to get join gather msg from pool!\n"); - return msg_id; - } + if (msg_id < 0) + return -WD_EBUSY; fill_join_gather_msg(msg, req, sess); + msg->priv = sess->priv; msg->tag = msg_id; - ret = wd_alg_driver_send(setting->driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (ret < 0) { if (ret != -WD_EBUSY) WD_ERR("wd join gather async send err!\n"); @@ -1781,7 +1853,7 @@ static int wd_join_gather_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_join_gather_setting.driver, ctx->ctx, &resp_msg); + ret = ctx->drv->recv(ctx->ctx, &resp_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { -- 2.43.0
From: Wenkai Lin <linwenkai6@hisilicon.com> For data digest algorithms, the API layer needs to be adapted to the new UADK heterogeneous hybrid acceleration framework, thereby ensuring that hash algorithms and authentication algorithm acceleration functions can adapt to and support the new hybrid acceleration framework, while supporting the fusion of hardware acceleration, instruction acceleration, and vector acceleration capabilities. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> --- drv/hash_mb/hash_mb.c | 100 ++++++++++---------- drv/isa_ce_sm3.c | 46 ++++----- wd_digest.c | 212 ++++++++++++++++++++++++++---------------- 3 files changed, 210 insertions(+), 148 deletions(-) diff --git a/drv/hash_mb/hash_mb.c b/drv/hash_mb/hash_mb.c index bd412b2..e0f3fd0 100644 --- a/drv/hash_mb/hash_mb.c +++ b/drv/hash_mb/hash_mb.c @@ -6,6 +6,7 @@ #include <stdlib.h> #include <string.h> #include "hash_mb.h" +#include "../wd_drv.h" #define MIN(a, b) (((a) > (b)) ? (b) : (a)) #define IPAD_VALUE 0x36 @@ -109,12 +110,20 @@ static void hash_mb_queue_uninit(struct wd_ctx_config_internal *config, int ctx_ int i; for (i = 0; i < ctx_num; i++) { + if (strcmp(config->ctxs[i].drv->drv_name, "hash_mb") || + config->ctxs[i].drv->calc_type != UADK_ALG_SVE_INSTR) + continue; + ctx = (struct wd_soft_ctx *)config->ctxs[i].ctx; mb_queue = ctx->priv; + if (!mb_queue) + continue; + pthread_spin_destroy(&mb_queue->r_lock); hash_mb_uninit_poll_queue(&mb_queue->sm3_poll_queue); hash_mb_uninit_poll_queue(&mb_queue->md5_poll_queue); free(mb_queue); + ctx->priv = NULL; } } @@ -143,11 +152,16 @@ static int hash_mb_queue_init(struct wd_ctx_config_internal *config) int i, ret; for (i = 0; i < ctx_num; i++) { - mb_queue = calloc(1, sizeof(struct hash_mb_queue)); + if (strcmp(config->ctxs[i].drv->drv_name, "hash_mb") || + config->ctxs[i].drv->calc_type != UADK_ALG_SVE_INSTR) + continue; + + mb_queue = malloc(sizeof(struct hash_mb_queue)); if (!mb_queue) { ret = -WD_ENOMEM; goto free_mb_queue; } + memset(mb_queue, 0, sizeof(struct hash_mb_queue)); mb_queue->ctx_mode = config->ctxs[i].ctx_mode; ctx = (struct wd_soft_ctx *)config->ctxs[i].ctx; @@ -168,9 +182,6 @@ static int hash_mb_queue_init(struct wd_ctx_config_internal *config) mb_queue->sm3_poll_queue.ops = &sm3_ops; mb_queue->md5_poll_queue.ops = &md5_ops; - mb_queue->recv_head = NULL; - mb_queue->recv_tail = NULL; - mb_queue->complete_cnt = 0; } return WD_SUCCESS; @@ -186,46 +197,34 @@ free_mb_queue: return ret; } -static int hash_mb_init(struct wd_alg_driver *drv, void *conf) +static int hash_mb_init(void *conf, void *priv) { struct wd_ctx_config_internal *config = conf; - struct hash_mb_ctx *priv; - int ret; + struct hash_mb_ctx *mb_ctx = priv; /* Fallback init is NULL */ - if (!drv || !conf) + if (!conf || !priv) return 0; - priv = malloc(sizeof(struct hash_mb_ctx)); - if (!priv) - return -WD_ENOMEM; - /* multibuff does not use epoll. */ config->epoll_en = 0; - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); + memcpy(&mb_ctx->config, config, sizeof(struct wd_ctx_config_internal)); - ret = hash_mb_queue_init(config); - if (ret) { - free(priv); - return ret; - } - - drv->priv = priv; - - return WD_SUCCESS; + return hash_mb_queue_init(config); } -static void hash_mb_exit(struct wd_alg_driver *drv) +static void hash_mb_exit(void *priv) { - struct hash_mb_ctx *priv; + struct hash_mb_ctx *mb_ctx = priv; + struct wd_ctx_config_internal *config; - if (!drv || !drv->priv) + if (!priv) { + WD_ERR("invalid: input parameter is NULL!\n"); return; + } - priv = (struct hash_mb_ctx *)drv->priv; - hash_mb_queue_uninit(&priv->config, priv->config.ctx_num); - free(priv); - drv->priv = NULL; + config = &mb_ctx->config; + hash_mb_queue_uninit(config, config->ctx_num); } static void hash_mb_pad_data(struct hash_pad *hash_pad, __u8 *in, __u32 partial, @@ -267,7 +266,7 @@ static inline void hash_xor(__u8 *key_out, __u8 *key_in, __u32 key_len, __u8 xor if (i < key_len) key_out[i] = key_in[i] ^ xor_value; else - key_out[i] = xor_value; + key_out[i] = 0x0 ^ xor_value; } } @@ -555,7 +554,21 @@ static int hash_mb_check_param(struct hash_mb_queue *mb_queue, struct wd_digest_ return WD_SUCCESS; } -static int hash_mb_send(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg) +static void hash_mb_add_finish_job(struct hash_mb_queue *mb_queue, struct hash_job *job) +{ + pthread_spin_lock(&mb_queue->r_lock); + if (mb_queue->complete_cnt) { + mb_queue->recv_tail->next = job; + mb_queue->recv_tail = job; + } else { + mb_queue->recv_head = job; + mb_queue->recv_tail = job; + } + mb_queue->complete_cnt++; + pthread_spin_unlock(&mb_queue->r_lock); +} + +static int hash_mb_send(handle_t ctx, void *drv_msg) { struct wd_soft_ctx *s_ctx = (struct wd_soft_ctx *)ctx; struct hash_mb_queue *mb_queue = s_ctx->priv; @@ -597,8 +610,10 @@ static int hash_mb_send(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg) /* If block not need process, return directly. */ ret = hash_do_partial(poll_queue, d_msg, hash_job); if (ret == -WD_EAGAIN) { - if (mb_queue->ctx_mode == CTX_MODE_ASYNC) - free(hash_job); + if (mb_queue->ctx_mode == CTX_MODE_ASYNC) { + hash_job->msg = d_msg; + hash_mb_add_finish_job(mb_queue, hash_job); + } d_msg->result = WD_SUCCESS; return WD_SUCCESS; @@ -691,20 +706,6 @@ static struct hash_job *hash_mb_get_job(struct hash_mb_poll_queue *poll_queue) return job; } -static void hash_mb_add_finish_job(struct hash_mb_queue *mb_queue, struct hash_job *job) -{ - pthread_spin_lock(&mb_queue->r_lock); - if (mb_queue->complete_cnt) { - mb_queue->recv_tail->next = job; - mb_queue->recv_tail = job; - } else { - mb_queue->recv_head = job; - mb_queue->recv_tail = job; - } - mb_queue->complete_cnt++; - pthread_spin_unlock(&mb_queue->r_lock); -} - static struct hash_mb_poll_queue *hash_get_poll_queue(struct hash_mb_queue *mb_queue) { if (!mb_queue->sm3_poll_queue.job_num && @@ -776,7 +777,7 @@ static int hash_mb_do_jobs(struct hash_mb_queue *mb_queue) return WD_SUCCESS; } -static int hash_mb_recv(struct wd_alg_driver *drv, handle_t ctx, void *drv_msg) +static int hash_mb_recv(handle_t ctx, void *drv_msg) { struct wd_soft_ctx *s_ctx = (struct wd_soft_ctx *)ctx; struct hash_mb_queue *mb_queue = s_ctx->priv; @@ -810,6 +811,7 @@ static int hash_mb_get_usage(void *param) .alg_name = (hash_alg_name),\ .calc_type = UADK_ALG_SVE_INSTR,\ .priority = 100,\ + .priv_size = sizeof(struct hash_mb_ctx),\ .queue_num = 1,\ .op_type_num = 1,\ .fallback = 0,\ @@ -818,6 +820,8 @@ static int hash_mb_get_usage(void *param) .send = hash_mb_send,\ .recv = hash_mb_recv,\ .get_usage = hash_mb_get_usage,\ + .alloc_ctx = wd_soft_alloc_ctx,\ + .free_ctx = wd_soft_free_ctx,\ } static struct wd_alg_driver hash_mb_driver[] = { diff --git a/drv/isa_ce_sm3.c b/drv/isa_ce_sm3.c index 627ab68..b41f6c5 100644 --- a/drv/isa_ce_sm3.c +++ b/drv/isa_ce_sm3.c @@ -338,8 +338,9 @@ static int do_hmac_sm3_ce(struct wd_digest_msg *msg, __u8 *out_hmac) return WD_SUCCESS; } -static int sm3_ce_drv_send(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg) +static int sm3_ce_drv_send(handle_t ctx, void *digest_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; struct wd_digest_msg *msg = (struct wd_digest_msg *)digest_msg; __u8 digest[SM3_DIGEST_SIZE] = {0}; int ret; @@ -349,6 +350,10 @@ static int sm3_ce_drv_send(struct wd_alg_driver *drv, handle_t ctx, void *digest return -WD_EINVAL; } + ret = wd_queue_is_busy(sfctx); + if (ret) + return ret; + if (msg->data_fmt == WD_SGL_BUF) { WD_ERR("invalid: SM3 CE driver do not support sgl data format!\n"); return -WD_EINVAL; @@ -370,42 +375,41 @@ static int sm3_ce_drv_send(struct wd_alg_driver *drv, handle_t ctx, void *digest ret = -WD_EINVAL; } + ret = wd_get_sqe_from_queue(sfctx, msg->tag); + if (ret) + return ret; + return ret; } -static int sm3_ce_drv_recv(struct wd_alg_driver *drv, handle_t ctx, void *digest_msg) +static int sm3_ce_drv_recv(handle_t ctx, void *digest_msg) { + struct wd_soft_ctx *sfctx = (struct wd_soft_ctx *)ctx; + struct wd_digest_msg *msg = (struct wd_digest_msg *)digest_msg; + int ret; + + ret = wd_put_sqe_to_queue(sfctx, &msg->tag, &msg->result); + if (ret) + return ret; + return WD_SUCCESS; } -static int sm3_ce_drv_init(struct wd_alg_driver *drv, void *conf) +static int sm3_ce_drv_init(void *conf, void *priv) { - struct wd_ctx_config_internal *config = (struct wd_ctx_config_internal *)conf; - struct sm3_ce_drv_ctx *priv; + struct wd_ctx_config_internal *config = conf; + struct sm3_ce_drv_ctx *sctx = priv; /* Fallback init is NULL */ - if (!drv || !conf) + if (!conf || !priv) return 0; - priv = malloc(sizeof(struct sm3_ce_drv_ctx)); - if (!priv) - return -WD_ENOMEM; - config->epoll_en = 0; - memcpy(&priv->config, config, sizeof(struct wd_ctx_config_internal)); - drv->priv = priv; + memcpy(&sctx->config, config, sizeof(struct wd_ctx_config_internal)); return WD_SUCCESS; } -static void sm3_ce_drv_exit(struct wd_alg_driver *drv) +static void sm3_ce_drv_exit(void *priv) { - struct sm3_ce_drv_ctx *sctx; - - if (!drv || !drv->priv) - return; - - sctx = (struct sm3_ce_drv_ctx *)drv->priv; - free(sctx); - drv->priv = NULL; } diff --git a/wd_digest.c b/wd_digest.c index e0341e6..2388346 100644 --- a/wd_digest.c +++ b/wd_digest.c @@ -40,7 +40,6 @@ struct wd_digest_setting { enum wd_status status; struct wd_ctx_config_internal config; struct wd_sched sched; - struct wd_alg_driver *driver; struct wd_async_msg_pool pool; void *dlhandle; void *dlh_list; @@ -85,20 +84,16 @@ static void wd_digest_close_driver(int init_type) } if (wd_digest_setting.dlhandle) { - wd_release_drv(wd_digest_setting.driver); dlclose(wd_digest_setting.dlhandle); wd_digest_setting.dlhandle = NULL; } #else - wd_release_drv(wd_digest_setting.driver); hisi_sec2_remove(); #endif } static int wd_digest_open_driver(int init_type) { - struct wd_alg_driver *driver = NULL; - const char *alg_name = "sm3"; #ifndef WD_STATIC_DRV char lib_path[PATH_MAX]; int ret; @@ -132,14 +127,6 @@ static int wd_digest_open_driver(int init_type) if (init_type == WD_TYPE_V2) return WD_SUCCESS; #endif - driver = wd_request_drv(alg_name, false); - if (!driver) { - wd_digest_close_driver(WD_TYPE_V1); - WD_ERR("failed to get %s driver support\n", alg_name); - return -WD_EINVAL; - } - - wd_digest_setting.driver = driver; return WD_SUCCESS; } @@ -217,6 +204,7 @@ static int digest_setup_memory_and_buffers(struct wd_digest_sess *sess, handle_t wd_digest_alloc_sess(struct wd_digest_sess_setup *setup) { struct wd_digest_sess *sess = NULL; + struct wd_sched_params params; bool ret; if (unlikely(!setup)) { @@ -237,7 +225,7 @@ handle_t wd_digest_alloc_sess(struct wd_digest_sess_setup *setup) sess->alg_name = wd_digest_alg_name[setup->alg]; sess->alg = setup->alg; sess->mode = setup->mode; - ret = wd_drv_alg_support(sess->alg_name, wd_digest_setting.driver); + ret = wd_drv_alg_support(sess->alg_name, &wd_digest_setting.config); if (!ret) { WD_ERR("failed to support this algorithm: %s!\n", sess->alg_name); goto err_sess; @@ -255,6 +243,14 @@ handle_t wd_digest_alloc_sess(struct wd_digest_sess_setup *setup) goto err_key; } + /* Set compat filtering parameters for session-ctx matching */ + memset(¶ms, 0, sizeof(params)); + params.alg_name = sess->alg_name; + params.ctxs = wd_digest_setting.config.ctxs; + wd_digest_setting.sched.set_param( + wd_digest_setting.sched.h_sched_ctx, + sess->sched_key, ¶ms); + return (handle_t)sess; err_key: @@ -275,11 +271,19 @@ void wd_digest_free_sess(handle_t h_sess) wd_memset_zero(sess->key, sess->key_bytes); sess->mm_ops.free(sess->mm_ops.usr, sess->key); - if (sess->sched_key) - free(sess->sched_key); + if (sess->sched_key) { + if (wd_digest_setting.sched.sched_uninit) + wd_digest_setting.sched.sched_uninit( + wd_digest_setting.sched.h_sched_ctx, + (handle_t)sess->sched_key); + else + free(sess->sched_key); + } free(sess); } +static bool wd_digest_atfork_registered; + static void wd_digest_clear_status(void) { wd_alg_clear_init(&wd_digest_setting.status); @@ -311,15 +315,8 @@ static int wd_digest_init_nolock(struct wd_ctx_config *config, if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_digest_setting.config, - wd_digest_setting.driver); - if (ret) - goto out_clear_pool; - return 0; -out_clear_pool: - wd_uninit_async_request_pool(&wd_digest_setting.pool); out_clear_sched: wd_clear_sched(&wd_digest_setting.sched); out_clear_ctx_config: @@ -328,11 +325,21 @@ out_clear_ctx_config: return ret; } +static void wd_digest_uninit_nolock(void) +{ + wd_uninit_async_request_pool(&wd_digest_setting.pool); + wd_clear_sched(&wd_digest_setting.sched); +} + int wd_digest_init(struct wd_ctx_config *config, struct wd_sched *sched) { + __u32 drv_count = 0; int ret; - pthread_atfork(NULL, NULL, wd_digest_clear_status); + if (!wd_digest_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_digest_clear_status) == 0) + wd_digest_atfork_registered = true; + } ret = wd_alg_try_init(&wd_digest_setting.status); if (ret) @@ -342,6 +349,14 @@ int wd_digest_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + /* init1 path is HW-only; CE/SVE drivers require init2 */ + if (sched->sched_policy == SCHED_POLICY_NONE || + sched->sched_policy == SCHED_POLICY_SINGLE) { + WD_ERR("init1 does not support NONE/SINGLE schedulers, use init2\n"); + ret = -WD_EINVAL; + goto out_clear_init; + } + ret = wd_digest_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; @@ -350,10 +365,38 @@ int wd_digest_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_close_driver; + ret = wd_get_drv_array("digest", TASK_HW, "hisi_sec2", + &wd_digest_setting.config.drv_array, &drv_count); + if (ret) { + WD_ERR("failed to get driver array for digest!\n"); + goto out_common_uninit; + } + + wd_digest_setting.config.drv_count = drv_count; + ret = wd_ctx_bind_drivers(&wd_digest_setting.config, NULL, WD_TYPE_V1); + if (ret) { + WD_ERR("failed to bind driver!\n"); + goto out_free_drv_array; + } + + ret = wd_alg_init_driver(&wd_digest_setting.config); + if (ret) { + WD_ERR("failed to init digest driver!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_digest_setting.status); return 0; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_digest_setting.config); +out_free_drv_array: + wd_put_drv_array(wd_digest_setting.config.drv_array, drv_count); + wd_digest_setting.config.drv_array = NULL; + wd_digest_setting.config.drv_count = 0; +out_common_uninit: + wd_digest_uninit_nolock(); out_close_driver: wd_digest_close_driver(WD_TYPE_V1); out_clear_init: @@ -361,29 +404,22 @@ out_clear_init: return ret; } -static int wd_digest_uninit_nolock(void) +void wd_digest_uninit(void) { enum wd_status status; wd_alg_get_init(&wd_digest_setting.status, &status); - if (status == WD_UNINIT) - return -WD_EINVAL; - - wd_uninit_async_request_pool(&wd_digest_setting.pool); - wd_clear_sched(&wd_digest_setting.sched); - wd_alg_uninit_driver(&wd_digest_setting.config, - wd_digest_setting.driver); - return 0; -} - -void wd_digest_uninit(void) -{ - int ret; - - ret = wd_digest_uninit_nolock(); - if (ret) + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_digest_setting.config); + wd_ctx_unbind_drivers(&wd_digest_setting.config); + wd_put_drv_array(wd_digest_setting.config.drv_array, + wd_digest_setting.config.drv_count); + wd_digest_setting.config.drv_array = NULL; + wd_digest_setting.config.drv_count = 0; + + wd_digest_uninit_nolock(); wd_digest_close_driver(WD_TYPE_V1); wd_alg_clear_init(&wd_digest_setting.status); } @@ -404,8 +440,12 @@ int wd_digest_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params digest_ctx_params = {0}; struct wd_ctx_nums digest_ctx_num = {0}; int state, ret = -WD_EINVAL; + int try_cnt = 0; - pthread_atfork(NULL, NULL, wd_digest_clear_status); + if (!wd_digest_atfork_registered) { + if (pthread_atfork(NULL, NULL, wd_digest_clear_status) == 0) + wd_digest_atfork_registered = true; + } state = wd_alg_try_init(&wd_digest_setting.status); if (state) @@ -428,38 +468,30 @@ int wd_digest_init2_(char *alg, __u32 sched_type, int task_type, while (ret != 0) { - memset(&wd_digest_setting.config, 0, sizeof(struct wd_ctx_config_internal)); - - /* Get alg driver and dev name */ - wd_digest_setting.driver = wd_alg_drv_bind(task_type, alg); - if (!wd_digest_setting.driver) { - WD_ERR("failed to bind %s driver.\n", alg); - goto out_dlopen; + if (try_cnt++ >= WD_INIT2_MAX_RETRY) { + WD_ERR("failed to init2 after %d retries.\n", + WD_INIT2_MAX_RETRY); + goto out_dlclose; } - + memset(&wd_digest_setting.config, 0, sizeof(struct wd_ctx_config_internal)); digest_ctx_params.ctx_set_num = &digest_ctx_num; ret = wd_ctx_param_init(&digest_ctx_params, ctx_params, - wd_digest_setting.driver, WD_DIGEST_TYPE, 1); + alg, WD_DIGEST_TYPE, 1); if (ret) { - if (ret == -WD_EAGAIN) { - wd_disable_drv(wd_digest_setting.driver); - wd_alg_drv_unbind(wd_digest_setting.driver); + if (ret == -WD_EAGAIN) continue; - } - goto out_driver; + goto out_dlclose; } (void)strcpy(wd_digest_init_attrs.alg, alg); wd_digest_init_attrs.sched_type = sched_type; - wd_digest_init_attrs.driver = wd_digest_setting.driver; + wd_digest_init_attrs.task_type = task_type; wd_digest_init_attrs.ctx_params = &digest_ctx_params; wd_digest_init_attrs.alg_init = wd_digest_init_nolock; wd_digest_init_attrs.alg_poll_ctx = wd_digest_poll_ctx; ret = wd_alg_attrs_init(&wd_digest_init_attrs); if (ret) { if (ret == -WD_ENODEV) { - wd_disable_drv(wd_digest_setting.driver); - wd_alg_drv_unbind(wd_digest_setting.driver); wd_ctx_param_uninit(&digest_ctx_params); continue; } @@ -467,16 +499,34 @@ int wd_digest_init2_(char *alg, __u32 sched_type, int task_type, goto out_params_uninit; } } + + ret = wd_ctx_bind_drivers(&wd_digest_setting.config, + wd_digest_init_attrs.ctx_config_internal, + WD_TYPE_V2); + if (ret) { + WD_ERR("failed to bind driver for digest!\n"); + goto out_common_uninit; + } + + ret = wd_alg_init_driver(&wd_digest_setting.config); + if (ret) { + WD_ERR("failed to init driver for digest!\n"); + goto out_unbind_drivers; + } + wd_alg_set_init(&wd_digest_setting.status); wd_ctx_param_uninit(&digest_ctx_params); return 0; +out_unbind_drivers: + wd_ctx_unbind_drivers(&wd_digest_setting.config); +out_common_uninit: + wd_digest_uninit_nolock(); + wd_alg_attrs_uninit(&wd_digest_init_attrs); out_params_uninit: wd_ctx_param_uninit(&digest_ctx_params); -out_driver: - wd_alg_drv_unbind(wd_digest_setting.driver); -out_dlopen: +out_dlclose: wd_digest_close_driver(WD_TYPE_V2); out_uninit: wd_alg_clear_init(&wd_digest_setting.status); @@ -485,14 +535,18 @@ out_uninit: void wd_digest_uninit2(void) { - int ret; + enum wd_status status; - ret = wd_digest_uninit_nolock(); - if (ret) + wd_alg_get_init(&wd_digest_setting.status, &status); + if (status != WD_INIT) return; + wd_alg_uninit_driver(&wd_digest_setting.config); + wd_ctx_unbind_drivers(&wd_digest_setting.config); + wd_digest_setting.config.drv_array = NULL; + wd_digest_setting.config.drv_count = 0; + wd_digest_uninit_nolock(); wd_alg_attrs_uninit(&wd_digest_init_attrs); - wd_alg_drv_unbind(wd_digest_setting.driver); wd_digest_close_driver(WD_TYPE_V2); wd_alg_clear_init(&wd_digest_setting.status); } @@ -651,13 +705,13 @@ static int send_recv_sync(struct wd_ctx_internal *ctx, struct wd_digest_sess *ds struct wd_msg_handle msg_handle; int ret; - msg_handle.send = wd_digest_setting.driver->send; - msg_handle.recv = wd_digest_setting.driver->recv; + msg_handle.send = ctx->drv->send; + msg_handle.recv = ctx->drv->recv; - wd_ctx_spin_lock(ctx, wd_digest_setting.driver->calc_type); - ret = wd_handle_msg_sync(wd_digest_setting.driver, &msg_handle, ctx->ctx, - msg, NULL, wd_digest_setting.config.epoll_en); - wd_ctx_spin_unlock(ctx, wd_digest_setting.driver->calc_type); + wd_ctx_spin_lock(ctx, ctx->ctx_type); + ret = wd_handle_msg_sync(&msg_handle, ctx->ctx, msg, + NULL, wd_digest_setting.config.epoll_en); + wd_ctx_spin_unlock(ctx, ctx->ctx_type); if (unlikely(ret)) return ret; @@ -742,15 +796,13 @@ int wd_do_digest_async(handle_t h_sess, struct wd_digest_req *req) msg_id = wd_get_msg_from_pool(&wd_digest_setting.pool, idx, (void **)&msg); - if (unlikely(msg_id < 0)) { - WD_ERR("failed to get msg from pool!\n"); - return msg_id; - } + if (unlikely(msg_id < 0)) + return -WD_EBUSY; fill_request_msg(msg, req, dsess); msg->tag = msg_id; - ret = wd_alg_driver_send(wd_digest_setting.driver, ctx->ctx, msg); + ret = ctx->drv->send(ctx->ctx, msg); if (unlikely(ret < 0)) { if (ret != -WD_EBUSY) WD_ERR("failed to send BD, hw is err!\n"); @@ -797,7 +849,7 @@ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) ctx = config->ctxs + idx; do { - ret = wd_alg_driver_recv(wd_digest_setting.driver, ctx->ctx, &recv_msg); + ret = ctx->drv->recv(ctx->ctx, &recv_msg); if (ret == -WD_EAGAIN) { return ret; } else if (ret < 0) { @@ -816,8 +868,10 @@ int wd_digest_poll_ctx(__u32 idx, __u32 expt, __u32 *count) msg->req.state = recv_msg.result; req = &msg->req; - if (likely(req)) + if (likely(req->cb)) req->cb(req); + else + WD_ERR("invalid: digest callback is NULL, tag %u!\n", recv_msg.tag); wd_put_msg_to_pool(&wd_digest_setting.pool, idx, recv_msg.tag); -- 2.43.0
From: Zhushuai Yin <yinzhushuai@huawei.com> With the UADK framework updated to support heterogeneous hybrid acceleration functionality, the corresponding uadk_tool test framework also needs to be adapted and updated. It must simultaneously support both the legacy init interface and the new init2 interface, match new scheduling algorithm types, and add new DAE test functionality. Signed-off-by: Zhushuai Yin <yinzhushuai@huawei.com> Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: lizhi <lizhi206@huawei.com> Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> --- uadk_tool/benchmark/hpre_uadk_benchmark.c | 173 ++++++++++++++-------- uadk_tool/benchmark/sec_uadk_benchmark.c | 55 +++++-- uadk_tool/benchmark/uadk_benchmark.c | 81 +++++++++- uadk_tool/benchmark/uadk_benchmark.h | 7 + uadk_tool/benchmark/zip_uadk_benchmark.c | 98 +++++++++--- 5 files changed, 314 insertions(+), 100 deletions(-) diff --git a/uadk_tool/benchmark/hpre_uadk_benchmark.c b/uadk_tool/benchmark/hpre_uadk_benchmark.c index fa26a61..58394c7 100644 --- a/uadk_tool/benchmark/hpre_uadk_benchmark.c +++ b/uadk_tool/benchmark/hpre_uadk_benchmark.c @@ -21,6 +21,7 @@ #define SQE_SIZE 128 #define POOL_MULTIPLY_FACTOR 2 #define HPRE_OP_TYPE_MAX 6 +#define MAX_DRAIN_RETRY 10000 struct hpre_rsa_key_in { void *e; @@ -503,7 +504,7 @@ static int init_hpre_ctx_config(struct acc_option *options) switch(subtype) { case RSA_TYPE: if (options->mem_type == UADK_AUTO) - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, HPRE_OP_TYPE_MAX, + g_sched = wd_sched_rr_alloc(options->sched_type, HPRE_OP_TYPE_MAX, max_node, wd_rsa_poll_ctx); else g_sched = wd_sched_rr_alloc(SCHED_POLICY_DEV, HPRE_OP_TYPE_MAX, @@ -511,7 +512,7 @@ static int init_hpre_ctx_config(struct acc_option *options) break; case DH_TYPE: if (options->mem_type == UADK_AUTO) - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, HPRE_OP_TYPE_MAX, + g_sched = wd_sched_rr_alloc(options->sched_type, HPRE_OP_TYPE_MAX, max_node, wd_dh_poll_ctx); else g_sched = wd_sched_rr_alloc(SCHED_POLICY_DEV, HPRE_OP_TYPE_MAX, @@ -523,7 +524,7 @@ static int init_hpre_ctx_config(struct acc_option *options) case X25519_TYPE: case X448_TYPE: if (options->mem_type == UADK_AUTO) - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, HPRE_OP_TYPE_MAX, + g_sched = wd_sched_rr_alloc(options->sched_type, HPRE_OP_TYPE_MAX, max_node, wd_ecc_poll_ctx); else g_sched = wd_sched_rr_alloc(SCHED_POLICY_DEV, HPRE_OP_TYPE_MAX, @@ -672,7 +673,10 @@ static int init_hpre_ctx_config2(struct acc_option *options) numa_bitmask_setall(cparams.bmp); - if (mode == CTX_MODE_SYNC) + if (options->sched_type == SCHED_POLICY_SINGLE) { + ctx_set_num->sync_ctx_num = g_ctxnum; + ctx_set_num->async_ctx_num = g_ctxnum; + } else if (mode == CTX_MODE_SYNC) ctx_set_num->sync_ctx_num = g_ctxnum; else ctx_set_num->async_ctx_num = g_ctxnum; @@ -681,26 +685,39 @@ static int init_hpre_ctx_config2(struct acc_option *options) switch (subtype) { case RSA_TYPE: if (options->mem_type == UADK_AUTO) - return wd_rsa_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + ret = wd_rsa_init2_(alg_name, options->sched_type, + options->task_type, &cparams); else - return wd_rsa_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); + ret = wd_rsa_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); + if (ret) + HPRE_TST_PRT("failed to do rsa init2!\n"); + break; case DH_TYPE: if (options->mem_type == UADK_AUTO) - return wd_dh_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + ret = wd_dh_init2_(alg_name, options->sched_type, + options->task_type, &cparams); else - return wd_dh_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); + ret = wd_dh_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); + if (ret) + HPRE_TST_PRT("failed to do dh init2!\n"); + break; case ECDH_TYPE: case ECDSA_TYPE: case SM2_TYPE: case X25519_TYPE: case X448_TYPE: if (options->mem_type == UADK_AUTO) - return wd_ecc_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + ret = wd_ecc_init2_(alg_name, options->sched_type, + options->task_type, &cparams); else - return wd_ecc_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); + ret = wd_ecc_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); + if (ret) + HPRE_TST_PRT("failed to do ecc init2!\n"); + break; default: - HPRE_TST_PRT("failed to parse alg subtype on uninit2!\n"); - return -EINVAL; + HPRE_TST_PRT("failed to parse alg subtype on init2!\n"); + ret = -EINVAL; + break; } out_freectx: @@ -820,9 +837,9 @@ void *hpre_uadk_poll(void *data) thread_data *pdata = (thread_data *)data; u32 expt = ACC_QUEUE_SIZE * g_thread_num; u32 id = pdata->td_id; - u32 last_time = 2; // poll need one more recv time u32 count = 0; u32 recv = 0; + u32 drain_retry = 0; int ret; if (id > g_ctxnum) @@ -847,17 +864,42 @@ void *hpre_uadk_poll(void *data) return NULL; } - while (last_time) { + while (1) { ret = uadk_poll_ctx(id, expt, &recv); count += recv; recv = 0; if (unlikely(ret != -WD_EAGAIN && ret < 0)) { HPRE_TST_PRT("poll ret: %d!\n", ret); + add_total_recv(count); goto recv_error; } + if (get_run_state() == 0) { + add_total_recv(count); + break; + } + } - if (get_run_state() == 0) - last_time--; + while (get_send_stopped() != g_thread_num || + get_total_recv() < get_total_sent()) { + ret = uadk_poll_ctx(id, expt, &recv); + if (unlikely(ret != -WD_EAGAIN && ret < 0)) { + HPRE_TST_PRT("poll ret: %d!\n", ret); + goto recv_error; + } + if (recv == 0) { + usleep(SEND_USLEEP); + drain_retry++; + } else { + add_total_recv(recv); + drain_retry = 0; + recv = 0; + } + if (drain_retry >= MAX_DRAIN_RETRY) { + HPRE_TST_PRT("drain timeout: sent=%llu recv=%llu stopped=%llu/%u\n", + get_total_sent(), get_total_recv(), + get_send_stopped(), g_thread_num); + break; + } } recv_error: @@ -872,10 +914,10 @@ void *hpre_uadk_poll2(void *data) thread_data *pdata = (thread_data *)data; u32 expt = ACC_QUEUE_SIZE * g_thread_num; poll_ctx uadk_poll = NULL; - u32 last_time = 2; // poll need one more recv time u32 count = 0; u32 recv = 0; - int ret; + u32 drain_retry = 0; + int ret; switch (pdata->subtype) { case RSA_TYPE: @@ -896,17 +938,42 @@ void *hpre_uadk_poll2(void *data) return NULL; } - while (last_time) { + while (1) { ret = uadk_poll(expt, &recv); count += recv; recv = 0; if (unlikely(ret != -WD_EAGAIN && ret < 0)) { HPRE_TST_PRT("poll ret: %d!\n", ret); + add_total_recv(count); goto recv_error; } + if (get_run_state() == 0) { + add_total_recv(count); + break; + } + } - if (get_run_state() == 0) - last_time--; + while (get_send_stopped() != g_thread_num || + get_total_recv() < get_total_sent()) { + ret = uadk_poll(expt, &recv); + if (unlikely(ret != -WD_EAGAIN && ret < 0)) { + HPRE_TST_PRT("poll ret: %d!\n", ret); + goto recv_error; + } + if (recv == 0) { + usleep(SEND_USLEEP); + drain_retry++; + } else { + add_total_recv(recv); + drain_retry = 0; + recv = 0; + } + if (drain_retry >= MAX_DRAIN_RETRY) { + HPRE_TST_PRT("drain timeout: sent=%llu recv=%llu stopped=%llu/%u\n", + get_total_sent(), get_total_recv(), + get_send_stopped(), g_thread_num); + break; + } } recv_error: @@ -1076,9 +1143,11 @@ static int get_rsa_key_from_sample(handle_t sess, char *privkey_file, } if (crt_privkey_file) { - memcpy(crt_privkey_file, wd_dq.data, (key_bits >> 4) * 5); - memcpy(crt_privkey_file + (key_bits >> 4) * 5, - wd_e.data, (key_bits >> 2)); + memcpy(crt_privkey_file, wd_dq.data, + dq_bytes + dp_bytes + qinv_bytes + q_bytes + p_bytes); + memcpy(crt_privkey_file + dq_bytes + dp_bytes + + qinv_bytes + q_bytes + p_bytes, + wd_e.data, e_bytes); } } else { @@ -1097,10 +1166,10 @@ static int get_rsa_key_from_sample(handle_t sess, char *privkey_file, if (privkey_file) { - memcpy(privkey_file, wd_d.data, key_size); - memcpy(privkey_file + key_size, wd_n.data, key_size); - memcpy(privkey_file + 2 * key_size, wd_e.data, key_size); - memcpy(privkey_file + 3 * key_size, wd_n.data, key_size); + memcpy(privkey_file, wd_d.data, d_bytes); + memcpy(privkey_file + key_size, wd_n.data, n_bytes); + memcpy(privkey_file + 2 * key_size, wd_e.data, e_bytes); + memcpy(privkey_file + 3 * key_size, wd_n.data, n_bytes); } } @@ -1735,20 +1804,12 @@ static void *rsa_uadk_async_run(void *arg) count++; } while(true); - /* Release memory after all tasks are complete. */ - if (count) { - i = 0; - while (get_recv_time() != g_ctxnum) { - if (i++ >= MAX_TRY_CNT) { - HPRE_TST_PRT("failed to wait poll thread finish!\n"); - break; - } + add_total_sent(count); + add_send_stopped(); + if (count) { + while (get_recv_time() != g_ctxnum) usleep(SEND_USLEEP); - } - - /* Wait for the device to complete the tasks. */ - usleep(SEND_USLEEP * MAX_TRY_CNT); } if (req.op_type == WD_RSA_GENKEY) { @@ -2021,20 +2082,12 @@ static void *dh_uadk_async_run(void *arg) count++; } while(true); - /* Release memory after all tasks are complete. */ - if (count) { - i = 0; - while (get_recv_time() != g_ctxnum) { - if (i++ >= MAX_TRY_CNT) { - HPRE_TST_PRT("failed to wait poll thread finish!\n"); - break; - } + add_total_sent(count); + add_send_stopped(); + if (count) { + while (get_recv_time() != g_ctxnum) usleep(SEND_USLEEP); - } - - /* Wait for the device to complete the tasks. */ - usleep(SEND_USLEEP * MAX_TRY_CNT); } free(tag); @@ -2789,20 +2842,12 @@ static void *ecc_uadk_async_run(void *arg) count++; } while(true); - /* Release memory after all tasks are complete. */ - if (count) { - i = 0; - while (get_recv_time() != g_ctxnum) { - if (i++ >= MAX_TRY_CNT) { - HPRE_TST_PRT("failed to wait poll thread finish!\n"); - break; - } + add_total_sent(count); + add_send_stopped(); + if (count) { + while (get_recv_time() != g_ctxnum) usleep(SEND_USLEEP); - } - - /* Wait for the device to complete the tasks. */ - usleep(SEND_USLEEP * MAX_TRY_CNT); } free(tag); diff --git a/uadk_tool/benchmark/sec_uadk_benchmark.c b/uadk_tool/benchmark/sec_uadk_benchmark.c index 4a64c94..7bfafda 100644 --- a/uadk_tool/benchmark/sec_uadk_benchmark.c +++ b/uadk_tool/benchmark/sec_uadk_benchmark.c @@ -730,8 +730,9 @@ static int init_ctx_config(struct acc_option *options) switch(subtype) { case CIPHER_TYPE: + case CIPHER_INSTR_TYPE: if (options->mem_type == UADK_AUTO) - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, SEC_OP_TYPE_MAX, + g_sched = wd_sched_rr_alloc(options->sched_type, SEC_OP_TYPE_MAX, max_node, wd_cipher_poll_ctx); else g_sched = wd_sched_rr_alloc(SCHED_POLICY_DEV, SEC_OP_TYPE_MAX, @@ -739,15 +740,16 @@ static int init_ctx_config(struct acc_option *options) break; case AEAD_TYPE: if (options->mem_type == UADK_AUTO) - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, SEC_OP_TYPE_MAX, + g_sched = wd_sched_rr_alloc(options->sched_type, SEC_OP_TYPE_MAX, max_node, wd_aead_poll_ctx); else g_sched = wd_sched_rr_alloc(SCHED_POLICY_DEV, SEC_OP_TYPE_MAX, max_node, wd_aead_poll_ctx); break; case DIGEST_TYPE: + case DIGEST_INSTR_TYPE: if (options->mem_type == UADK_AUTO) - g_sched = wd_sched_rr_alloc(SCHED_POLICY_RR, SEC_OP_TYPE_MAX, + g_sched = wd_sched_rr_alloc(options->sched_type, SEC_OP_TYPE_MAX, max_node, wd_digest_poll_ctx); else g_sched = wd_sched_rr_alloc(SCHED_POLICY_DEV, SEC_OP_TYPE_MAX, @@ -778,14 +780,19 @@ static int init_ctx_config(struct acc_option *options) /* init */ switch(subtype) { case CIPHER_TYPE: + case CIPHER_INSTR_TYPE: ret = wd_cipher_init(&g_ctx_cfg, g_sched); break; case AEAD_TYPE: ret = wd_aead_init(&g_ctx_cfg, g_sched); break; case DIGEST_TYPE: + case DIGEST_INSTR_TYPE: ret = wd_digest_init(&g_ctx_cfg, g_sched); break; + default: + SEC_TST_PRT("failed to parse alg subtype!\n"); + goto free_sched; } if (ret) { SEC_TST_PRT("failed to init sec ctx!\n"); @@ -814,12 +821,14 @@ static void uninit_ctx_config(int subtype) /* uninit */ switch(subtype) { case CIPHER_TYPE: + case CIPHER_INSTR_TYPE: wd_cipher_uninit(); break; case AEAD_TYPE: wd_aead_uninit(); break; case DIGEST_TYPE: + case DIGEST_INSTR_TYPE: wd_digest_uninit(); break; default: @@ -845,6 +854,7 @@ static void uninit_ctx_config2(int subtype) wd_aead_uninit2(); break; case DIGEST_TYPE: + case DIGEST_INSTR_TYPE: wd_digest_uninit2(); break; default: @@ -885,7 +895,12 @@ static int init_ctx_config2(struct acc_option *options) numa_bitmask_setall(cparams.bmp); - if (mode == CTX_MODE_SYNC) + /* SINGLE scheduler hardcodes idx=0 for sync, idx=1 for async, + * so both sync and async contexts must be allocated together */ + if (options->sched_type == SCHED_POLICY_SINGLE) { + ctx_set_num->sync_ctx_num = g_ctxnum; + ctx_set_num->async_ctx_num = g_ctxnum; + } else if (mode == CTX_MODE_SYNC) ctx_set_num->sync_ctx_num = g_ctxnum; else ctx_set_num->async_ctx_num = g_ctxnum; @@ -894,33 +909,39 @@ static int init_ctx_config2(struct acc_option *options) switch(subtype) { case CIPHER_TYPE: if (options->mem_type == UADK_AUTO) - ret = wd_cipher_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + ret = wd_cipher_init2_(alg_name, options->sched_type, options->task_type, &cparams); else ret = wd_cipher_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); if (ret) SEC_TST_PRT("failed to do cipher init2!\n"); break; case CIPHER_INSTR_TYPE: - ret = wd_cipher_init2(alg_name, SCHED_POLICY_NONE, TASK_INSTR); + ret = wd_cipher_init2_(alg_name, options->sched_type, options->task_type, &cparams); if (ret) - SEC_TST_PRT("failed to do cipher intruction init2!\n"); + SEC_TST_PRT("failed to do cipher instruction init2!\n"); break; case AEAD_TYPE: if (options->mem_type == UADK_AUTO) - ret = wd_aead_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + ret = wd_aead_init2_(alg_name, options->sched_type, options->task_type, &cparams); else ret = wd_aead_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); if (ret) SEC_TST_PRT("failed to do aead init2!\n"); break; case DIGEST_TYPE: - if (options->mem_type == UADK_AUTO) - ret = wd_digest_init2_(alg_name, SCHED_POLICY_RR, options->task_type, &cparams); - else + if (options->mem_type == UADK_AUTO) { + cparams.op_type_num = 1; + ret = wd_digest_init2_(alg_name, options->sched_type, options->task_type, &cparams); + } else ret = wd_digest_init2_(alg_name, SCHED_POLICY_DEV, options->task_type, &cparams); if (ret) SEC_TST_PRT("failed to do digest init2!\n"); break; + case DIGEST_INSTR_TYPE: + ret = wd_digest_init2_(alg_name, options->sched_type, options->task_type, &cparams); + if (ret) + SEC_TST_PRT("failed to do digest instruction init2!\n"); + break; } if (ret) { SEC_TST_PRT("failed to do cipher init2!\n"); @@ -931,7 +952,6 @@ out_freectx: free(ctx_set_num); return ret; - } static void get_aead_data(u8 *addr, u32 size) @@ -1662,12 +1682,14 @@ static void *sec_uadk_poll(void *data) switch(pdata->subtype) { case CIPHER_TYPE: + case CIPHER_INSTR_TYPE: uadk_poll_ctx = wd_cipher_poll_ctx; break; case AEAD_TYPE: uadk_poll_ctx = wd_aead_poll_ctx; break; case DIGEST_TYPE: + case DIGEST_INSTR_TYPE: uadk_poll_ctx = wd_digest_poll_ctx; break; default: @@ -1715,6 +1737,12 @@ static void *sec_uadk_poll2(void *data) case DIGEST_TYPE: uadk_poll_policy = wd_digest_poll; break; + case CIPHER_INSTR_TYPE: + uadk_poll_policy = wd_cipher_poll; + break; + case DIGEST_INSTR_TYPE: + uadk_poll_policy = wd_digest_poll; + break; default: SEC_TST_PRT("<<<<<<async poll interface is NULL!\n"); return NULL; @@ -2362,6 +2390,7 @@ int sec_uadk_sync_threads(struct acc_option *options) uadk_sec_sync_run = sec_uadk_aead_sync; break; case DIGEST_TYPE: + case DIGEST_INSTR_TYPE: uadk_sec_sync_run = sec_uadk_digest_sync; break; default: @@ -2418,12 +2447,14 @@ int sec_uadk_async_threads(struct acc_option *options) switch (options->subtype) { case CIPHER_TYPE: + case CIPHER_INSTR_TYPE: uadk_sec_async_run = sec_uadk_cipher_async; break; case AEAD_TYPE: uadk_sec_async_run = sec_uadk_aead_async; break; case DIGEST_TYPE: + case DIGEST_INSTR_TYPE: uadk_sec_async_run = sec_uadk_digest_async; break; } diff --git a/uadk_tool/benchmark/uadk_benchmark.c b/uadk_tool/benchmark/uadk_benchmark.c index 09e99e2..a38cb3e 100644 --- a/uadk_tool/benchmark/uadk_benchmark.c +++ b/uadk_tool/benchmark/uadk_benchmark.c @@ -21,6 +21,9 @@ /*----------------------------------------head struct--------------------------------------------------------*/ static unsigned int g_run_state = 1; static struct acc_option *g_run_options; +static __u64 g_total_sent; +static __u64 g_total_recv; +static __u64 g_send_stopped; static pthread_mutex_t acc_mutex = PTHREAD_MUTEX_INITIALIZER; static struct _recv_data { double pkg_len; @@ -132,7 +135,7 @@ static struct acc_alg_item alg_options[] = { {"ofb(sm4)", "sm4-128-ofb", SM4_128_OFB}, {"cfb(sm4)", "sm4-128-cfb", SM4_128_CFB}, {"xts(sm4)", "sm4-128-xts", SM4_128_XTS}, - {"xts(sm4)", "sm4-128-xts-gb", SM4_128_XTS_GB}, + {"xts-gb(sm4)", "sm4-128-xts-gb", SM4_128_XTS_GB}, {"ccm(aes)", "aes-128-ccm", AES_128_CCM}, {"ccm(aes)", "aes-192-ccm", AES_192_CCM}, {"ccm(aes)", "aes-256-ccm", AES_256_CCM}, @@ -188,6 +191,9 @@ void init_recv_data(void) g_recv_data.pkg_len = 0.0; g_recv_data.send_times = 0; g_recv_data.recv_times = 0; + __atomic_store_n(&g_total_sent, 0, __ATOMIC_RELEASE); + __atomic_store_n(&g_total_recv, 0, __ATOMIC_RELEASE); + __atomic_store_n(&g_send_stopped, 0, __ATOMIC_RELEASE); } int get_run_state(void) @@ -200,6 +206,36 @@ void set_run_state(int state) g_run_state = state; } +void add_total_sent(__u32 cnt) +{ + __atomic_add_fetch(&g_total_sent, cnt, __ATOMIC_RELEASE); +} + +__u64 get_total_sent(void) +{ + return __atomic_load_n(&g_total_sent, __ATOMIC_ACQUIRE); +} + +void add_total_recv(__u32 cnt) +{ + __atomic_add_fetch(&g_total_recv, cnt, __ATOMIC_RELEASE); +} + +__u64 get_total_recv(void) +{ + return __atomic_load_n(&g_total_recv, __ATOMIC_ACQUIRE); +} + +void add_send_stopped(void) +{ + __atomic_add_fetch(&g_send_stopped, 1, __ATOMIC_RELEASE); +} + +__u64 get_send_stopped(void) +{ + return __atomic_load_n(&g_send_stopped, __ATOMIC_ACQUIRE); +} + int uadk_parse_dev_id(char *dev_name) { char *last_dash = NULL; @@ -359,7 +395,7 @@ void get_rand_data(u8 *addr, u32 size) } for (i = 0; i < num_u64; i++) { - /* Use nrand48��it will auto update rand_state */ + /* Use nrand48, it will auto update rand_state */ rand48_result = nrand48(rand_state); *((u64 *)addr + i) = rand48_result; } @@ -479,10 +515,17 @@ static void parse_alg_param(struct acc_option *option) option->subtype = ECDSA_TYPE; } else if (option->algtype <= SM4_128_XTS_GB) { snprintf(option->algclass, MAX_ALG_NAME, "%s", "cipher"); - if (option->modetype == INSTR_MODE) + if (option->modetype == INSTR_MODE) { + option->subtype = CIPHER_INSTR_TYPE; + option->sched_type = SCHED_POLICY_NONE; + option->task_type = TASK_INSTR; + } else if (option->modetype == MULTIBUF_MODE) { option->subtype = CIPHER_INSTR_TYPE; - else + option->sched_type = SCHED_POLICY_SINGLE; + option->task_type = TASK_INSTR; + } else { option->subtype = CIPHER_TYPE; + } option->acctype = SEC_TYPE; } else if (option->algtype <= SM4_128_GCM) { snprintf(option->algclass, MAX_ALG_NAME, "%s", "aead"); @@ -493,9 +536,11 @@ static void parse_alg_param(struct acc_option *option) option->subtype = DIGEST_TYPE; option->acctype = SEC_TYPE; if (option->modetype == INSTR_MODE) { + option->subtype = DIGEST_INSTR_TYPE; option->sched_type = SCHED_POLICY_NONE; option->task_type = TASK_INSTR; } else if (option->modetype == MULTIBUF_MODE) { + option->subtype = DIGEST_INSTR_TYPE; option->sched_type = SCHED_POLICY_SINGLE; option->task_type = TASK_INSTR; } @@ -611,6 +656,8 @@ static void dump_param(struct acc_option *option) ACC_TST_PRT(" [--latency]: %u\n", option->latency); ACC_TST_PRT(" [--init2]: %u\n", option->inittype); ACC_TST_PRT(" [--device]: %s\n", option->device); + ACC_TST_PRT(" [--sched]: %u\n", option->sched_type); + ACC_TST_PRT(" [--task]: %u\n", option->task_type); } int acc_benchmark_run(struct acc_option *option) @@ -620,8 +667,6 @@ int acc_benchmark_run(struct acc_option *option) int i, ret = 0; int status; - option->sched_type = SCHED_POLICY_RR; - option->task_type = TASK_HW; parse_alg_param(option); dump_param(option); g_run_options = option; @@ -694,6 +739,8 @@ int acc_default_case(struct acc_option *option) option->multis = 1; option->ctxnums = 2; option->inittype = INIT_TYPE; + option->sched_type = SCHED_POLICY_RR; + option->task_type = TASK_HW; return acc_benchmark_run(option); } @@ -739,6 +786,12 @@ void print_benchmark_help(void) ACC_TST_PRT(" select init2 mode in the init interface of UADK SVA\n"); ACC_TST_PRT(" [--device]:\n"); ACC_TST_PRT(" select device to do task\n"); + ACC_TST_PRT(" [--memory]:\n"); + ACC_TST_PRT(" set memory type to do task\n"); + ACC_TST_PRT(" [--sched 0~6]:\n"); + ACC_TST_PRT(" set scheduler policy (0:RR 1:NONE 2:SINGLE 3:DEV 4:LOOP 5:HUNGRY 6:INSTR)\n"); + ACC_TST_PRT(" [--task 0~2]:\n"); + ACC_TST_PRT(" set task type (0:MIX 1:HW 2:INSTR)\n"); ACC_TST_PRT(" [--help] = usage\n"); ACC_TST_PRT("Example\n"); ACC_TST_PRT(" ./uadk_tool benchmark --alg aes-128-cbc --mode sva --opt 0 --sync\n"); @@ -783,6 +836,8 @@ int acc_cmd_parse(int argc, char *argv[], struct acc_option *option) {"device", required_argument, 0, 18}, {"memory", required_argument, 0, 19}, {"sgl", no_argument, 0, 20}, + {"sched", required_argument, 0, 21}, + {"task", required_argument, 0, 22}, {0, 0, 0, 0} }; @@ -860,6 +915,12 @@ int acc_cmd_parse(int argc, char *argv[], struct acc_option *option) case 20: option->data_fmt = WD_SGL_BUF; break; + case 21: + option->sched_type = strtol(optarg, NULL, 0); + break; + case 22: + option->task_type = strtol(optarg, NULL, 0); + break; default: ACC_TST_PRT("invalid: bad input parameter!\n"); print_benchmark_help(); @@ -942,6 +1003,14 @@ int acc_option_convert(struct acc_option *option) goto param_err; } + if (option->sched_type >= SCHED_POLICY_BUTT) { + ACC_TST_PRT("uadk benchmark scheduler type set error!\n"); + goto param_err; + } + + if (option->task_type >= TASK_MAX_TYPE) + option->task_type = TASK_HW; + return 0; param_err: diff --git a/uadk_tool/benchmark/uadk_benchmark.h b/uadk_tool/benchmark/uadk_benchmark.h index 83fd7fa..e06c0ea 100644 --- a/uadk_tool/benchmark/uadk_benchmark.h +++ b/uadk_tool/benchmark/uadk_benchmark.h @@ -117,6 +117,7 @@ enum alg_type { X25519_TYPE, X448_TYPE, CIPHER_INSTR_TYPE, + DIGEST_INSTR_TYPE, }; enum sync_type { @@ -229,6 +230,12 @@ extern void get_rand_data(u8 *addr, u32 size); extern void add_recv_data(u32 cnt, u32 pkglen); extern void add_send_complete(void); extern u32 get_recv_time(void); +extern void add_total_sent(u32 cnt); +extern u64 get_total_sent(void); +extern void add_total_recv(u32 cnt); +extern u64 get_total_recv(void); +extern void add_send_stopped(void); +extern u64 get_send_stopped(void); extern void cal_avg_latency(u32 count); extern int get_alg_name(int alg, char *alg_name); extern void segmentfault_handler(int sig); diff --git a/uadk_tool/benchmark/zip_uadk_benchmark.c b/uadk_tool/benchmark/zip_uadk_benchmark.c index d0f1795..86e6404 100644 --- a/uadk_tool/benchmark/zip_uadk_benchmark.c +++ b/uadk_tool/benchmark/zip_uadk_benchmark.c @@ -20,7 +20,8 @@ #define MAX_POOL_LENTH_COMP 1 #define COMPRESSION_RATIO_FACTOR 0.7 #define CHUNK_SIZE (128 * 1024) -#define MAX_UNRECV_PACKET_NUM 1 +#define MAX_DRAIN_RETRY 10000 +#define ZIP_ASYNC_DRAIN_RETRY MAX_DRAIN_RETRY struct uadk_bd { u8 *src; u8 *dst; @@ -42,11 +43,6 @@ enum ZIP_OP_MODE { STREAM_MODE }; -enum ZIP_THREAD_STATE { - THREAD_PROCESSING, - THREAD_COMPLETED -}; - struct zip_async_tag { handle_t sess; u32 td_id; @@ -83,7 +79,6 @@ static unsigned int g_thread_num; static unsigned int g_ctxnum; static unsigned int g_pktlen; static unsigned int g_prefetch; -static unsigned int g_state; static unsigned int g_dev_id; static unsigned int g_data_fmt; @@ -397,7 +392,7 @@ static int init_ctx_config2(struct acc_option *options) /* init */ if (options->mem_type == UADK_AUTO) - ret = wd_comp_init2_(alg_name, SCHED_POLICY_RR, TASK_HW, &cparams); + ret = wd_comp_init2_(alg_name, options->sched_type, options->task_type, &cparams); else ret = wd_comp_init2_(alg_name, SCHED_POLICY_DEV, TASK_HW, &cparams); if (ret) { @@ -982,19 +977,48 @@ static void *zip_uadk_poll(void *data) u32 id = pdata->td_id; u32 count = 0; u32 recv = 0; + u32 drain_retry = 0; int ret; if (id > g_ctxnum) return NULL; - while (g_state == THREAD_PROCESSING) { + while (1) { ret = wd_comp_poll_ctx(id, expt, &recv); count += recv; recv = 0; if (unlikely(ret != -WD_EAGAIN && ret < 0)) { ZIP_TST_PRT("poll ret: %d!\n", ret); + add_total_recv(count); goto recv_error; } + if (get_run_state() == 0) { + add_total_recv(count); + break; + } + } + + while (get_send_stopped() != g_thread_num || + get_total_recv() < get_total_sent()) { + ret = wd_comp_poll_ctx(id, expt, &recv); + if (unlikely(ret != -WD_EAGAIN && ret < 0)) { + ZIP_TST_PRT("poll ret: %d!\n", ret); + goto recv_error; + } + if (recv == 0) { + usleep(SEND_USLEEP); + drain_retry++; + } else { + add_total_recv(recv); + drain_retry = 0; + recv = 0; + } + if (drain_retry >= ZIP_ASYNC_DRAIN_RETRY) { + ZIP_TST_PRT("drain timeout: sent=%llu recv=%llu stopped=%llu/%u\n", + get_total_sent(), get_total_recv(), + get_send_stopped(), g_thread_num); + break; + } } recv_error: @@ -1008,16 +1032,45 @@ static void *zip_uadk_poll2(void *data) u32 expt = ACC_QUEUE_SIZE * g_thread_num; u32 count = 0; u32 recv = 0; + u32 drain_retry = 0; int ret; - while (g_state == THREAD_PROCESSING) { + while (1) { ret = wd_comp_poll(expt, &recv); count += recv; recv = 0; if (unlikely(ret != -WD_EAGAIN && ret < 0)) { ZIP_TST_PRT("poll ret: %d!\n", ret); + add_total_recv(count); goto recv_error; } + if (get_run_state() == 0) { + add_total_recv(count); + break; + } + } + + while (get_send_stopped() != g_thread_num || + get_total_recv() < get_total_sent()) { + ret = wd_comp_poll(expt, &recv); + if (unlikely(ret != -WD_EAGAIN && ret < 0)) { + ZIP_TST_PRT("poll ret: %d!\n", ret); + goto recv_error; + } + if (recv == 0) { + usleep(SEND_USLEEP); + drain_retry++; + } else { + add_total_recv(recv); + drain_retry = 0; + recv = 0; + } + if (drain_retry >= ZIP_ASYNC_DRAIN_RETRY) { + ZIP_TST_PRT("drain timeout: sent=%llu recv=%llu stopped=%llu/%u\n", + get_total_sent(), get_total_recv(), + get_send_stopped(), g_thread_num); + break; + } } recv_error: @@ -1331,6 +1384,15 @@ static void *zip_uadk_blk_lz77_async_run(void *arg) count++; __atomic_add_fetch(&pdata->send_cnt, 1, __ATOMIC_RELAXED); } + + add_total_sent(count); + add_send_stopped(); + + if (count) { + while (get_recv_time() != g_ctxnum) + usleep(SEND_USLEEP); + } + wd_comp_free_sess(h_sess); add_send_complete(); @@ -1574,6 +1636,14 @@ static void *zip_uadk_blk_async_run(void *arg) __atomic_add_fetch(&pdata->send_cnt, 1, __ATOMIC_RELAXED); } + add_total_sent(count); + add_send_stopped(); + + if (count) { + while (get_recv_time() != g_ctxnum) + usleep(SEND_USLEEP); + } + wd_comp_free_sess(h_sess); add_send_complete(); @@ -1726,14 +1796,6 @@ static int zip_uadk_async_threads(struct acc_option *options) } } - /* wait for the poll to clear packets */ - g_state = THREAD_PROCESSING; - for (i = 0; i < g_thread_num;) { - if (threads_args[i].send_cnt <= threads_args[i].tag->recv_cnt + MAX_UNRECV_PACKET_NUM) - i++; - } - g_state = THREAD_COMPLETED; // finish poll - for (i = 0; i < g_ctxnum; i++) { ret = pthread_join(pollid[i], NULL); if (ret) { -- 2.43.0
The ZIP hardware design is compatible with the old BD, eliminating the need for uadk adaptation. Therefore, the special processing for v5 has been removed. In stateful decompression, the hardware reports its context state in ctx_core_status (the low 9 bits of dw30/isize). When ctx_core_status is non-zero while neither input is consumed nor output produced, the hardware needs the request to be resent. Return WD_EAGAIN to the user so that the request can be retried. Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com> --- drv/hisi_comp.c | 24 ++++++++++++++++++------ v1/drv/hisi_zip_udrv.c | 17 +++++++++++++++-- v1/drv/hisi_zip_udrv.h | 1 + 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/drv/hisi_comp.c b/drv/hisi_comp.c index c75e734..b798811 100644 --- a/drv/hisi_comp.c +++ b/drv/hisi_comp.c @@ -60,6 +60,7 @@ #define HZ_DECOMP_END 0x13 #define HZ_CTX_ST_MASK 0x000f +#define HZ_CTX_CORE_STATUS_MASK 0x1ff #define HZ_CTX_BFINAL_MASK 0x80 #define HZ_CTX_STORE_MASK 0x7ffff #define HZ_LSTBLK_MASK 0x0100 @@ -81,6 +82,7 @@ #define max_in_data_size(outl) ((__u32)(((__u64)(outl) << 3) / 9) & 0xfffffffc) #define HZ_MAX_SIZE (8 * 1024 * 1024) +#define LZ4_MAX_SIZE ((__u32)0x100000) #define HW_CTX_SIZE 0x10000 #define RSV_OFFSET 64 @@ -726,15 +728,12 @@ static int check_lz4_msg(struct wd_comp_msg *msg, enum wd_buff_type buf_type) return -WD_EINVAL; } - if (buf_type != WD_FLAT_BUF) - return 0; - - if (unlikely(msg->req.src_len == 0 || msg->req.src_len > HZ_MAX_SIZE)) { - WD_ERR("invalid: lz4 input size can't be zero or more than 8M size max!\n"); + if (unlikely(msg->req.src_len == 0 || msg->req.src_len > LZ4_MAX_SIZE)) { + WD_ERR("invalid: lz4 input size can't be zero or more than 1M size max!\n"); return -WD_EINVAL; } - if (unlikely(msg->avail_out > HZ_MAX_SIZE)) + if (unlikely(msg->avail_out > HZ_MAX_SIZE && buf_type == WD_FLAT_BUF)) msg->avail_out = HZ_MAX_SIZE; return 0; @@ -1672,6 +1671,7 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, struct wd_comp_msg *msg) { __u32 buf_type = (sqe->dw9 & HZ_BUF_TYPE_MASK) >> BUF_TYPE_SHIFT; + __u16 ctx_core_status = sqe->isize & HZ_CTX_CORE_STATUS_MASK; __u32 ctx_win_len = sqe->ctx_dw2 & CTX_WIN_LEN_MASK; __u16 ctx_st = sqe->ctx_dw0 & HZ_CTX_ST_MASK; __u16 lstblk = sqe->dw3 & HZ_LSTBLK_MASK; @@ -1731,6 +1731,18 @@ static int parse_zip_sqe(struct hisi_qp *qp, struct hisi_zip_sqe *sqe, (sqe->ctx_dw0 & HZ_CTX_BFINAL_MASK) && (sqe->ctx_dw1 & HZ_CTX_STORE_MASK)) recv_msg->req.status = WD_EAGAIN; + /* + * The ctx_core_status reflects the hardware context state. + * In stateful decompression, if it is non-zero while neither + * input is consumed nor output produced, the hardware + * needs the request to be resent with more input and output, + * so report WD_EAGAIN to the user. + */ + if (!recv_msg->req.status && recv_msg->stream_mode == WD_COMP_STATEFUL && + recv_msg->req.op_type == WD_DIR_DECOMPRESS && ctx_core_status && + !recv_msg->in_cons && !recv_msg->produced) + recv_msg->req.status = WD_EAGAIN; + /* * It need to analysis the data cache by hardware. * If the cache data is a complete huffman block, diff --git a/v1/drv/hisi_zip_udrv.c b/v1/drv/hisi_zip_udrv.c index b73a929..96604cd 100644 --- a/v1/drv/hisi_zip_udrv.c +++ b/v1/drv/hisi_zip_udrv.c @@ -447,7 +447,7 @@ int qm_parse_zip_sqe(void *hw_msg, const struct qm_queue_info *info, qm_parse_zip_sqe_set_status(recv_msg, status, lstblk, ctx_st); if (ctx_st == HW_DECOMPING_NO_SPACE && recv_msg->in_size == recv_msg->in_cons && ctx_bfinal && (sqe->ctx_dw1 & HZ_CTX_STORE_MASK)) - recv_msg->status = WCRYPTO_DECOMP_BLK_NOSTART; + recv_msg->status = WCRYPTO_DECOMP_END_NOSPACE; return 1; } @@ -850,6 +850,7 @@ int qm_parse_zip_sqe_v3(void *hw_msg, const struct qm_queue_info *info, { struct wcrypto_comp_msg *recv_msg = info->req_cache[i]; struct hisi_zip_sqe_v3 *sqe = hw_msg; + __u16 ctx_core_status = sqe->isize & HZ_CTX_CORE_STATUS_MASK; __u16 ctx_bfinal = sqe->ctx_dw0 & HZ_CTX_BFINAL_MASK; __u32 ctx_win_len = sqe->ctx_dw2 & CTX_WIN_LEN_MASK; __u16 ctx_st = sqe->ctx_dw0 & HZ_CTX_ST_MASK; @@ -912,7 +913,19 @@ int qm_parse_zip_sqe_v3(void *hw_msg, const struct qm_queue_info *info, qm_parse_zip_sqe_set_status(recv_msg, status, lstblk, ctx_st); if (ctx_st == HW_DECOMPING_NO_SPACE && recv_msg->in_size == recv_msg->in_cons && ctx_bfinal && (sqe->ctx_dw1 & HZ_CTX_STORE_MASK)) - recv_msg->status = WCRYPTO_DECOMP_BLK_NOSTART; + recv_msg->status = WCRYPTO_DECOMP_END_NOSPACE; + + /* + * The ctx_core_status reflects the hardware context state. + * In stateful decompression, if it is non-zero while neither + * input is consumed nor output produced, the hardware + * needs the request to be resent with more input and output, + * so report WD_EAGAIN to the user. + */ + if (!recv_msg->status && recv_msg->stream_mode == WCRYPTO_COMP_STATEFUL && + recv_msg->op_type == WCRYPTO_INFLATE && ctx_core_status && + !recv_msg->in_cons && !recv_msg->produced) + recv_msg->status = WD_EAGAIN; /* * It need to analysis the data cache by hardware. diff --git a/v1/drv/hisi_zip_udrv.h b/v1/drv/hisi_zip_udrv.h index 1037f43..f1abb5e 100644 --- a/v1/drv/hisi_zip_udrv.h +++ b/v1/drv/hisi_zip_udrv.h @@ -120,6 +120,7 @@ struct hisi_zip_sqe_v3 { #define HZ_REF_VTYPE_SHIFT 12 #define HZ_BLK_SIZE_SHIFT 16 #define HZ_CTX_ST_MASK 0x000f +#define HZ_CTX_CORE_STATUS_MASK 0x1ff #define HZ_CTX_BFINAL_MASK 0x80 #define HZ_CTX_STORE_MASK 0x7ffff #define HZ_LSTBLK_MASK 0x0100 -- 2.43.0
From: Longfang Liu <liulongfang@huawei.com> Strengthen the initialization logic in wd_alg_init2_ by adding WD_INIT2_MAX_RETRY (32) retry limit across all 10 algorithm modules (aead, agg, cipher, comp, dh, digest, ecc, join_gather, rsa, udma) to prevent infinite retry loops when no device is available. Signed-off-by: Longfang Liu <liulongfang@huawei.com> --- include/wd_alg_common.h | 1 + wd_aead.c | 5 +++++ wd_cipher.c | 8 +++++++- wd_dh.c | 6 ++++++ wd_ecc.c | 6 ++++++ wd_join_gather.c | 6 ++++++ wd_rsa.c | 6 ++++++ 7 files changed, 37 insertions(+), 1 deletion(-) diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index be4e3b5..05cc5c0 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -28,6 +28,7 @@ extern "C" { #define MAX_STR_LEN 256 #define CTX_TYPE_INVALID 9999 #define POLL_TIME 1000 +#define WD_INIT2_MAX_RETRY 32 /* Key size of chiper */ #define MAX_CIPHER_KEY_SIZE 64 diff --git a/wd_aead.c b/wd_aead.c index c0983a6..ba3c116 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -816,6 +816,11 @@ int wd_aead_init2_(char *alg, __u32 sched_type, int task_type, goto out_uninit; while (ret != 0) { + if (try_cnt++ >= WD_INIT2_MAX_RETRY) { + WD_ERR("failed to init2 after %d retries.\n", + WD_INIT2_MAX_RETRY); + goto out_dlclose; + } memset(&wd_aead_setting.config, 0, sizeof(struct wd_ctx_config_internal)); /* Init ctx param and prepare for ctx request */ aead_ctx_params.ctx_set_num = aead_ctx_num; diff --git a/wd_cipher.c b/wd_cipher.c index e0670f5..5c605ca 100644 --- a/wd_cipher.c +++ b/wd_cipher.c @@ -502,6 +502,7 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_nums cipher_ctx_num[WD_CIPHER_DECRYPTION + 1] = {0}; struct wd_ctx_params cipher_ctx_params = {0}; int state, ret = -WD_EINVAL; + int try_cnt = 0; bool flag; if (!wd_cipher_atfork_registered) { @@ -530,6 +531,11 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, goto out_uninit; while (ret != 0) { + if (try_cnt++ >= WD_INIT2_MAX_RETRY) { + WD_ERR("failed to init2 after %d retries.\n", + WD_INIT2_MAX_RETRY); + goto out_dlclose; + } memset(&wd_cipher_setting.config, 0, sizeof(struct wd_ctx_config_internal)); @@ -557,7 +563,7 @@ int wd_cipher_init2_(char *alg, __u32 sched_type, int task_type, wd_ctx_param_uninit(&cipher_ctx_params); continue; } - WD_ERR("fail to init alg attrs.\n"); + WD_ERR("failed to init alg attrs.\n"); goto out_params_uninit; } } diff --git a/wd_dh.c b/wd_dh.c index 612e15f..8026fd6 100644 --- a/wd_dh.c +++ b/wd_dh.c @@ -240,6 +240,7 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param struct wd_ctx_nums dh_ctx_num[WD_DH_PHASE2] = {0}; struct wd_ctx_params dh_ctx_params = {0}; int state, ret = -WD_EINVAL; + int try_cnt = 0; if (!wd_dh_atfork_registered) { if (pthread_atfork(NULL, NULL, wd_dh_clear_status) == 0) @@ -266,6 +267,11 @@ int wd_dh_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_param goto out_clear_init; while (ret) { + if (try_cnt++ >= WD_INIT2_MAX_RETRY) { + WD_ERR("failed to init2 after %d retries.\n", + WD_INIT2_MAX_RETRY); + goto out_driver; + } memset(&wd_dh_setting.config, 0, sizeof(struct wd_ctx_config_internal)); /* Init ctx param and prepare for ctx request */ dh_ctx_params.ctx_set_num = dh_ctx_num; diff --git a/wd_ecc.c b/wd_ecc.c index 2d1aa77..4b141c4 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -311,6 +311,7 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para struct wd_ctx_nums ecc_ctx_num[WD_EC_OP_MAX] = {0}; struct wd_ctx_params ecc_ctx_params = {0}; int state, ret = -WD_EINVAL; + int try_cnt = 0; bool flag; if (!wd_ecc_atfork_registered) { @@ -339,6 +340,11 @@ int wd_ecc_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para goto out_clear_init; while (ret) { + if (try_cnt++ >= WD_INIT2_MAX_RETRY) { + WD_ERR("failed to init2 after %d retries.\n", + WD_INIT2_MAX_RETRY); + goto out_driver; + } memset(&wd_ecc_setting.config, 0, sizeof(struct wd_ctx_config_internal)); /* Init ctx param and prepare for ctx request */ diff --git a/wd_join_gather.c b/wd_join_gather.c index 8857c8f..e43dbf9 100644 --- a/wd_join_gather.c +++ b/wd_join_gather.c @@ -803,6 +803,7 @@ int wd_join_gather_init(char *alg, __u32 sched_type, int task_type, struct wd_ctx_params join_gather_ctx_params = {0}; struct wd_ctx_nums join_gather_ctx_num = {0}; int ret = -WD_EINVAL; + int try_cnt = 0; int state; bool flag; @@ -832,6 +833,11 @@ int wd_join_gather_init(char *alg, __u32 sched_type, int task_type, goto out_uninit; while (ret != 0) { + if (try_cnt++ >= WD_INIT2_MAX_RETRY) { + WD_ERR("failed to init2 after %d retries.\n", + WD_INIT2_MAX_RETRY); + goto out_driver; + } memset(&wd_join_gather_setting.config, 0, sizeof(struct wd_ctx_config_internal)); join_gather_ctx_params.ctx_set_num = &join_gather_ctx_num; ret = wd_ctx_param_init(&join_gather_ctx_params, ctx_params, alg, diff --git a/wd_rsa.c b/wd_rsa.c index 754c28a..c1f2d9a 100644 --- a/wd_rsa.c +++ b/wd_rsa.c @@ -282,6 +282,7 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para struct wd_ctx_nums rsa_ctx_num[WD_RSA_GENKEY] = {0}; struct wd_ctx_params rsa_ctx_params = {0}; int state, ret = -WD_EINVAL; + int try_cnt = 0; if (!wd_rsa_atfork_registered) { if (pthread_atfork(NULL, NULL, wd_rsa_clear_status) == 0) @@ -308,6 +309,11 @@ int wd_rsa_init2_(char *alg, __u32 sched_type, int task_type, struct wd_ctx_para goto out_clear_init; while (ret) { + if (try_cnt++ >= WD_INIT2_MAX_RETRY) { + WD_ERR("failed to init2 after %d retries.\n", + WD_INIT2_MAX_RETRY); + goto out_dlclose; + } memset(&wd_rsa_setting.config, 0, sizeof(struct wd_ctx_config_internal)); /* Init ctx param and prepare for ctx request */ -- 2.43.0
From: Wenkai Lin <linwenkai6@hisilicon.com> Add --enable-debug configure option in configure.ac and Makefile.am. When enabled, build with -g -O0 -fno-omit-frame-pointer -DDEBUG to preserve symbols and disable optimization for full stack traces during debugging. Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com> --- Makefile.am | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/Makefile.am b/Makefile.am index d2f6cfe..23d3d23 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,6 +1,7 @@ ACLOCAL_AMFLAGS = -I m4 -I./include AUTOMAKE_OPTIONS = foreign subdir-objects -AM_CFLAGS=-Wall -Werror -fno-strict-aliasing -I$(top_srcdir)/include +AM_CFLAGS=-std=gnu11 -Wall -Werror -Wextra -Wno-unused-parameter -Wfloat-equal \ + -fno-common -fno-strict-aliasing -I$(top_srcdir)/include AM_CFLAGS+=-fPIC -fPIE -pie -fstack-protector-strong -ftrapv -Wl,-z,relro,-z,now if WD_DEBUG @@ -10,6 +11,15 @@ else # Release build: optimize and strip symbols AM_CFLAGS+=-O2 -D_FORTIFY_SOURCE=2 -Wl,-s endif +AM_CFLAGS += -Wall -Wuninitialized -Wno-error -Wno-error=format -Wundef \ +-Wunused -Wdate-time -Wfloat-equal -Wshadow -Wvla -Wdisabled-optimization \ +-Wempty-body -Wignored-qualifiers -Wimplicit-fallthrough=3 -Wtype-limits \ +-Wshift-negative-value -Wswitch-default -Wframe-larger-than=8192 -Wshift-overflow=2 \ +-Wwrite-strings -Wmissing-format-attribute -Wformat-nonliteral -Wduplicated-cond \ +-Wtrampolines -Wlogical-op -Wsuggest-attribute=format -Wduplicated-branches \ +-Wmissing-include-dirs -Wformat-signedness -Wmissing-declarations -Wreturn-local-addr \ +-Wredundant-decls -Wfloat-conversion -Wmissing-prototypes -Wstrict-prototypes + CLEANFILES = if WITH_LOG_FILE @@ -77,9 +87,10 @@ libwd_la_SOURCES=wd.c wd_mempool.c wd_bmm.c wd_bmm.h wd.h wd_alg.c wd_alg.h \ v1/drv/hisi_sec_udrv.c v1/drv/hisi_sec_udrv.h libwd_udma_la_SOURCES=wd_udma.h wd_udma_drv.h wd_udma.c \ - wd_util.c wd_util.h wd_sched.c wd_sched.h wd.c wd.h + wd_util.c wd_util.h wd_sched.c wd_sched.h wd.c wd.h -libwd_dae_la_SOURCES=wd_dae.h wd_agg.h wd_agg_drv.h wd_agg.c wd_join_gather.h wd_join_gather_drv.h wd_join_gather.c \ +libwd_dae_la_SOURCES=wd_dae.h wd_agg.h wd_agg_drv.h wd_agg.c \ + wd_join_gather.h wd_join_gather_drv.h wd_join_gather.c \ wd_util.c wd_util.h wd_sched.c wd_sched.h wd.c wd.h libwd_comp_la_SOURCES=wd_comp.c wd_comp.h wd_comp_drv.h wd_util.c wd_util.h \ @@ -87,7 +98,8 @@ libwd_comp_la_SOURCES=wd_comp.c wd_comp.h wd_comp_drv.h wd_util.c wd_util.h \ libhisi_zip_la_SOURCES=drv/hisi_comp.c hisi_comp.h drv/hisi_qm_udrv.c \ drv/hisi_comp_huf.c drv/hisi_comp_huf.h \ - hisi_qm_udrv.h wd_comp_drv.h + hisi_qm_udrv.h wd_comp_drv.h \ + drv/wd_drv.h drv/wd_drv.c libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ wd_aead.c wd_aead.h wd_aead_drv.h \ @@ -101,27 +113,32 @@ libwd_crypto_la_SOURCES=wd_cipher.c wd_cipher.h wd_cipher_drv.h \ libhisi_sec_la_SOURCES=drv/hisi_sec.c drv/hisi_qm_udrv.c \ lib/crypto/aes.c lib/crypto/sm4.c lib/crypto/galois.c \ - hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h sm4.h galois.h \n drv/wd_drv.h drv/wd_drv.c + hisi_qm_udrv.h wd_cipher_drv.h wd_aead_drv.h aes.h sm4.h galois.h \ + drv/wd_drv.h drv/wd_drv.c libhisi_hpre_la_SOURCES=drv/hisi_hpre.c drv/hisi_qm_udrv.c \ - hisi_qm_udrv.h + hisi_qm_udrv.h wd_rsa_drv.h wd_dh_drv.h wd_ecc_drv.h \ + drv/wd_drv.h drv/wd_drv.c if ARCH_ARM64 libisa_ce_la_SOURCES=arm_arch_ce.h drv/isa_ce_sm3.c drv/isa_ce_sm3_armv8.S isa_ce_sm3.h \ - drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h wd_util.c wd_util.h \n drv/wd_drv.h drv/wd_drv.c + drv/isa_ce_sm4.c drv/isa_ce_sm4_armv8.S drv/isa_ce_sm4.h wd_util.c wd_util.h \ + drv/wd_drv.h drv/wd_drv.c libisa_sve_la_SOURCES=drv/hash_mb/hash_mb.c wd_digest_drv.h drv/hash_mb/hash_mb.h \ drv/hash_mb/sm3_sve_common.S drv/hash_mb/sm3_mb_asimd_x1.S \ drv/hash_mb/sm3_mb_asimd_x4.S drv/hash_mb/sm3_mb_sve.S \ drv/hash_mb/md5_sve_common.S drv/hash_mb/md5_mb_asimd_x1.S \ - drv/hash_mb/md5_mb_asimd_x4.S drv/hash_mb/md5_mb_sve.S \n drv/wd_drv.h drv/wd_drv.c + drv/hash_mb/md5_mb_asimd_x4.S drv/hash_mb/md5_mb_sve.S \ + drv/wd_drv.h drv/wd_drv.c endif libhisi_dae_la_SOURCES=drv/hisi_dae.c hisi_dae.h drv/hisi_qm_udrv.c \ - hisi_qm_udrv.h drv/hisi_dae_join_gather.c drv/hisi_dae_common.c \n drv/wd_drv.h drv/wd_drv.c + hisi_qm_udrv.h drv/hisi_dae_join_gather.c drv/hisi_dae_common.c \ + drv/wd_drv.h drv/wd_drv.c libhisi_udma_la_SOURCES=drv/hisi_udma.c drv/hisi_qm_udrv.c \ - hisi_qm_udrv.h + hisi_qm_udrv.h drv/wd_drv.h drv/wd_drv.c if WD_STATIC_DRV AM_CFLAGS += -DWD_STATIC_DRV -fPIC @@ -129,8 +146,8 @@ AM_CFLAGS += -DWD_NO_LOG libwd_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma -lpthread -lrt -libwd_comp_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma -lpthread -libwd_comp_la_DEPENDENCIES = libwd.la +libwd_comp_la_LIBADD = $(libwd_la_OBJECTS) $(libhisi_zip_la_OBJECTS) -ldl -lpthread -lnuma +libwd_comp_la_DEPENDENCIES = libwd.la libhisi_zip.la libhisi_zip_la_LIBADD = -ldl -lpthread -- 2.43.0
From: Weili Qian <qianweili@huawei.com> Drop redundant priv sentinel field from all algorithm modules (aead, cipher, comp, dh, digest, ecc, rsa) and replace with wd_alg_get_init() status guard. Clear comp drv_count on uninit to prevent use-after-free. Fix build errors Signed-off-by: Weili Qian <qianweili@huawei.com> --- drv/hisi_hpre.c | 3 --- drv/hisi_sec.c | 1 - drv/wd_drv.c | 20 ++----------------- include/wd_internal.h | 2 +- wd.c | 3 ++- wd_aead.c | 46 +++++++++++++++++++++---------------------- wd_sched.c | 3 ++- 7 files changed, 30 insertions(+), 48 deletions(-) diff --git a/drv/hisi_hpre.c b/drv/hisi_hpre.c index 9e46d77..0fa6978 100644 --- a/drv/hisi_hpre.c +++ b/drv/hisi_hpre.c @@ -2964,7 +2964,6 @@ static int hpre_ecc_get_extend_ops(void *ops) .drv_data = NULL, \ .extend_ops = NULL, \ .fallback = 0,\ - .init_state = 0,\ .init = hpre_ecc_init,\ .exit = hpre_exit,\ .send = ecc_send,\ @@ -2993,7 +2992,6 @@ static struct wd_alg_driver hpre_rsa_driver = { .op_type_num = 1, .drv_data = NULL, .fallback = 0, - .init_state = 0, .init = hpre_rsa_dh_init, .exit = hpre_exit, .send = rsa_send, @@ -3013,7 +3011,6 @@ static struct wd_alg_driver hpre_dh_driver = { .op_type_num = 1, .drv_data = NULL, .fallback = 0, - .init_state = 0, .init = hpre_rsa_dh_init, .exit = hpre_exit, .send = dh_send, diff --git a/drv/hisi_sec.c b/drv/hisi_sec.c index 91fa4ce..10bb34e 100644 --- a/drv/hisi_sec.c +++ b/drv/hisi_sec.c @@ -793,7 +793,6 @@ static int sec_aead_get_extend_ops(void *ops) .op_type_num = 1,\ .drv_data = NULL,\ .fallback = 0,\ - .init_state = 0,\ .init = hisi_sec_init,\ .exit = hisi_sec_exit,\ .send = alg_type##_send,\ diff --git a/drv/wd_drv.c b/drv/wd_drv.c index d6a7626..9290c8c 100644 --- a/drv/wd_drv.c +++ b/drv/wd_drv.c @@ -10,7 +10,6 @@ int wd_soft_alloc_ctx(char *alg_name, void *params, handle_t *ctx) { - struct wd_drv_ctx_params *ctx_params = (struct wd_drv_ctx_params *)params; struct wd_soft_ctx *sfctx; if (!params || !ctx) { @@ -163,21 +162,6 @@ out_free_list: return ret; } -static int wd_get_alg_class_type(const char *alg_name, char *alg_type) -{ - int ret; - - ret = wd_get_alg_type(alg_name, alg_type); - if (ret) - return -WD_EINVAL; - - /* all ECC types use sm2 to find device. */ - if (!strcmp(alg_type, "ecc")) - strcpy(alg_type, "sm2"); - - return WD_SUCCESS; -} - /** * wd_hw_alloc_ctx() - HW driver's alloc_ctx callback. * @@ -210,8 +194,8 @@ int wd_hw_alloc_ctx(char *alg_name, void *params, handle_t *ctx) } target_numa = ctx_params->numa_id; - /* Get algorithm class type and device list */ - ret = wd_get_alg_class_type(alg_name, alg_type); + /* Get algorithm type and device list */ + ret = wd_get_alg_type(alg_name, alg_type); if (ret) { WD_ERR("invalid: alg_name is NULL!\n"); return -WD_EINVAL; diff --git a/include/wd_internal.h b/include/wd_internal.h index 95ceaa2..38fb29e 100644 --- a/include/wd_internal.h +++ b/include/wd_internal.h @@ -49,7 +49,7 @@ struct wd_ctx_config_internal { void *priv; bool epoll_en; unsigned long *msg_cnt; - char *alg_name; + const char *alg_name; struct wd_alg_driver **drv_array; __u32 drv_count; diff --git a/wd.c b/wd.c index ede0bf2..f973c70 100644 --- a/wd.c +++ b/wd.c @@ -1165,7 +1165,8 @@ int wd_get_dev_usage(struct uacce_dev *dev, const char *alg_name, __u8 alg_op_ty { char *dev_name; int ret; - if (!dev || !alg_name) { + + if (!dev || !alg_name || !strlen(dev->algs)) { WD_ERR("invalid: dev or alg name is NULL!\n"); return -WD_EINVAL; } diff --git a/wd_aead.c b/wd_aead.c index ba3c116..4e25e21 100644 --- a/wd_aead.c +++ b/wd_aead.c @@ -625,13 +625,13 @@ static int wd_aead_param_check(struct wd_aead_sess *sess, return 0; } +static bool wd_aead_atfork_registered; + static void wd_aead_clear_status(void) { wd_alg_clear_init(&wd_aead_setting.status); } -static bool wd_aead_atfork_registered; - static int wd_aead_init_nolock(struct wd_ctx_config *config, struct wd_sched *sched) { int ret; @@ -657,15 +657,8 @@ static int wd_aead_init_nolock(struct wd_ctx_config *config, struct wd_sched *sc if (ret < 0) goto out_clear_sched; - ret = wd_alg_init_driver(&wd_aead_setting.config, - wd_aead_setting.driver); - if (ret) - goto out_clear_pool; - return 0; -out_clear_pool: - wd_uninit_async_request_pool(&wd_aead_setting.pool); out_clear_sched: wd_clear_sched(&wd_aead_setting.sched); out_clear_ctx_config: @@ -674,6 +667,12 @@ out_clear_ctx_config: return ret; } +static void wd_aead_uninit_nolock(void) +{ + wd_uninit_async_request_pool(&wd_aead_setting.pool); + wd_clear_sched(&wd_aead_setting.sched); +} + int wd_aead_init(struct wd_ctx_config *config, struct wd_sched *sched) { __u32 drv_count; @@ -684,13 +683,6 @@ int wd_aead_init(struct wd_ctx_config *config, struct wd_sched *sched) wd_aead_atfork_registered = true; } - /* init1 path is HW-only; CE/SVE drivers require init2 */ - if (sched->sched_policy == SCHED_POLICY_NONE || - sched->sched_policy == SCHED_POLICY_SINGLE) { - WD_ERR("init1 does not support NONE/SINGLE schedulers, use init2\n"); - return -WD_EINVAL; - } - ret = wd_alg_try_init(&wd_aead_setting.status); if (ret) return ret; @@ -699,10 +691,22 @@ int wd_aead_init(struct wd_ctx_config *config, struct wd_sched *sched) if (ret) goto out_clear_init; + /* init1 path is HW-only; CE/SVE drivers require init2 */ + if (sched->sched_policy == SCHED_POLICY_NONE || + sched->sched_policy == SCHED_POLICY_SINGLE) { + WD_ERR("init1 does not support NONE/SINGLE schedulers, use init2\n"); + ret = -WD_EINVAL; + goto out_clear_init; + } + ret = wd_aead_open_driver(WD_TYPE_V1); if (ret) goto out_clear_init; + ret = wd_aead_init_nolock(config, sched); + if (ret) + goto out_close_driver; + /* Driver discovery */ ret = wd_get_drv_array("aead", TASK_HW, "hisi_sec2", &wd_aead_setting.config.drv_array, &drv_count); @@ -744,12 +748,6 @@ out_clear_init: return ret; } -static void wd_aead_uninit_nolock(void) -{ - wd_uninit_async_request_pool(&wd_aead_setting.pool); - wd_clear_sched(&wd_aead_setting.sched); -} - void wd_aead_uninit(void) { enum wd_status status; @@ -798,7 +796,7 @@ int wd_aead_init2_(char *alg, __u32 sched_type, int task_type, state = wd_alg_try_init(&wd_aead_setting.status); if (state) - goto out_uninit; + return state; if (!alg || sched_type >= SCHED_POLICY_BUTT || task_type < 0 || task_type >= TASK_MAX_TYPE) { @@ -895,6 +893,8 @@ void wd_aead_uninit2(void) wd_alg_uninit_driver(&wd_aead_setting.config); wd_ctx_unbind_drivers(&wd_aead_setting.config); + wd_aead_setting.config.drv_array = NULL; + wd_aead_setting.config.drv_count = 0; wd_aead_uninit_nolock(); wd_alg_attrs_uninit(&wd_aead_init_attrs); diff --git a/wd_sched.c b/wd_sched.c index f704d59..95072ea 100644 --- a/wd_sched.c +++ b/wd_sched.c @@ -2035,6 +2035,7 @@ static void wd_sched_set_param(handle_t h_sched_ctx, { struct wd_sched_params *params = (struct wd_sched_params *)sched_param; struct wd_sched_key *skey = (struct wd_sched_key *)sched_key; + struct wd_sched_ctx *sched_ctx = (struct wd_sched_ctx *)h_sched_ctx; if (unlikely(!params || !skey)) { WD_INFO("info: sched parmas or skey is NULL!\n"); @@ -2051,7 +2052,7 @@ static void wd_sched_set_param(handle_t h_sched_ctx, /* If compat info provided, fix up pre-fetched ctxs */ if (skey->alg_name && skey->ctxs) { - wd_sched_skey_compat_filter(h_sched_ctx, skey, + wd_sched_skey_compat_filter(sched_ctx, skey, &skey->sync_domain, SCHED_MODE_SYNC); wd_sched_skey_compat_filter(sched_ctx, skey, &skey->async_domain, SCHED_MODE_ASYNC); -- 2.43.0
participants (1)
-
Chenghai Huang