From: Longfang Liu <liulongfang@huawei.com> Add kernel-state reserved memory handling functionality to the uadk SVA framework to adapt to No-SVA features, including functions for applying for and initializing reserved memory pools, applying for, using, and releasing memory within the memory pool. Signed-off-by: Longfang Liu <liulongfang@huawei.com> Signed-off-by: Zongyu Wu <wuzongyu1@huawei.com> --- Makefile.am | 9 +- include/uacce.h | 7 + include/wd.h | 23 + include/wd_alg_common.h | 36 +- include/wd_bmm.h | 44 ++ include/wd_internal.h | 70 +++ include/wd_util.h | 2 + libwd.map | 14 + wd.c | 38 +- wd_bmm.c | 1057 +++++++++++++++++++++++++++++++++++++++ wd_util.c | 107 +++- 11 files changed, 1355 insertions(+), 52 deletions(-) create mode 100644 include/wd_bmm.h create mode 100644 include/wd_internal.h create mode 100644 wd_bmm.c diff --git a/Makefile.am b/Makefile.am index f897533..0e1203a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -32,12 +32,13 @@ YEAR = 2025 AM_CFLAGS+= -DUADK_VERSION_NUMBER="\"UADK version: ${MAJOR}.${MINOR}.${REVISION}\"" AM_CFLAGS+= -DUADK_RELEASED_TIME="\"Released ${MONTH} ${DAY}, ${YEAR}\"" -pkginclude_HEADERS = include/wd.h include/wd_cipher.h include/wd_aead.h \ +pkginclude_HEADERS = include/wd.h include/wd_internal.h include/wd_cipher.h include/wd_aead.h \ include/wd_comp.h include/wd_dh.h include/wd_digest.h \ include/wd_rsa.h include/uacce.h include/wd_alg_common.h \ include/wd_ecc.h include/wd_sched.h include/wd_alg.h \ include/wd_zlibwrapper.h include/wd_dae.h include/wd_agg.h \ - include/wd_udma.h include/wd_join_gather.h + include/wd_udma.h include/wd_join_gather.h \ + include/wd_bmm.h nobase_pkginclude_HEADERS = v1/wd.h v1/wd_cipher.h v1/wd_aead.h v1/uacce.h v1/wd_dh.h \ v1/wd_digest.h v1/wd_rsa.h v1/wd_bmm.h @@ -48,7 +49,7 @@ uadk_driversdir=$(libdir)/uadk uadk_drivers_LTLIBRARIES=libhisi_sec.la libhisi_hpre.la libhisi_zip.la \ libisa_ce.la libisa_sve.la libhisi_dae.la libhisi_udma.la -libwd_la_SOURCES=wd.c wd_mempool.c wd.h wd_alg.c wd_alg.h \ +libwd_la_SOURCES=wd.c wd_mempool.c wd_bmm.c wd_bmm.h wd.h wd_alg.c wd_alg.h \ v1/wd.c v1/wd.h v1/wd_adapter.c v1/wd_adapter.h \ v1/wd_rsa.c v1/wd_rsa.h \ v1/wd_aead.c v1/wd_aead.h \ @@ -126,7 +127,7 @@ libwd_comp_la_DEPENDENCIES = libwd.la libhisi_zip_la_LIBADD = -ldl -libwd_crypto_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma +libwd_crypto_la_LIBADD = -lwd -ldl -lnuma -lm -lpthread libwd_crypto_la_DEPENDENCIES = libwd.la libwd_udma_la_LIBADD = $(libwd_la_OBJECTS) -ldl -lnuma -lm -lpthread diff --git a/include/uacce.h b/include/uacce.h index f7fae27..c6bb4fb 100644 --- a/include/uacce.h +++ b/include/uacce.h @@ -15,6 +15,12 @@ extern "C" { #define UACCE_CMD_START _IO('W', 0) #define UACCE_CMD_PUT_Q _IO('W', 1) +#define UACCE_CMD_GET_SS_DMA _IOR('W', 3, unsigned long) + +/* Pass DMA SS region slice size by granularity 64KB */ +#define UACCE_GRAN_SIZE 0x10000ull +#define UACCE_GRAN_SHIFT 16 +#define UACCE_GRAN_NUM_MASK 0xfffull /** * UACCE Device flags: @@ -33,6 +39,7 @@ enum { enum uacce_qfrt { UACCE_QFRT_MMIO = 0, /* device mmio region */ UACCE_QFRT_DUS = 1, /* device user share */ + UACCE_QFRT_SS, /* static share memory */ UACCE_QFRT_MAX, }; diff --git a/include/wd.h b/include/wd.h index b62d355..b97e5c7 100644 --- a/include/wd.h +++ b/include/wd.h @@ -38,6 +38,7 @@ typedef unsigned long long __u64; /* Required compiler attributes */ #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) #define handle_t uintptr_t typedef struct wd_dev_mask wd_dev_mask_t; @@ -115,6 +116,28 @@ enum wd_alg_type { WD_AEAD, }; +/* Memory APIs for UADK API Layer */ +typedef void *(*wd_alloc)(void *usr, size_t size); +typedef void (*wd_free)(void *usr, void *va); + + /* Memory VA to DMA address map and unmap */ +typedef void *(*wd_map)(void *usr, void *va, size_t sz); +typedef void (*wd_unmap)(void *usr, void *va, void *dma, size_t sz); +typedef __u32 (*wd_bufsize)(void *usr); + +/* Memory from user, it is given at ctx creating. */ +struct wd_mm_ops { + wd_alloc alloc; /* Memory allocation */ + wd_free free; /* Memory free */ + wd_map iova_map; /* Get iova from user space VA */ + + /* Destroy the mapping between the PA of VA and iova */ + wd_unmap iova_unmap; + wd_bufsize get_bufsize; /* Optional */ + void *usr; /* Data for the above operations */ + bool sva_mode; /* Record whether the OS is SVA or No-SVA mode */ +}; + /* * If the actual size of data is inconsistent * with dsize, undefined behavior occurs. diff --git a/include/wd_alg_common.h b/include/wd_alg_common.h index fd77426..a294877 100644 --- a/include/wd_alg_common.h +++ b/include/wd_alg_common.h @@ -12,6 +12,7 @@ #include <numa.h> #include "wd.h" #include "wd_alg.h" +#include "wd_internal.h" #ifdef __cplusplus extern "C" { @@ -24,7 +25,6 @@ extern "C" { #define BITS_TO_BYTES(bits) (((bits) + 7) >> 3) #define BYTES_TO_BITS(bytes) ((bytes) << 3) -#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) #define MAX_STR_LEN 256 #define CTX_TYPE_INVALID 9999 #define POLL_TIME 1000 @@ -60,6 +60,13 @@ enum wd_init_type { WD_TYPE_V2, }; +enum wd_mem_type { + UADK_MEM_AUTO, + UADK_MEM_USER, + UADK_MEM_PROXY, + UADK_MEM_MAX, +}; + /* * struct wd_ctx - Define one ctx and related type. * @ctx: The ctx itself. @@ -132,27 +139,6 @@ struct wd_ctx_params { struct wd_cap_config *cap; }; -struct wd_soft_ctx { - void *priv; -}; - -struct wd_ctx_internal { - handle_t ctx; - __u8 op_type; - __u8 ctx_mode; - __u16 sqn; - pthread_spinlock_t lock; -}; - -struct wd_ctx_config_internal { - __u32 ctx_num; - int shmid; - struct wd_ctx_internal *ctxs; - void *priv; - bool epoll_en; - unsigned long *msg_cnt; -}; - /* * struct wd_comp_sched - Define a scheduler. * @name: Name of this scheduler. @@ -181,12 +167,6 @@ struct wd_sched { typedef int (*wd_alg_init)(struct wd_ctx_config *config, struct wd_sched *sched); typedef int (*wd_alg_poll_ctx)(__u32 idx, __u32 expt, __u32 *count); -struct wd_datalist { - void *data; - __u32 len; - struct wd_datalist *next; -}; - #ifdef __cplusplus } #endif diff --git a/include/wd_bmm.h b/include/wd_bmm.h new file mode 100644 index 0000000..76b56a0 --- /dev/null +++ b/include/wd_bmm.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* + * Copyright 2025 Huawei Technologies Co.,Ltd. All rights reserved. + */ + +#ifndef _WD_SVA_BMM_H +#define _WD_SVA_BMM_H + +#include <stdint.h> +#include "wd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Memory pool creating parameters */ +struct wd_mempool_setup { + __u32 block_size; /* Block buffer size */ + __u32 block_num; /* Block buffer number */ + __u32 align_size; /* Block buffer starting address align size */ + struct wd_mm_ops ops; /* memory from user if don't use UADK memory */ +}; + +void *wd_mempool_alloc(handle_t h_ctx, struct wd_mempool_setup *setup); +void wd_mempool_free(handle_t h_ctx, void *pool); +void *wd_mem_alloc(void *pool, size_t size); +void wd_mem_free(void *pool, void *buf); + +void *wd_mem_map(void *pool, void *buf, size_t sz); +void wd_mem_unmap(void *pool, void *buf_dma, void *buf, size_t sz); +int wd_get_free_num(void *pool, __u32 *free_num); +int wd_get_fail_num(void *pool, __u32 *fail_num); +__u32 wd_get_bufsize(void *pool); + +handle_t wd_find_ctx(const char *alg_name); +void wd_remove_ctx_list(void); +int wd_insert_ctx_list(handle_t h_ctx, char *alg_name); +__u32 wd_get_dev_id(void *pool); + +#ifdef __cplusplus +} +#endif + +#endif /* _WD_SVA_BMM_H */ diff --git a/include/wd_internal.h b/include/wd_internal.h new file mode 100644 index 0000000..cd90ebf --- /dev/null +++ b/include/wd_internal.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* + * Copyright 2025 Huawei Technologies Co.,Ltd. All rights reserved. + */ + +#ifndef WD_INTERNAL_H +#define WD_INTERNAL_H + +#include <pthread.h> +#include <stdbool.h> +#include "wd.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define DECIMAL_NUMBER 10 +#define MAX_FD_NUM 65535 + +struct wd_ctx_h { + int fd; + char dev_path[MAX_DEV_NAME_LEN]; + char *dev_name; + char *drv_name; + unsigned long qfrs_offs[UACCE_QFRT_MAX]; + void *qfrs_base[UACCE_QFRT_MAX]; + struct uacce_dev *dev; + void *priv; +}; + +struct wd_soft_ctx { + int fd; + void *priv; +}; + +struct wd_ce_ctx { + int fd; + char *drv_name; + void *priv; +}; + +struct wd_ctx_internal { + handle_t ctx; + __u8 op_type; + __u8 ctx_mode; + __u16 sqn; + pthread_spinlock_t lock; +}; + +struct wd_ctx_config_internal { + __u32 ctx_num; + int shmid; + struct wd_ctx_internal *ctxs; + void *priv; + bool epoll_en; + unsigned long *msg_cnt; + char *alg_name; +}; + +struct wd_datalist { + void *data; + __u32 len; + struct wd_datalist *next; +}; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/wd_util.h b/include/wd_util.h index 4a5204d..a337284 100644 --- a/include/wd_util.h +++ b/include/wd_util.h @@ -553,6 +553,8 @@ static inline void wd_ctx_spin_unlock(struct wd_ctx_internal *ctx, int type) pthread_spin_unlock(&ctx->lock); } +int wd_mem_ops_init(handle_t h_ctx, struct wd_mm_ops *mm_ops, int mem_type); + #ifdef __cplusplus } #endif diff --git a/libwd.map b/libwd.map index 5522ec0..b1b90b3 100644 --- a/libwd.map +++ b/libwd.map @@ -49,5 +49,19 @@ global: wd_enable_drv; wd_disable_drv; wd_get_alg_head; + + wd_find_ctx; + wd_get_dev_id; + wd_remove_ctx_list; + wd_insert_ctx_list; + wd_mempool_alloc; + wd_mempool_free; + wd_mem_alloc; + wd_mem_free; + wd_mem_map; + wd_mem_unmap; + wd_get_free_num; + wd_get_fail_num; + wd_get_bufsize; local: *; }; diff --git a/wd.c b/wd.c index c1cc282..3e867b6 100644 --- a/wd.c +++ b/wd.c @@ -20,6 +20,7 @@ #include "wd.h" #include "wd_alg.h" +#include "wd_internal.h" #define SYS_CLASS_DIR "/sys/class/uacce" #define FILE_MAX_SIZE (8 << 20) @@ -33,16 +34,18 @@ enum UADK_LOG_LEVEL { static int uadk_log_level = WD_LOG_INVALID; -struct wd_ctx_h { - int fd; - char dev_path[MAX_DEV_NAME_LEN]; - char *dev_name; - char *drv_name; - unsigned long qfrs_offs[UACCE_QFRT_MAX]; - void *qfrs_base[UACCE_QFRT_MAX]; - struct uacce_dev *dev; - void *priv; -}; +static int wd_check_ctx_type(handle_t h_ctx) +{ + struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; + + /* A simple and efficient method to check the queue type */ + if (ctx->fd < 0 || ctx->fd > MAX_FD_NUM) { + WD_INFO("Invalid: this ctx not HW ctx.\n"); + return -WD_HW_EACCESS; + } + + return 0; +} static void wd_parse_log_level(void) { @@ -446,7 +449,7 @@ void wd_release_ctx(handle_t h_ctx) { struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; - if (!ctx) + if (!ctx || wd_check_ctx_type(h_ctx)) return; close(ctx->fd); @@ -461,7 +464,7 @@ int wd_ctx_start(handle_t h_ctx) struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; int ret; - if (!ctx) + if (!ctx || wd_check_ctx_type(h_ctx)) return -WD_EINVAL; ret = wd_ctx_set_io_cmd(h_ctx, UACCE_CMD_START, NULL); @@ -527,6 +530,7 @@ void wd_ctx_unmap_qfr(handle_t h_ctx, enum uacce_qfrt qfrt) unsigned long wd_ctx_get_region_size(handle_t h_ctx, enum uacce_qfrt qfrt) { struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; + if (!ctx || qfrt >= UACCE_QFRT_MAX) return 0; return ctx->qfrs_offs[qfrt]; @@ -585,8 +589,16 @@ int wd_ctx_wait(handle_t h_ctx, __u16 ms) int wd_is_sva(handle_t h_ctx) { struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; + int ret; - if (!ctx || !ctx->dev) + if (!ctx) + return -WD_EINVAL; + + ret = wd_check_ctx_type(h_ctx); + if (ret) + return ret; + + if (!ctx->dev) return -WD_EINVAL; if ((unsigned int)ctx->dev->flags & UACCE_DEV_SVA) diff --git a/wd_bmm.c b/wd_bmm.c new file mode 100644 index 0000000..21c46ca --- /dev/null +++ b/wd_bmm.c @@ -0,0 +1,1057 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* + * Copyright 2025 Huawei Technologies Co.,Ltd. All rights reserved. + */ + +/* Block Memory Management (lib): Adapted for SVA mode */ +#define _GNU_SOURCE +#include <dirent.h> +#include <numa.h> +#include <sched.h> +#include <stdio.h> +#include <string.h> +#include <stdint.h> +#include <stdlib.h> +#include <sys/queue.h> +#include <sys/mman.h> +#include <sys/ioctl.h> +#include <unistd.h> + +#include "wd_internal.h" +#include "wd_bmm.h" +#include "uacce.h" +#include "wd.h" + +#define __ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) +#define ALIGN(x, a) __ALIGN_MASK(x, (typeof(x))(a)-1) +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#define UACCE_DEV_IOMMU (1<<7) + +#define TAG_FREE 0x12345678 /* block is free */ +#define TAG_USED 0x87654321 /* block is busy */ +#define MAX_ALIGN_SIZE 0x1000 /* 4KB */ +#define MAX_BLOCK_SIZE 0x10000000 /* 256MB */ +#define BLK_BALANCE_SZ 0x100000ul +#define NUM_TIMES(x) (87 * (x) / 100) + +#define BYTE_SIZE 8 +#define BIT_SHIFT 3 + +struct wd_ss_region { + unsigned long long pa; + void *va; + size_t size; + TAILQ_ENTRY(wd_ss_region) next; +}; +TAILQ_HEAD(wd_ss_region_list, wd_ss_region); + +struct ctx_info { + int fd; + int iommu_type; + void *ss_va; + size_t ss_mm_size; + struct wd_ss_region_list ss_list; + struct wd_ss_region_list *head; + unsigned long qfrs_offset[UACCE_QFRT_MAX]; +}; + +struct wd_blk_hd { + unsigned int blk_tag; + unsigned int blk_num; + void *blk_dma; + void *blk; +}; + +struct wd_blkpool { + pthread_spinlock_t pool_lock; + unsigned int free_blk_num; + unsigned int alloc_failures; + struct ctx_info *cinfo; + struct wd_blk_hd *blk_array; // memory blk array + unsigned int total_blocks; // total blk numbers + unsigned char *free_bitmap; // free blk bitmap, 0 mean unused + unsigned int bitmap_size; // bitmap's memory size + void *usr_mem_start; + void *act_start; + unsigned int act_hd_sz; + unsigned int act_blk_sz; + unsigned long act_mem_sz; + unsigned int dev_id; + struct wd_mempool_setup setup; +}; + +struct mem_ctx_node { + char alg_name[CRYPTO_MAX_ALG_NAME]; + handle_t h_ctx; + int numa_id; + bool used; + TAILQ_ENTRY(mem_ctx_node) list_node; +}; +static TAILQ_HEAD(, mem_ctx_node) g_mem_ctx_list = TAILQ_HEAD_INITIALIZER(g_mem_ctx_list); +static pthread_mutex_t g_mem_ctx_mutex = PTHREAD_MUTEX_INITIALIZER; + +handle_t wd_find_ctx(const char *alg_name) +{ + struct mem_ctx_node *close_node = NULL; + struct mem_ctx_node *node; + int min_distance = 0xFFFF; + int cpu = sched_getcpu(); + int nid = numa_node_of_cpu(cpu); + handle_t h_ctx = 0; + int numa_dis; + + if (!alg_name) { + WD_ERR("Invalid: alg_name is NULL!\n"); + return 0; + } + + pthread_mutex_lock(&g_mem_ctx_mutex); + TAILQ_FOREACH(node, &g_mem_ctx_list, list_node) { + if (node->used == false && strstr(node->alg_name, alg_name)) { + if (node->numa_id == nid) { + h_ctx = node->h_ctx; + node->used = true; + break; + } + + /* Query the queue with the shortest NUMA distance */ + numa_dis = numa_distance(nid, node->numa_id); + if (numa_dis < min_distance) { + min_distance = numa_dis; + close_node = node; + } + } + } + + /* If no ctx matching the NUMA ID, use the shortest distance instead ctx */ + if (!h_ctx && close_node) { + h_ctx = close_node->h_ctx; + close_node->used = true; + } + pthread_mutex_unlock(&g_mem_ctx_mutex); + + if (!h_ctx) + WD_ERR("Failed to find mem ctx for alg: %s\n", alg_name); + + return h_ctx; +} + +void wd_remove_ctx_list(void) +{ + struct mem_ctx_node *node; + + pthread_mutex_lock(&g_mem_ctx_mutex); + /* Free all list node */ + while ((node = TAILQ_FIRST(&g_mem_ctx_list)) != NULL) { + /* Use TAILQ_REMOVE to remove list node */ + TAILQ_REMOVE(&g_mem_ctx_list, node, list_node); + free(node); + } + + pthread_mutex_unlock(&g_mem_ctx_mutex); +} + +int wd_insert_ctx_list(handle_t h_ctx, char *alg_name) +{ + struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; + struct mem_ctx_node *new_node; + int numa_id; + + if (!alg_name || !h_ctx) { + WD_ERR("Invalid: input params is NULL!\n"); + return -WD_EINVAL; + } + + /* A simple and efficient method to check the queue type */ + if (ctx->fd < 0 || ctx->fd > MAX_FD_NUM) { + WD_INFO("Invalid ctx: this ctx not HW ctx.\n"); + return 0; + } + + numa_id = ctx->dev->numa_id; + new_node = malloc(sizeof(struct mem_ctx_node)); + if (new_node) { + pthread_mutex_lock(&g_mem_ctx_mutex); + strncpy(new_node->alg_name, alg_name, CRYPTO_MAX_ALG_NAME - 1); + new_node->alg_name[CRYPTO_MAX_ALG_NAME - 1] = '\0'; + new_node->numa_id = numa_id; + new_node->h_ctx = h_ctx; + new_node->used = false; + TAILQ_INSERT_TAIL(&g_mem_ctx_list, new_node, list_node); + pthread_mutex_unlock(&g_mem_ctx_mutex); + return 0; + } + + return -WD_ENOMEM; +} + +static void wd_free_slice(struct ctx_info *cinfo) +{ + struct wd_ss_region *rgn; + + while (true) { + rgn = TAILQ_FIRST(&cinfo->ss_list); + if (!rgn) + break; + TAILQ_REMOVE(&cinfo->ss_list, rgn, next); + free(rgn); + } +} + +static void wd_add_slice(struct ctx_info *cinfo, struct wd_ss_region *rgn) +{ + struct wd_ss_region *rg; + + rg = TAILQ_LAST(&cinfo->ss_list, wd_ss_region_list); + if (rg) { + if (rg->pa + rg->size == rgn->pa) { + rg->size += rgn->size; + free(rgn); + return; + } + } + + TAILQ_INSERT_TAIL(&cinfo->ss_list, rgn, next); +} + +static void wd_show_ss_slices(struct ctx_info *cinfo) +{ + struct wd_ss_region *rgn; + int i = 0; + + TAILQ_FOREACH(rgn, cinfo->head, next) { + WD_ERR("slice-%d:size = 0x%lx\n", i, rgn->size); + i++; + } +} + +static void bitmap_set_bit(unsigned char *bitmap, unsigned int bit_index) +{ + if (!bitmap) + return; + + bitmap[bit_index >> BIT_SHIFT] |= (1 << (bit_index % BYTE_SIZE)); +} + +static void bitmap_clear_bit(unsigned char *bitmap, unsigned int bit_index) +{ + if (!bitmap) + return; + + bitmap[bit_index >> BIT_SHIFT] &= ~(1 << (bit_index % BYTE_SIZE)); +} + +static bool bitmap_test_bit(const unsigned char *bitmap, unsigned int bit_index) +{ + if (!bitmap) + return false; + + /* bit is 1, it indicates that the block has already been used and is not free */ + if ((bitmap[bit_index >> BIT_SHIFT] >> (bit_index % BYTE_SIZE)) & 0x1) + return false; + + return true; +} + +static void *wd_mmap_qfr(struct ctx_info *cinfo, enum uacce_qfrt qfrt, size_t size) +{ + off_t off; + + off = qfrt * getpagesize(); + + return mmap(0, size, PROT_READ | PROT_WRITE, + MAP_SHARED, cinfo->fd, off); +} + +static void wd_unmap_reserve_mem(void *addr, size_t size) +{ + int ret; + + if (!addr) + return; + + ret = munmap(addr, size); + if (ret) + WD_ERR("wd qfr unmap failed!\n"); +} + +static void *wd_map_reserve_mem(struct wd_blkpool *pool, size_t size) +{ + struct ctx_info *cinfo = pool->cinfo; + struct wd_ss_region *rgn; + unsigned long info; + size_t tmp = size; + unsigned long i = 0; + void *ptr; + int ret = 1; + + if (!cinfo) { + WD_ERR("ctx queue information is NULL!\n"); + return NULL; + } + + /* Make sure memory map granularity size align */ + if (!cinfo->iommu_type) + tmp = ALIGN(tmp, UACCE_GRAN_SIZE); + + ptr = wd_mmap_qfr(cinfo, UACCE_QFRT_SS, tmp); + if (ptr == MAP_FAILED) { + WD_ERR("wd drv mmap fail!(err = %d)\n", errno); + return NULL; + } + + cinfo->ss_va = ptr; + cinfo->ss_mm_size = tmp; + tmp = 0; + while (ret > 0) { + info = i; + ret = ioctl(cinfo->fd, UACCE_CMD_GET_SS_DMA, &info); + if (ret < 0) { + wd_show_ss_slices(cinfo); + WD_ERR("get DMA fail!\n"); + goto err_out; + } + + rgn = malloc(sizeof(*rgn)); + if (!rgn) { + WD_ERR("alloc ss region fail!\n"); + goto err_out; + } + memset(rgn, 0, sizeof(*rgn)); + + if (cinfo->iommu_type) + rgn->size = cinfo->ss_mm_size; + else + rgn->size = (info & UACCE_GRAN_NUM_MASK) << + UACCE_GRAN_SHIFT; + rgn->pa = info & (~UACCE_GRAN_NUM_MASK); + rgn->va = ptr + tmp; + tmp += rgn->size; + wd_add_slice(cinfo, rgn); + + i++; + } + + return ptr; + +err_out: + wd_free_slice(cinfo); + wd_unmap_reserve_mem(cinfo->ss_va, cinfo->ss_mm_size); + + return NULL; +} + +static int wd_pool_params_check(struct wd_mempool_setup *setup) +{ + if (!setup->block_num || !setup->block_size || + setup->block_size > MAX_BLOCK_SIZE) { + WD_ERR("Invalid: block_size or block_num(%x, %u)!\n", + setup->block_size, setup->block_num); + return -WD_EINVAL; + } + + /* Check parameters, and align_size must be 2^N */ + if (setup->align_size <= 0x1 || setup->align_size > MAX_ALIGN_SIZE || + (setup->align_size & (setup->align_size - 0x1))) { + WD_ERR("Invalid align_size.\n"); + return -WD_EINVAL; + } + + return WD_SUCCESS; +} + +static int wd_ctx_info_init(struct wd_ctx_h *ctx, struct wd_blkpool *p) +{ + struct ctx_info *cinfo; + + cinfo = calloc(1, sizeof(struct ctx_info)); + if (!cinfo) { + WD_ERR("failed to alloc ctx info memory.\n"); + return -WD_ENOMEM; + } + + cinfo->fd = ctx->fd; + cinfo->iommu_type = (unsigned int)ctx->dev->flags & UACCE_DEV_IOMMU; + cinfo->head = &cinfo->ss_list; + TAILQ_INIT(&cinfo->ss_list); + (void)memcpy(cinfo->qfrs_offset, ctx->qfrs_offs, + sizeof(cinfo->qfrs_offset)); + p->cinfo = (void *)cinfo; + + return 0; +} + +static int wd_pool_pre_layout(handle_t h_ctx, + struct wd_blkpool *p, + struct wd_mempool_setup *sp) +{ + struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; + struct ctx_info *cinfo = NULL; + unsigned int asz; + int ret; + + if (!ctx && !sp->ops.alloc) { + WD_ERR("ctx is NULL!\n"); + return -WD_EINVAL; + } + + if (!sp->ops.alloc) { + ret = wd_ctx_info_init(ctx, p); + if (ret) { + WD_ERR("failed to init ctx info.\n"); + return ret; + } + cinfo = p->cinfo; + } + + ret = wd_pool_params_check(sp); + if (ret) { + free(p->cinfo); + p->cinfo = NULL; + return ret; + } + + asz = sp->align_size; + + /* Get actual value by align */ + p->act_hd_sz = ALIGN(sizeof(struct wd_blk_hd), asz); + p->act_blk_sz = ALIGN(sp->block_size, asz); + p->act_mem_sz = (p->act_hd_sz + p->act_blk_sz) * + (unsigned long)sp->block_num + asz; + + /* + * When we use WD reserve memory and the blk_sz is larger than 1M, + * in order to ensure the mem_pool to be success, + * ensure that the allocated memory is an integer multiple of 1M. + */ + if (!sp->ops.alloc && (cinfo && !cinfo->iommu_type)) + p->act_mem_sz = ((p->act_mem_sz + BLK_BALANCE_SZ - 1) & ~(BLK_BALANCE_SZ - 1)) << 1; + + return WD_SUCCESS; +} + +/** + * wd_iova_map - Map virtual address to physical address + * @cinfo: context information + * @va: virtual address to map + * @sz: size of the mapping (not used in current implementation) + * + * When IOMMU is enabled, the PA is actually an IOVA; userspace still sees it + * as consistent and contiguous with the VA. + * When IOMMU is disabled, the PA refers to the kernel's physical address, which + * must be physically contiguous to be allocated by the kernel. + * Therefore, the PA address can be obtained from the offset of the VA. + * + */ +static void *wd_iova_map(struct ctx_info *cinfo, void *va, size_t sz) +{ + struct wd_ss_region *rgn; + unsigned long offset; + void *dma_addr; + + if (!cinfo || !va) { + WD_ERR("wd iova map: parameter err!\n"); + return NULL; + } + + /* Search through all memory regions to find where va belongs */ + TAILQ_FOREACH(rgn, cinfo->head, next) { + if (rgn->va <= va && va < rgn->va + rgn->size) { + /* Calculate offset within the region */ + offset = (uintptr_t)va - (uintptr_t)rgn->va; + /* Add base physical address of the region */ + dma_addr = (void *)((uintptr_t)rgn->pa + offset); + return dma_addr; + } + } + + WD_ERR("wd iova map: va not found in any region\n"); + return NULL; +} + +/** + * wd_iova_unmap - Unmap physical address (no-op in non-IOMMU mode) + * @cinfo: context information + * @va: virtual address + * @dma: physical address + * @sz: size of the mapping (not used in current implementation) + * + * In non-IOMMU mode, this function does nothing as there's no need to unmap. + * In IOMMU mode, this would typically involve unmapping the DMA address. + */ +static void wd_iova_unmap(struct ctx_info *cinfo, void *va, void *dma, size_t sz) +{ + /* For no-iommu, dma-unmap doing nothing */ +} + +static void wd_pool_uninit(struct wd_blkpool *p) +{ + struct ctx_info *cinfo = p->cinfo; + struct wd_blk_hd *fhd = NULL; + unsigned long block_size; + unsigned int i; + + block_size = (unsigned long)p->act_hd_sz + p->act_blk_sz; + /* Clean up the allocated resources. */ + for (i = 0; i < p->total_blocks; i++) { + /* Release the previously allocated blocks. */ + fhd = &p->blk_array[i]; + wd_iova_unmap(cinfo, fhd->blk, fhd->blk_dma, block_size); + } + + free(p->free_bitmap); + p->free_bitmap = NULL; + free(p->blk_array); + p->blk_array = NULL; +} + +static int wd_pool_init(struct wd_blkpool *p) +{ + struct ctx_info *cinfo = p->cinfo; + __u32 blk_size = p->setup.block_size; + void *dma_start, *dma_end, *va; + struct wd_blk_hd *fhd = NULL; + struct wd_blk_hd *hd = NULL; + unsigned int i, j, act_num; + unsigned long block_size; + unsigned int dma_num = 0; + + p->act_start = (void *)ALIGN((uintptr_t)p->usr_mem_start, + p->setup.align_size); + + /* Calculate the actual number of allocatable blocks */ + block_size = (unsigned long)(p->act_hd_sz + p->act_blk_sz); + if (block_size == 0) { + WD_ERR("Invalid block size with header.\n"); + return -WD_EINVAL; + } + act_num = p->act_mem_sz / block_size; + if (!act_num) { + WD_ERR("Invalid memory size.\n"); + return -WD_EINVAL; + } + + /* Allocate block array */ + p->blk_array = (struct wd_blk_hd *)malloc(act_num * sizeof(struct wd_blk_hd)); + if (!p->blk_array) { + WD_ERR("Failed to allocate block array.\n"); + return -WD_ENOMEM; + } + + /* Allocate bitmap */ + p->total_blocks = act_num; + p->bitmap_size = (act_num + BYTE_SIZE - 1) >> BIT_SHIFT; + p->free_bitmap = (unsigned char *)calloc(1, p->bitmap_size); + if (!p->free_bitmap) { + WD_ERR("Failed to allocate free bitmap.\n"); + goto bitmap_error; + } + + /* Initialize all blocks. */ + for (i = 0; i < act_num; i++) { + /* Calculate the virtual address of the current block. */ + va = (void *)((uintptr_t)p->act_start + block_size * i); + + /* Get the physical address. */ + dma_start = wd_iova_map(cinfo, va, 0); + dma_end = wd_iova_map(cinfo, va + blk_size - 1, 0); + if (!dma_start || !dma_end) { + WD_ERR("wd_iova_map err.\n"); + /* Clean up the allocated resources. */ + goto init_blk_error; + } + + /* Check whether the physical addresses are contiguous. */ + if ((uintptr_t)dma_end - (uintptr_t)dma_start != blk_size - 1) { + /* If OS kernel is not open SMMU, need to check dma address */ + WD_INFO("wd dma address not continuous.\n"); + /* Mark as unavailable, bit value is 1. */ + bitmap_set_bit(p->free_bitmap, i); + continue; + } + + /* Initialize the block. */ + hd = &p->blk_array[i]; + hd->blk_dma = dma_start; + hd->blk = va; + hd->blk_tag = TAG_FREE; + hd->blk_num = 0; + + dma_num++; + } + + /* + * if dma_num <= (1 / 1.15) * user's block_num, we think the pool + * is created with failure. + */ + if (dma_num <= NUM_TIMES(p->setup.block_num)) { + WD_ERR("dma_num = %u, not enough.\n", dma_num); + goto init_blk_error; + } + + p->free_blk_num = dma_num; + p->setup.block_num = dma_num; + + return WD_SUCCESS; + +init_blk_error: + /* Clean up the allocated resources. */ + for (j = 0; j < i; j++) { + /* Release the previously allocated blocks. */ + fhd = &p->blk_array[j]; + wd_iova_unmap(cinfo, fhd->blk, fhd->blk_dma, block_size); + } + free(p->free_bitmap); + +bitmap_error: + free(p->blk_array); + + return -WD_ENOMEM; +} + +static int usr_pool_init(struct wd_blkpool *p) +{ + struct wd_mempool_setup *sp = &p->setup; + __u32 blk_size = sp->block_size; + struct wd_blk_hd *hd = NULL; + __u32 i; + + p->act_start = (void *)ALIGN((uintptr_t)p->usr_mem_start, + sp->align_size); + for (i = 0; i < sp->block_num; i++) { + hd = (void *)((uintptr_t)p->act_start + (p->act_hd_sz + p->act_blk_sz) * i); + hd->blk = (void *)((uintptr_t)hd + p->act_hd_sz); + hd->blk_dma = sp->ops.iova_map(sp->ops.usr, hd->blk, blk_size); + if (!hd->blk_dma) { + WD_ERR("failed to map usr blk.\n"); + return -WD_ENOMEM; + } + hd->blk_tag = TAG_FREE; + } + + p->free_blk_num = sp->block_num; + + return WD_SUCCESS; +} + +static int wd_parse_dev_id(char *dev_name) +{ + char *last_dash = NULL; + char *endptr; + int dev_id; + + if (!dev_name) + return -WD_EINVAL; + + /* Find the last '-' in the string. */ + last_dash = strrchr(dev_name, '-'); + if (!last_dash || *(last_dash + 1) == '\0') + return -WD_EINVAL; + + /* Parse the following number */ + dev_id = strtol(last_dash + 1, &endptr, DECIMAL_NUMBER); + /* Check whether it is truly all digits */ + if (*endptr != '\0' || dev_id < 0) + return -WD_EINVAL; + + return dev_id; +} + +static int wd_mempool_init(handle_t h_ctx, struct wd_blkpool *pool, + struct wd_mempool_setup *setup) +{ + struct wd_ctx_h *ctx = (struct wd_ctx_h *)h_ctx; + struct ctx_info *cinfo = pool->cinfo; + void *addr = NULL; + int ret; + + /* Use user's memory, and its ops alloc function */ + if (setup->ops.alloc && setup->ops.free && setup->ops.iova_map) { + addr = setup->ops.alloc(setup->ops.usr, pool->act_mem_sz); + if (!addr) { + WD_ERR("failed to allocate memory in user pool.\n"); + return -WD_EINVAL; + } + + pool->usr_mem_start = addr; + if (usr_pool_init(pool)) { + WD_ERR("failed to initialize user pool.\n"); + setup->ops.free(setup->ops.usr, addr); + return -WD_EINVAL; + } + } else { + /* Use wd to reserve memory */ + addr = wd_map_reserve_mem(pool, pool->act_mem_sz); + if (!addr) { + WD_ERR("wd pool failed to reserve memory.\n"); + return -WD_ENOMEM; + } + + pool->usr_mem_start = addr; + if (wd_pool_init(pool)) { + WD_ERR("failed to initialize wd pool.\n"); + goto err_out; + } + setup->block_num = pool->setup.block_num; + } + + ret = wd_parse_dev_id(ctx->dev_path); + if (ret < 0) { + wd_pool_uninit(pool); + goto err_out; + } + pool->dev_id = ret; + + return WD_SUCCESS; + +err_out: + if (pool->cinfo) { + wd_free_slice(cinfo); + wd_unmap_reserve_mem(cinfo->ss_va, cinfo->ss_mm_size); + pool->cinfo = NULL; + } + return -WD_EINVAL; +} + +void *wd_mempool_alloc(handle_t h_ctx, struct wd_mempool_setup *setup) +{ + struct wd_blkpool *pool = NULL; + int ret; + + if (!setup || !h_ctx) { + WD_ERR("Input param is NULL!\n"); + return NULL; + } + + ret = wd_is_sva(h_ctx); + if (ret < 0) { + WD_ERR("failed to check device ctx!\n"); + return NULL; + } else if (ret == UACCE_DEV_SVA) { + WD_ERR("the device is SVA mode!\n"); + return NULL; + } + + pool = calloc(1, sizeof(*pool)); + if (!pool) { + WD_ERR("failed to malloc pool.\n"); + return NULL; + } + ret = pthread_spin_init(&pool->pool_lock, PTHREAD_PROCESS_PRIVATE); + if (ret) + goto err_pool_alloc; + + memcpy(&pool->setup, setup, sizeof(pool->setup)); + + ret = wd_pool_pre_layout(h_ctx, pool, setup); + if (ret) + goto err_pool_layout; + + ret = wd_mempool_init(h_ctx, pool, setup); + if (ret) + goto err_pool_init; + + return pool; + +err_pool_init: + if (pool->cinfo) { + free(pool->cinfo); + pool->cinfo = NULL; + } +err_pool_layout: + pthread_spin_destroy(&pool->pool_lock); +err_pool_alloc: + free(pool); + + return NULL; +} + +void wd_mempool_free(handle_t h_ctx, void *pool) +{ + struct wd_mempool_setup *setup; + struct wd_blkpool *p = pool; + + if (!p || !h_ctx) { + WD_ERR("pool destroy err, pool or ctx is NULL.\n"); + return; + } + + setup = &p->setup; + if (p->free_blk_num != setup->block_num) { + WD_ERR("Can not destroy blk pool, as it's in use.\n"); + return; + } + + if (setup->ops.free) + setup->ops.free(setup->ops.usr, p->usr_mem_start); + + if (p->cinfo) { + /* Free block array memory */ + if (p->blk_array) + free(p->blk_array); + + if (p->free_bitmap) + free(p->free_bitmap); + + wd_free_slice(p->cinfo); + wd_unmap_reserve_mem(p->cinfo->ss_va, p->cinfo->ss_mm_size); + free(p->cinfo); + p->cinfo = NULL; + } + + pthread_spin_destroy(&p->pool_lock); + free(p); +} + +void wd_mem_free(void *pool, void *buf) +{ + struct wd_blkpool *p = pool; + struct wd_blk_hd *current_hd; + struct wd_blk_hd *hd; + unsigned int current_idx; + unsigned int blk_idx; + unsigned long offset; + unsigned int i, num; + unsigned long sz; + + if (unlikely(!p || !buf)) { + WD_ERR("free blk parameters err!\n"); + return; + } + + sz = p->act_hd_sz + p->act_blk_sz; + if (!sz) { + WD_ERR("memory pool blk size is zero!\n"); + return; + } + + if ((uintptr_t)buf < (uintptr_t)p->act_start) { + WD_ERR("free block addr is error.\n"); + return; + } + + /* Calculate the block index. */ + offset = (unsigned long)((uintptr_t)buf - (uintptr_t)p->act_start); + blk_idx = offset / sz; + + /* Check if the index is valid. */ + if (blk_idx >= p->total_blocks) { + WD_ERR("Invalid block index<%u>.\n", blk_idx); + return; + } + + /* Get the block header. */ + hd = &p->blk_array[blk_idx]; + num = hd->blk_num; + + pthread_spin_lock(&p->pool_lock); + /* Release all related blocks. */ + for (i = 0; i < num; i++) { + // Recalculate the index (since it is contiguous). + current_idx = blk_idx + i; + current_hd = &p->blk_array[current_idx]; + current_hd->blk_tag = TAG_FREE; + current_hd->blk_num = 0; + bitmap_clear_bit(p->free_bitmap, current_idx); + } + p->free_blk_num += num; + pthread_spin_unlock(&p->pool_lock); +} + +static int wd_find_contiguous_blocks(struct wd_blkpool *p, + unsigned int required_blocks, + unsigned int *start_block) +{ +#define MAX_SKIP_ATTEMPTS 10 + unsigned int consecutive_count = 0; + unsigned int skip_attempts = 0; + struct wd_blk_hd *hd, *tl; + unsigned int i; + + if (required_blocks == 0 || required_blocks > p->total_blocks) + return -WD_EINVAL; + + for (i = 0; i < p->total_blocks; i++) { + if (!bitmap_test_bit(p->free_bitmap, i)) { + consecutive_count = 0; + continue; + } + + if (consecutive_count == 0) + *start_block = i; + consecutive_count++; + + if (consecutive_count < required_blocks) + continue; + + /* Check DMA contiguity only if more than one block is needed */ + if (required_blocks > 1) { + hd = &p->blk_array[*start_block]; + tl = &p->blk_array[*start_block + required_blocks - 1]; + + if (((uintptr_t)tl->blk_dma - (uintptr_t)hd->blk_dma) != + ((uintptr_t)tl->blk - (uintptr_t)hd->blk)) { + /* Not contiguous, skip this start and try again */ + if (++skip_attempts > MAX_SKIP_ATTEMPTS) + return -WD_ENOMEM; + + i = *start_block; // will be incremented by loop + consecutive_count = 0; + continue; + } + } + + /* Found and DMA is contiguous */ + return WD_SUCCESS; + } + + return -WD_ENOMEM; +} + +void *wd_mem_alloc(void *pool, size_t size) +{ + unsigned int required_blocks; + unsigned int start_block = 0; + struct wd_blk_hd *hd = NULL; + struct wd_blkpool *p = pool; + unsigned int j; + int ret; + + if (unlikely(!p || !size)) { + WD_ERR("blk alloc pool is null!\n"); + return NULL; + } + + if (!p->act_blk_sz) { + WD_ERR("blk pool is error!\n"); + return NULL; + } + + /* Calculate the number of blocks required. */ + required_blocks = (size + p->act_blk_sz - 1) / p->act_blk_sz; + if (required_blocks > p->free_blk_num) { + p->alloc_failures++; + WD_ERR("Not enough free blocks.\n"); + return NULL; + } + + pthread_spin_lock(&p->pool_lock); + /* Find contiguous free blocks. */ + ret = wd_find_contiguous_blocks(p, required_blocks, &start_block); + if (ret != 0) { + p->alloc_failures++; + pthread_spin_unlock(&p->pool_lock); + WD_ERR("Failed to find contiguous blocks.\n"); + return NULL; + } + + /* Mark all required blocks as used */ + for (j = start_block; j < start_block + required_blocks; j++) { + p->blk_array[j].blk_tag = TAG_USED; + bitmap_set_bit(p->free_bitmap, j); + } + + p->free_blk_num -= required_blocks; + hd = &p->blk_array[start_block]; + hd->blk_num = required_blocks; + pthread_spin_unlock(&p->pool_lock); + + return hd->blk; +} + +void *wd_mem_map(void *pool, void *buf, size_t sz) +{ + struct wd_blkpool *p = pool; + struct wd_blk_hd *hd; + unsigned long offset; + unsigned long blk_sz; + unsigned long blk_idx; + + if (unlikely(!pool || !buf)) { + WD_ERR("blk map err, pool is NULL!\n"); + return NULL; + } + + if (!sz || (uintptr_t)buf < (uintptr_t)p->act_start) { + WD_ERR("map buf addr is error.\n"); + return NULL; + } + /* Calculate the block index. */ + offset = (unsigned long)((uintptr_t)buf - (uintptr_t)p->act_start); + blk_sz = p->act_hd_sz + p->act_blk_sz; + blk_idx = offset / blk_sz; + + /* Check if the index is valid. */ + if (blk_idx >= p->total_blocks) { + WD_ERR("Invalid block index<%lu> in map.\n", blk_idx); + return NULL; + } + + hd = &p->blk_array[blk_idx]; + if (unlikely(hd->blk_tag != TAG_USED || + (uintptr_t)buf < (uintptr_t)hd->blk)) { + WD_ERR("dma map fail!\n"); + return NULL; + } + + return (void *)((uintptr_t)hd->blk_dma + ((uintptr_t)buf - + (uintptr_t)hd->blk)); +} + +void wd_mem_unmap(void *pool, void *buf_dma, void *buf, size_t sz) +{ + /* do nothing at no-iommu mode */ +} + +int wd_get_free_num(void *pool, __u32 *free_num) +{ + struct wd_blkpool *p = pool; + + if (!p || !free_num) { + WD_ERR("get_free_blk_num err, parameter err!\n"); + return -WD_EINVAL; + } + + *free_num = __atomic_load_n(&p->free_blk_num, __ATOMIC_RELAXED); + + return WD_SUCCESS; +} + +int wd_get_fail_num(void *pool, __u32 *fail_num) +{ + struct wd_blkpool *p = pool; + + if (!p || !fail_num) { + WD_ERR("get_blk_alloc_failure err, pool is NULL!\n"); + return -WD_EINVAL; + } + + *fail_num = __atomic_load_n(&p->alloc_failures, __ATOMIC_RELAXED); + + return WD_SUCCESS; +} + +__u32 wd_get_bufsize(void *pool) +{ + struct wd_blkpool *p = pool; + + if (!p) { + WD_ERR("get dev id is null!\n"); + return 0; + } + + return p->act_blk_sz; +} + +__u32 wd_get_dev_id(void *pool) +{ + struct wd_blkpool *p = pool; + + if (!p) { + WD_ERR("failed to get dev id!\n"); + return 0; + } + + return p->dev_id; +} + diff --git a/wd_util.c b/wd_util.c index e8a2934..d0d83eb 100644 --- a/wd_util.c +++ b/wd_util.c @@ -13,6 +13,8 @@ #include <ctype.h> #include "wd_sched.h" #include "wd_util.h" +#include "wd_alg.h" +#include "wd_bmm.h" #define WD_ASYNC_DEF_POLL_NUM 1 #define WD_ASYNC_DEF_QUEUE_DEPTH 1024 @@ -100,11 +102,6 @@ struct acc_alg_item { const char *algtype; }; -struct wd_ce_ctx { - char *drv_name; - void *priv; -}; - static struct acc_alg_item alg_options[] = { {"zlib", "zlib"}, {"gzip", "gzip"}, @@ -172,6 +169,93 @@ static struct acc_alg_item alg_options[] = { {"", ""} }; +static void *wd_internal_alloc(void *usr, size_t size) +{ + if (size != 0) + return malloc(size); + else + return NULL; +} + +static void wd_internal_free(void *usr, void *va) +{ + if (va != NULL) + free(va); +} + +static __u32 wd_mem_bufsize(void *usr) +{ + /* Malloc memory min size is 1 Byte */ + return 1; +} + +int wd_mem_ops_init(handle_t h_ctx, struct wd_mm_ops *mm_ops, int mem_type) +{ + int ret; + + ret = wd_is_sva(h_ctx); + if (ret == UACCE_DEV_SVA || ret == -WD_HW_EACCESS) { + /* + * In software queue scenario, all memory is handled as virtual memory + * and processed in the same way as SVA mode + */ + mm_ops->sva_mode = true; + } else if (!ret) { + mm_ops->sva_mode = false; + } else { + WD_ERR("failed to check ctx!\n"); + return ret; + } + + /* + * Under SVA mode, there is no need to consider the memory type; + * directly proceed with virtual memory handling + */ + if (mm_ops->sva_mode) { + mm_ops->alloc = (void *)wd_internal_alloc; + mm_ops->free = (void *)wd_internal_free; + mm_ops->iova_map = NULL; + mm_ops->iova_unmap = NULL; + mm_ops->get_bufsize = (void *)wd_mem_bufsize; + mm_ops->usr = NULL; + return 0; + } + + switch (mem_type) { + case UADK_MEM_AUTO: + /* + * The memory pool needs to be allocated according to + * the block size when it is first executed in the UADK + */ + mm_ops->usr = NULL; + WD_ERR("automatic under No-SVA mode is not supported!\n"); + return -WD_EINVAL; + case UADK_MEM_USER: + if (!mm_ops->alloc || !mm_ops->free || !mm_ops->iova_map || + !mm_ops->iova_unmap || !mm_ops->usr) { // The user create a memory pool + WD_ERR("failed to check memory ops, some ops function is NULL!\n"); + return -WD_EINVAL; + } + break; + case UADK_MEM_PROXY: + if (!mm_ops->usr) { + WD_ERR("failed to check memory pool!\n"); + return -WD_EINVAL; + } + mm_ops->alloc = (void *)wd_mem_alloc; + mm_ops->free = (void *)wd_mem_free; + mm_ops->iova_map = (void *)wd_mem_map; + mm_ops->iova_unmap = (void *)wd_mem_unmap; + mm_ops->get_bufsize = (void *)wd_get_bufsize; + break; + default: + WD_ERR("failed to check memory type!\n"); + return -WD_EINVAL; + } + + return 0; +} + static void clone_ctx_to_internal(struct wd_ctx *ctx, struct wd_ctx_internal *ctx_in) { @@ -257,6 +341,12 @@ int wd_init_ctx_config(struct wd_ctx_config_internal *in, WD_ERR("failed to init ctxs lock!\n"); goto err_out; } + + ret = wd_insert_ctx_list(cfg->ctxs[i].ctx, in->alg_name); + if (ret) { + WD_ERR("failed to add ctx to mem list!\n"); + goto err_out; + } } in->ctxs = ctxs; @@ -318,6 +408,7 @@ void wd_clear_ctx_config(struct wd_ctx_config_internal *in) in->ctxs = NULL; } + wd_remove_ctx_list(); wd_shm_delete(in); } @@ -2485,7 +2576,7 @@ static int wd_init_ctx_set(struct wd_init_attrs *attrs, struct uacce_dev_list *l /* If the ctx set number is 0, the initialization is skipped. */ if (!ctx_set_num) - return 0; + return -WD_ENOPROC; dev = wd_find_dev_by_numa(list, numa_id); if (WD_IS_ERR(dev)) @@ -2573,7 +2664,9 @@ static int wd_init_ctx_and_sched(struct wd_init_attrs *attrs, struct bitmask *bm for (j = 0; j < op_type_num; j++) { ctx_nums = ctx_params->ctx_set_num[j]; ret = wd_init_ctx_set(attrs, list, idx, i, j); - if (ret) + if (ret == -WD_ENOPROC) + continue; + else if (ret) goto free_ctxs; ret = wd_instance_sched_set(attrs->sched, ctx_nums, idx, i, j); if (ret) -- 2.33.0