- Acc - mailweb.openeuler.org

[PATCH 1/2] uadk: fix spin lock destroy
by Weili Qian 18 Jul '22

18 Jul '22

Fix missing destroyed lock when initialization fails. Signed-off-by: Wenkai Lin <linwenkai6(a)hisilicon.com> Signed-off-by: Weili Qian <qianweili(a)huawei.com> --- drv/hisi_qm_udrv.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drv/hisi_qm_udrv.c b/drv/hisi_qm_udrv.c index 9eaf60c..b4e763d 100644 --- a/drv/hisi_qm_udrv.c +++ b/drv/hisi_qm_udrv.c @@ -335,6 +335,14 @@ err_out: return ret; } +static void hisi_qm_clear_info(struct hisi_qp *qp) +{ + struct hisi_qm_queue_info *q_info = &qp->q_info; + + pthread_spin_destroy(&q_info->lock); + hisi_qm_unset_region(qp->h_ctx, q_info); +} + static int get_free_num(struct hisi_qm_queue_info *q_info) { /* The device should reserve one buffer. */ @@ -374,7 +382,7 @@ handle_t hisi_qm_alloc_qp(struct hisi_qm_priv *config, handle_t ctx) qp->h_sgl_pool = hisi_qm_create_sglpool(HISI_SGL_NUM_IN_BD, HISI_SGE_NUM_IN_SGL); if (!qp->h_sgl_pool) - goto out_qp; + goto free_info; ret = wd_ctx_start(qp->h_ctx); if (ret) @@ -390,6 +398,8 @@ handle_t hisi_qm_alloc_qp(struct hisi_qm_priv *config, handle_t ctx) free_pool: hisi_qm_destroy_sglpool(qp->h_sgl_pool); +free_info: + hisi_qm_clear_info(qp); out_qp: free(qp); out: -- 2.30.0

1 1

[PATCH] engine: add receiving timeout count
by Kai Ye 18 Jul '22

18 Jul '22

Increase the count of packet receiving timeout as doing async jobs. Prevents falling into an infinite loop. Signed-off-by: Kai Ye <yekai13(a)huawei.com> --- src/uadk.h | 5 +++-- src/uadk_cipher.c | 7 +++++-- src/uadk_dh.c | 7 +++++-- src/uadk_digest.c | 7 +++++-- src/uadk_pkey.c | 7 +++++-- src/uadk_rsa.c | 7 +++++-- 6 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/uadk.h b/src/uadk.h index d210c3d..267da89 100644 --- a/src/uadk.h +++ b/src/uadk.h @@ -21,8 +21,9 @@ #include <uadk/wd_sched.h> #include "uadk_utils.h" -#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) -#define ENV_STRING_LEN 256 +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#define ENV_STRING_LEN 256 +#define ENGINE_RECV_MAX_CNT 60000000 enum { KUNPENG920, diff --git a/src/uadk_cipher.c b/src/uadk_cipher.c index 0b99753..5993386 100644 --- a/src/uadk_cipher.c +++ b/src/uadk_cipher.c @@ -503,6 +503,7 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, static int uadk_e_cipher_poll(void *ctx) { struct cipher_priv_ctx *priv = (struct cipher_priv_ctx *) ctx; + __u64 rx_cnt = 0; __u32 recv = 0; /* Poll one packet currently */ int expt = 1; @@ -519,9 +520,11 @@ static int uadk_e_cipher_poll(void *ctx) return 0; else if (ret < 0 && ret != -EAGAIN) return ret; - } while (ret == -EAGAIN); + } while (ret == -EAGAIN && (rx_cnt++ < ENGINE_RECV_MAX_CNT)); - return ret; + fprintf(stderr, "failed to recv msg: timeout!\n"); + + return -ETIMEDOUT; } static int uadk_e_cipher_env_poll(void *ctx) diff --git a/src/uadk_dh.c b/src/uadk_dh.c index 75c9e5a..018a3b6 100644 --- a/src/uadk_dh.c +++ b/src/uadk_dh.c @@ -204,6 +204,7 @@ static __u32 dh_pick_next_ctx(handle_t sched_ctx, static int uadk_e_dh_poll(void *ctx) { + __u64 rx_cnt = 0; __u32 recv = 0; int expect = 1; int idx = 1; @@ -215,9 +216,11 @@ static int uadk_e_dh_poll(void *ctx) return UADK_E_POLL_SUCCESS; else if (ret < 0 && ret != -EAGAIN) return ret; - } while (ret == -EAGAIN); + } while (ret == -EAGAIN && (rx_cnt++ < ENGINE_RECV_MAX_CNT)); - return ret; + fprintf(stderr, "failed to recv msg: timeout!\n"); + + return -ETIMEDOUT; } static void uadk_e_dh_cb(void *req_t) diff --git a/src/uadk_digest.c b/src/uadk_digest.c index 70f51d3..0fb2cfb 100644 --- a/src/uadk_digest.c +++ b/src/uadk_digest.c @@ -331,6 +331,7 @@ static int sched_single_poll_policy(handle_t h_sched_ctx, static int uadk_e_digest_poll(void *ctx) { + __u64 rx_cnt = 0; __u32 recv = 0; int expt = 1; int ret = 0; @@ -341,9 +342,11 @@ static int uadk_e_digest_poll(void *ctx) return 0; else if (ret < 0 && ret != -EAGAIN) return ret; - } while (ret == -EAGAIN); + } while (ret == -EAGAIN && (rx_cnt++ < ENGINE_RECV_MAX_CNT)); - return ret; + fprintf(stderr, "failed to recv msg: timeout!\n"); + + return -ETIMEDOUT; } static int uadk_e_digest_env_poll(void *ctx) diff --git a/src/uadk_pkey.c b/src/uadk_pkey.c index 5300327..019d4fd 100644 --- a/src/uadk_pkey.c +++ b/src/uadk_pkey.c @@ -113,6 +113,7 @@ void uadk_ecc_cb(void *req_t) static int uadk_ecc_poll(void *ctx) { unsigned int recv = 0; + __u64 rx_cnt = 0; int expt = 1; int ret; @@ -122,9 +123,11 @@ static int uadk_ecc_poll(void *ctx) return 0; else if (ret < 0 && ret != -EAGAIN) return ret; - } while (ret == -EAGAIN); + } while (ret == -EAGAIN && (rx_cnt++ < ENGINE_RECV_MAX_CNT)); - return ret; + fprintf(stderr, "failed to recv msg: timeout!\n"); + + return -ETIMEDOUT; } /* make resource configure static */ diff --git a/src/uadk_rsa.c b/src/uadk_rsa.c index 1579539..42c3506 100644 --- a/src/uadk_rsa.c +++ b/src/uadk_rsa.c @@ -658,6 +658,7 @@ static int rsa_poll_policy(handle_t h_sched_ctx, __u32 expect, __u32 *count) static int uadk_e_rsa_poll(void *ctx) { + __u64 rx_cnt = 0; __u32 recv = 0; int expt = 1; int ret; @@ -668,9 +669,11 @@ static int uadk_e_rsa_poll(void *ctx) return UADK_E_POLL_SUCCESS; else if (ret < 0 && ret != -EAGAIN) return ret; - } while (ret == -EAGAIN); + } while (ret == -EAGAIN && (rx_cnt++ < ENGINE_RECV_MAX_CNT)); - return ret; + fprintf(stderr, "failed to recv msg: timeout!\n"); + + return -ETIMEDOUT; } static struct rsa_res_config rsa_res_config = { -- 2.30.0

1 0

[PATCH 0/5] uadk - add wd_comp_init2
by Yang Shen 07 Jul '22

07 Jul '22

The current uadk initialization process is: 1.Call wd_request_ctx() to request ctxs from devices. 2.Call wd_sched_rr_alloc() to create a sched(or some other scheduler alloc function if exits). 3.Initialize the sched. 4.Call wd_<alg>_init() with ctx_config and sched. Logic is reasonable. But in practice, the step of `wd_ request_ Ctx() ` and `wd_ sched_ rr_alloc() ` are very tedious. This makes it difficult for users to use the interface. One of the main reasons for this is that uadk has made a lot of configurations in the scheduler in order to provide users with better performance. Based on this consideration, the current uadk requires the user to arrange the division of hardware resources according to the device topology during initialization. Therefore, as a high-level interface, this scheme can provide customized scheme configuration for users with deep needs. All algorithm initialization interfaces have the same input parameters and behavioral logic. The pre-processing of the wd_<alg>_init is actually the configuration of `struct wd_ctx_config` and `struct wd_sched`. Therefore, the next thing to be done is to use limited and easy-to-use input parameters to describe users' requirements on the two input parameters, ensuring that the functions of the new interface init2 are the same as those of init. For ease of description, v1 is used to refer to the existing interface, and v2 is used to refer to the layer of encapsulation. At present, at least 4 parameters are required to meet the user configuration requirements with the V1 interface function remains unchanged. @device_list: The available uacce device list. Users can get it by wd_get_accel_list(). @numa_bitmask: The bitmask provided by libnuma. Users can use this parameter to control requesting ctxs devices in the bind NUMA scenario. @ctx_nums: The requested ctx number for each numa node. Due to users may have different requirements for different types of ctx numbers, needs a two-dimensional array as input. @sched_type: Scheduling type the user wants to use. What's more, some users want uadk to provide the default value about input parameters for some performance insensitive scenes. C code has no way to. Yang Shen (5): uadk - support algorithms initialization reentry protect uadk - support return error number as pointer uadk - mv some function to header file uadk/comp - add wd_comp_init2 uadk/docs - support a simple interface for initialization Makefile.am | 4 +- docs/wd_alg_init2.md | 176 +++++++++++++++++++++++++++ include/wd.h | 54 ++++++++- include/wd_alg_common.h | 24 ++++ include/wd_comp.h | 28 +++++ include/wd_util.h | 57 +++++++++ wd.c | 97 ++++++++++++--- wd_aead.c | 33 +++-- wd_cipher.c | 35 ++++-- wd_comp.c | 259 ++++++++++++++++++++++++++++++++++++++-- wd_dh.c | 34 ++++-- wd_digest.c | 33 +++-- wd_ecc.c | 33 +++-- wd_rsa.c | 33 +++-- wd_util.c | 75 +++++++++++- 15 files changed, 870 insertions(+), 105 deletions(-) create mode 100644 docs/wd_alg_init2.md -- 2.24.0

2 7

帐户验证!
by openeuler.orgIT 06 Jul '22

06 Jul '22

1 0

通知acc@openeuler.org
by noreply 02 Jul '22

02 Jul '22

1 0

[PATCH 0/2] uadk/sched - make a numa node map
by Yang Shen 01 Jul '22

01 Jul '22

Now the uadk scheduler will search the nearest node when call session_sched_init(). But this is a repetitive work because session_sched_init() is called by wd_xxx_create_sess(). So we can build an cpu-device numa node map. Update the map when a new numa node has been enabled with a device. And then, session_sched_init() can quick get the right device id from a certain cpu. Yang Shen (2): uadk/sched - adjust some functions location and rename for tag uadk/sched - add mapping from cpus to devices wd_sched.c | 312 ++++++++++++++++++++++++++++------------------------- 1 file changed, 163 insertions(+), 149 deletions(-) -- 2.24.0

1 2

[PATCH v3] uacce: Handle parent device removal or parent driver module rmmod
by Zhangfei Gao 01 Jul '22

01 Jul '22

From: Jean-Philippe Brucker <jean-philippe(a)linaro.org> The uacce driver must deal with a possible removal of the parent device or parent driver module rmmod at any time. Although uacce_remove(), called on device removal and on driver unbind, prevents future use of the uacce fops by removing the cdev, fops that were called before that point may still be running. Serialize uacce_fops_open() and uacce_remove() with uacce->mutex. Serialize other fops against uacce_remove() with q->mutex. Since we need to protect uacce_fops_poll() which gets called on the fast path, replace uacce->queues_lock with q->mutex to improve scalability. The other fops are only used during setup. uacce_queue_is_valid(), checked under q->mutex or uacce->mutex, denotes whether uacce_remove() has disabled all queues. If that is the case, don't go any further since the parent device is being removed and uacce->ops should not be called anymore. Reported-by: Yang Shen <shenyang39(a)huawei.com> Signed-off-by: Zhangfei Gao <zhangfei.gao(a)linaro.org> Signed-off-by: Jean-Philippe Brucker <jean-philippe(a)linaro.org> --- v3: Drop patch 1, which set cdev owner with parent driver onwer, to block parent driver rmmod. Suggested from Greg, this behavior introduce layering violation. Only keep patch 2, which can cover both parent moduel rmmod and parent device removal. drivers/misc/uacce/uacce.c | 133 ++++++++++++++++++++++++------------- include/linux/uacce.h | 6 +- 2 files changed, 91 insertions(+), 48 deletions(-) diff --git a/drivers/misc/uacce/uacce.c b/drivers/misc/uacce/uacce.c index 281c54003edc..b70a013139c7 100644 --- a/drivers/misc/uacce/uacce.c +++ b/drivers/misc/uacce/uacce.c @@ -9,43 +9,38 @@ static struct class *uacce_class; static dev_t uacce_devt; -static DEFINE_MUTEX(uacce_mutex); static DEFINE_XARRAY_ALLOC(uacce_xa); -static int uacce_start_queue(struct uacce_queue *q) +/* + * If the parent driver or the device disappears, the queue state is invalid and + * ops are not usable anymore. + */ +static bool uacce_queue_is_valid(struct uacce_queue *q) { - int ret = 0; + return q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED; +} - mutex_lock(&uacce_mutex); +static int uacce_start_queue(struct uacce_queue *q) +{ + int ret; - if (q->state != UACCE_Q_INIT) { - ret = -EINVAL; - goto out_with_lock; - } + if (q->state != UACCE_Q_INIT) + return -EINVAL; if (q->uacce->ops->start_queue) { ret = q->uacce->ops->start_queue(q); if (ret < 0) - goto out_with_lock; + return ret; } q->state = UACCE_Q_STARTED; - -out_with_lock: - mutex_unlock(&uacce_mutex); - - return ret; + return 0; } static int uacce_put_queue(struct uacce_queue *q) { struct uacce_device *uacce = q->uacce; - mutex_lock(&uacce_mutex); - - if (q->state == UACCE_Q_ZOMBIE) - goto out; - if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue) uacce->ops->stop_queue(q); @@ -54,8 +49,6 @@ static int uacce_put_queue(struct uacce_queue *q) uacce->ops->put_queue(q); q->state = UACCE_Q_ZOMBIE; -out: - mutex_unlock(&uacce_mutex); return 0; } @@ -65,20 +58,36 @@ static long uacce_fops_unl_ioctl(struct file *filep, { struct uacce_queue *q = filep->private_data; struct uacce_device *uacce = q->uacce; + long ret = -ENXIO; + + /* + * uacce->ops->ioctl() may take the mmap_lock when copying arg to/from + * user. Avoid a circular lock dependency with uacce_fops_mmap(), which + * gets called with mmap_lock held, by taking uacce->mutex instead of + * q->mutex. Doing this in uacce_fops_mmap() is not possible because + * uacce_fops_open() calls iommu_sva_bind_device(), which takes + * mmap_lock, while holding uacce->mutex. + */ + mutex_lock(&uacce->mutex); + if (!uacce_queue_is_valid(q)) + goto out_unlock; switch (cmd) { case UACCE_CMD_START_Q: - return uacce_start_queue(q); - + ret = uacce_start_queue(q); + break; case UACCE_CMD_PUT_Q: - return uacce_put_queue(q); - + ret = uacce_put_queue(q); + break; default: - if (!uacce->ops->ioctl) - return -EINVAL; - - return uacce->ops->ioctl(q, cmd, arg); + if (uacce->ops->ioctl) + ret = uacce->ops->ioctl(q, cmd, arg); + else + ret = -EINVAL; } +out_unlock: + mutex_unlock(&uacce->mutex); + return ret; } #ifdef CONFIG_COMPAT @@ -136,6 +145,13 @@ static int uacce_fops_open(struct inode *inode, struct file *filep) if (!q) return -ENOMEM; + mutex_lock(&uacce->mutex); + + if (!uacce->parent) { + ret = -EINVAL; + goto out_with_mem; + } + ret = uacce_bind_queue(uacce, q); if (ret) goto out_with_mem; @@ -152,10 +168,9 @@ static int uacce_fops_open(struct inode *inode, struct file *filep) filep->private_data = q; uacce->inode = inode; q->state = UACCE_Q_INIT; - - mutex_lock(&uacce->queues_lock); + mutex_init(&q->mutex); list_add(&q->list, &uacce->queues); - mutex_unlock(&uacce->queues_lock); + mutex_unlock(&uacce->mutex); return 0; @@ -163,18 +178,20 @@ static int uacce_fops_open(struct inode *inode, struct file *filep) uacce_unbind_queue(q); out_with_mem: kfree(q); + mutex_unlock(&uacce->mutex); return ret; } static int uacce_fops_release(struct inode *inode, struct file *filep) { struct uacce_queue *q = filep->private_data; + struct uacce_device *uacce = q->uacce; - mutex_lock(&q->uacce->queues_lock); - list_del(&q->list); - mutex_unlock(&q->uacce->queues_lock); + mutex_lock(&uacce->mutex); uacce_put_queue(q); uacce_unbind_queue(q); + list_del(&q->list); + mutex_unlock(&uacce->mutex); kfree(q); return 0; @@ -217,10 +234,9 @@ static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) vma->vm_private_data = q; qfr->type = type; - mutex_lock(&uacce_mutex); - - if (q->state != UACCE_Q_INIT && q->state != UACCE_Q_STARTED) { - ret = -EINVAL; + mutex_lock(&q->mutex); + if (!uacce_queue_is_valid(q)) { + ret = -ENXIO; goto out_with_lock; } @@ -248,12 +264,12 @@ static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) } q->qfrs[type] = qfr; - mutex_unlock(&uacce_mutex); + mutex_unlock(&q->mutex); return ret; out_with_lock: - mutex_unlock(&uacce_mutex); + mutex_unlock(&q->mutex); kfree(qfr); return ret; } @@ -262,12 +278,20 @@ static __poll_t uacce_fops_poll(struct file *file, poll_table *wait) { struct uacce_queue *q = file->private_data; struct uacce_device *uacce = q->uacce; + __poll_t ret = 0; + + mutex_lock(&q->mutex); + if (!uacce_queue_is_valid(q)) + goto out_unlock; poll_wait(file, &q->wait, wait); + if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q)) - return EPOLLIN | EPOLLRDNORM; + ret = EPOLLIN | EPOLLRDNORM; - return 0; +out_unlock: + mutex_unlock(&q->mutex); + return ret; } static const struct file_operations uacce_fops = { @@ -450,7 +474,7 @@ struct uacce_device *uacce_alloc(struct device *parent, goto err_with_uacce; INIT_LIST_HEAD(&uacce->queues); - mutex_init(&uacce->queues_lock); + mutex_init(&uacce->mutex); device_initialize(&uacce->dev); uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id); uacce->dev.class = uacce_class; @@ -507,13 +531,23 @@ void uacce_remove(struct uacce_device *uacce) if (uacce->inode) unmap_mapping_range(uacce->inode->i_mapping, 0, 0, 1); + /* + * uacce_fops_open() may be running concurrently, even after we remove + * the cdev. Holding uacce->mutex ensures that open() does not obtain a + * removed uacce device. + */ + mutex_lock(&uacce->mutex); /* ensure no open queue remains */ - mutex_lock(&uacce->queues_lock); list_for_each_entry_safe(q, next_q, &uacce->queues, list) { + /* + * Taking q->mutex ensures that fops do not use the defunct + * uacce->ops after the queue is disabled. + */ + mutex_lock(&q->mutex); uacce_put_queue(q); + mutex_unlock(&q->mutex); uacce_unbind_queue(q); } - mutex_unlock(&uacce->queues_lock); /* disable sva now since no opened queues */ uacce_disable_sva(uacce); @@ -521,6 +555,13 @@ void uacce_remove(struct uacce_device *uacce) if (uacce->cdev) cdev_device_del(uacce->cdev, &uacce->dev); xa_erase(&uacce_xa, uacce->dev_id); + /* + * uacce exists as long as there are open fds, but ops will be freed + * now. Ensure that bugs cause NULL deref rather than use-after-free. + */ + uacce->ops = NULL; + uacce->parent = NULL; + mutex_unlock(&uacce->mutex); put_device(&uacce->dev); } EXPORT_SYMBOL_GPL(uacce_remove); diff --git a/include/linux/uacce.h b/include/linux/uacce.h index 48e319f40275..9ce88c28b0a8 100644 --- a/include/linux/uacce.h +++ b/include/linux/uacce.h @@ -70,6 +70,7 @@ enum uacce_q_state { * @wait: wait queue head * @list: index into uacce queues list * @qfrs: pointer of qfr regions + * @mutex: protects queue state * @state: queue state machine * @pasid: pasid associated to the mm * @handle: iommu_sva handle returned by iommu_sva_bind_device() @@ -80,6 +81,7 @@ struct uacce_queue { wait_queue_head_t wait; struct list_head list; struct uacce_qfile_region *qfrs[UACCE_MAX_REGION]; + struct mutex mutex; enum uacce_q_state state; u32 pasid; struct iommu_sva *handle; @@ -97,9 +99,9 @@ struct uacce_queue { * @dev_id: id of the uacce device * @cdev: cdev of the uacce * @dev: dev of the uacce + * @mutex: protects uacce operation * @priv: private pointer of the uacce * @queues: list of queues - * @queues_lock: lock for queues list * @inode: core vfs */ struct uacce_device { @@ -113,9 +115,9 @@ struct uacce_device { u32 dev_id; struct cdev *cdev; struct device dev; + struct mutex mutex; void *priv; struct list_head queues; - struct mutex queues_lock; struct inode *inode; }; -- 2.36.1

1 0

[PATCH 1/2] uadk/ecc - remove a redefined macro
by Yang Shen 01 Jul '22

01 Jul '22

Macro 'offsetof' has been defined in 'stddef.h'. So remove the redefined one in 'wd_ecc_drv.h'. Signed-off-by: Yang Shen <shenyang39(a)huawei.com> --- include/drv/wd_ecc_drv.h | 2 -- wd_ecc.c | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/include/drv/wd_ecc_drv.h b/include/drv/wd_ecc_drv.h index ef98606..857da20 100644 --- a/include/drv/wd_ecc_drv.h +++ b/include/drv/wd_ecc_drv.h @@ -45,8 +45,6 @@ extern "C" { #define WD_X448 0x2 #define WD_SM2P256 0x3 -#define offsetof(t, m) ((size_t)(uintptr_t)&((t *)0)->m) - /* ECC message format */ struct wd_ecc_msg { struct wd_ecc_req req; diff --git a/wd_ecc.c b/wd_ecc.c index 1ff6ed3..4cf287b 100644 --- a/wd_ecc.c +++ b/wd_ecc.c @@ -8,6 +8,7 @@ #include <errno.h> #include <pthread.h> #include <stdlib.h> +#include <stddef.h> #include <string.h> #include <time.h> #include <dlfcn.h> -- 2.24.0

1 1

Fwd: [RFC v2 00/15] vfio: Adopt iommufd
by Zhangfei Gao 29 Jun '22

29 Jun '22

qemu: https://github.com/luxis1999/qemu/tree/qemu-for-5.17-rc6-vm-rfcv2. kernel: >> https://github.com/luxis1999/iommufd/tree/iommufd-v5.18-rc4-nesting -------- Forwarded Message -------- Subject: [RFC v2 00/15] vfio: Adopt iommufd Date: Wed, 8 Jun 2022 05:31:24 -0700 From: Yi Liu <yi.l.liu(a)intel.com> To: alex.williamson(a)redhat.com, cohuck(a)redhat.com, qemu-devel(a)nongnu.org CC: david(a)gibson.dropbear.id.au, thuth(a)redhat.com, farman(a)linux.ibm.com, mjrosato(a)linux.ibm.com, akrowiak(a)linux.ibm.com, pasic(a)linux.ibm.com, jjherne(a)linux.ibm.com, jasowang(a)redhat.com, kvm(a)vger.kernel.org, jgg(a)nvidia.com, nicolinc(a)nvidia.com, eric.auger(a)redhat.com, eric.auger.pro(a)gmail.com, kevin.tian(a)intel.com, yi.l.liu(a)intel.com, chao.p.peng(a)intel.com, yi.y.sun(a)intel.com, peterx(a)redhat.com, shameerali.kolothum.thodi(a)huawei.com, zhangfei.gao(a)linaro.org, berrange(a)redhat.com With the introduction of iommufd[1], the Linux kernel provides a generic interface for userspace drivers to propagate their DMA mappings to kernel for assigned devices. This series does the porting of the VFIO devices onto the /dev/iommu uapi and let it coexist with the legacy implementation. Other devices like vpda, vfio mdev and etc. are not considered yet. At QEMU level, interactions with the /dev/iommu are abstracted by a new iommufd object (compiled in with the CONFIG_IOMMUFD option). Any QEMU device (e.g. vfio device) wishing to use /dev/iommu must be linked with an iommufd object. In this series, the vfio-pci device is granted with such capability: It gets a new optional parameter named iommufd which allows to pass an iommufd object: -object iommufd,id=iommufd0 -device vfio-pci,host=0000:02:00.0,iommufd=iommufd0 Note the /dev/iommu can be externally opened by a management layer. In such a case the fd is passed along with the iommufd object: -object iommufd,id=iommufd0,fd=22 -device vfio-pci,host=0000:02:00.0,iommufd=iommufd0 If the fd parameter is not passed, the fd (/dev/iommu) is opened by QEMU. If no iommufd option is passed to the vfio-pci device, iommufd is not used and the end-user gets the behavior based on the legacy vfio iommu interfaces: -device vfio-pci,host=0000:02:00.0 While the legacy kernel interface is group-centric, the new iommufd interface is device-centric, relying on device fd and iommufd. To support both interfaces in the QEMU VFIO device we reworked the vfio container abstraction so that the generic VFIO code can use either backend. The VFIOContainer object becomes a base object derived into a) the legacy VFIO container and b) the new iommufd based container. The base object implements generic code such as code related to memory_listener and address space management whereas the derived objects implement callbacks specific to either BE, legacy and iommufd. Indeed each backend has its own way to setup secure context and dma management interface. The below diagram shows how it looks like with both BEs. VFIO AddressSpace/Memory +-------+ +----------+ +-----+ +-----+ | pci | | platform | | ap | | ccw | +---+---+ +----+-----+ +--+--+ +--+--+ +----------------------+ | | | | | AddressSpace | | | | | +------------+---------+ +---V-----------V-----------V--------V----+ / | VFIOAddressSpace | <------------+ | | | MemoryListener | VFIOContainer list | +-------+----------------------------+----+ | | | | +-------V------+ +--------V----------+ | iommufd | | vfio legacy | | container | | container | +-------+------+ +--------+----------+ | | | /dev/iommu | /dev/vfio/vfio | /dev/vfio/devices/vfioX | /dev/vfio/$group_id Userspace | | ===========+============================+================================ Kernel | device fd | +---------------+ | group/container fd | (BIND_IOMMUFD | | (SET_CONTAINER/SET_IOMMU) | ATTACH_IOAS) | | device fd | | | | +-------V------------V-----------------+ iommufd | | vfio | (map/unmap | +---------+--------------------+-------+ ioas_copy) | | | map/unmap | | | +------V------+ +-----V------+ +------V--------+ | iommfd core | | device | | vfio iommu | +-------------+ +------------+ +---------------+ [Secure Context setup] - iommufd BE: uses device fd and iommufd to setup secure context (bind_iommufd, attach_ioas) - vfio legacy BE: uses group fd and container fd to setup secure context (set_container, set_iommu) [Device access] - iommufd BE: device fd is opened through /dev/vfio/devices/vfioX - vfio legacy BE: device fd is retrieved from group fd ioctl [DMA Mapping flow] - VFIOAddressSpace receives MemoryRegion add/del via MemoryListener - VFIO populates DMA map/unmap via the container BEs *) iommufd BE: uses iommufd *) vfio legacy BE: uses container fd Test done: - PCI and Platform device were tested - ccw and ap were only compile-tested - limited device hotplug test - vIOMMU test run for both legacy and iommufd backends (limited tests) This series was co-developed by Eric Auger and me based on the iommufd exploration kernel (https://github.com/luxis1999/iommufd/tree/iommufd-v5.17-rc6) which encompasses: - Jason’s IOMMUFD Generic interface series <https://lore.kernel.org/kvm/0-v1-e79cd8d168e8+6-iommufd_jgg@nvidia.com/> - Jason's vfio ccw life circle cleanup (Eric Farman has renewed it recently) <https://lore.kernel.org/all/0-v3-57c1502c62fd+2190-ccw_mdev_jgg@nvidia.com/ https://lore.kernel.org/kvm/20220602171948.2790690-1-farman@linux.ibm.com/> - Yi's move vfio_device allo/free into vfio core (not pushed to upstream yet) - Yi's vfio device cdev (introduces device-centric interface, not pushed to upstream yet) - Nicolin's vfio iommufd compat code (not pushed to upstream yet) This QEMU series can be fount at https://github.com/luxis1999/qemu/tree/qemu-for-5.17-rc6-vm-rfcv2. As iommufd kernel is in the early step (only iommufd generic interface is in mailing list), so this series hasn't made the iommufd backend fully on par with legacy backend w.r.t. features like: - p2p mappings - coherency tracking - live migration - vfio pci device hot reset - and etc. Only vfio-pci device supports iommufd BE, other types of devices still uses legacy BE. TODOs: - Add DMA alias check for iommufd BE (group level) - Make pci.c to be BE agnostic. Needs kernel change as well to fix the VFIO_DEVICE_PCI_HOT_RESET gap - Cleanup the VFIODevice fields as it's used in both backends - Add device fd parameter to vfio-device in case the iommufd option is used - Add locks - Replace list with g_tree - More tests Change log: v2: - remove the first three patches of rfcv1 - add open cdev helper suggested by Jason - remove the QOMification of the VFIOContainer and simply use standard ops (David) - add "-object iommufd" suggested by Alex v1: https://lore.kernel.org/qemu-devel/20220414104710.28534-1-yi.l.liu@intel.co… Thanks, Yi & Eric Eric Auger (10): scripts/update-linux-headers: Add iommufd.h linux-headers: Import latest vfio.h and iommufd.h vfio/container: Introduce vfio_[attach/detach]_device vfio/platform: Use vfio_[attach/detach]_device vfio/ap: Use vfio_[attach/detach]_device vfio/ccw: Use vfio_[attach/detach]_device vfio/container-base: Introduce [attach/detach]_device container callbacks vfio/container-base: Introduce VFIOContainer reset callback backends/iommufd: Introduce the iommufd object vfio/as: Allow the selection of a given iommu backend Yi Liu (5): vfio/common: Split common.c into common.c, container.c and as.c vfio: Add base container util/char_dev: Add open_cdev() vfio/iommufd: Implement the iommufd backend vfio/iommufd: Add IOAS_COPY_DMA support MAINTAINERS | 13 + backends/Kconfig | 5 + backends/iommufd.c | 265 +++ backends/meson.build | 1 + backends/trace-events | 12 + hw/vfio/ap.c | 62 +- hw/vfio/as.c | 1035 +++++++++++ hw/vfio/ccw.c | 118 +- hw/vfio/common.c | 2365 ++----------------------- hw/vfio/container-base.c | 172 ++ hw/vfio/container.c | 1298 ++++++++++++++ hw/vfio/iommufd.c | 546 ++++++ hw/vfio/meson.build | 6 + hw/vfio/migration.c | 5 +- hw/vfio/pci.c | 83 +- hw/vfio/platform.c | 42 +- hw/vfio/spapr.c | 22 +- hw/vfio/trace-events | 11 + include/hw/vfio/vfio-common.h | 101 +- include/hw/vfio/vfio-container-base.h | 148 ++ include/qemu/char_dev.h | 16 + include/sysemu/iommufd.h | 47 + linux-headers/linux/iommufd.h | 223 +++ linux-headers/linux/vfio.h | 84 + qapi/qom.json | 16 +- qemu-options.hx | 12 + scripts/update-linux-headers.sh | 2 +- util/chardev_open.c | 58 + util/meson.build | 1 + 29 files changed, 4291 insertions(+), 2478 deletions(-) create mode 100644 backends/iommufd.c create mode 100644 hw/vfio/as.c create mode 100644 hw/vfio/container-base.c create mode 100644 hw/vfio/container.c create mode 100644 hw/vfio/iommufd.c create mode 100644 include/hw/vfio/vfio-container-base.h create mode 100644 include/qemu/char_dev.h create mode 100644 include/sysemu/iommufd.h create mode 100644 linux-headers/linux/iommufd.h create mode 100644 util/chardev_open.c Base commit: 9b1f58854959c5a9bdb347e3e04c252ab7fc9ef5 -- 2.27.0

1 0

Fwd: [RFC v2 00/15] vfio: Adopt iommufd
by zhangfei.gao＠foxmail.com 29 Jun '22

29 Jun '22

qemu: https://github.com/luxis1999/qemu/tree/qemu-for-5.17-rc6-vm-rfcv2. kernel: >> https://github.com/luxis1999/iommufd/tree/iommufd-v5.18-rc4-nesting -------- Forwarded Message -------- Subject: [RFC v2 00/15] vfio: Adopt iommufd Date: Wed, 8 Jun 2022 05:31:24 -0700 From: Yi Liu <yi.l.liu(a)intel.com> To: alex.williamson(a)redhat.com, cohuck(a)redhat.com, qemu-devel(a)nongnu.org CC: david(a)gibson.dropbear.id.au, thuth(a)redhat.com, farman(a)linux.ibm.com, mjrosato(a)linux.ibm.com, akrowiak(a)linux.ibm.com, pasic(a)linux.ibm.com, jjherne(a)linux.ibm.com, jasowang(a)redhat.com, kvm(a)vger.kernel.org, jgg(a)nvidia.com, nicolinc(a)nvidia.com, eric.auger(a)redhat.com, eric.auger.pro(a)gmail.com, kevin.tian(a)intel.com, yi.l.liu(a)intel.com, chao.p.peng(a)intel.com, yi.y.sun(a)intel.com, peterx(a)redhat.com, shameerali.kolothum.thodi(a)huawei.com, zhangfei.gao(a)linaro.org, berrange(a)redhat.com With the introduction of iommufd[1], the Linux kernel provides a generic interface for userspace drivers to propagate their DMA mappings to kernel for assigned devices. This series does the porting of the VFIO devices onto the /dev/iommu uapi and let it coexist with the legacy implementation. Other devices like vpda, vfio mdev and etc. are not considered yet. At QEMU level, interactions with the /dev/iommu are abstracted by a new iommufd object (compiled in with the CONFIG_IOMMUFD option). Any QEMU device (e.g. vfio device) wishing to use /dev/iommu must be linked with an iommufd object. In this series, the vfio-pci device is granted with such capability: It gets a new optional parameter named iommufd which allows to pass an iommufd object: -object iommufd,id=iommufd0 -device vfio-pci,host=0000:02:00.0,iommufd=iommufd0 Note the /dev/iommu can be externally opened by a management layer. In such a case the fd is passed along with the iommufd object: -object iommufd,id=iommufd0,fd=22 -device vfio-pci,host=0000:02:00.0,iommufd=iommufd0 If the fd parameter is not passed, the fd (/dev/iommu) is opened by QEMU. If no iommufd option is passed to the vfio-pci device, iommufd is not used and the end-user gets the behavior based on the legacy vfio iommu interfaces: -device vfio-pci,host=0000:02:00.0 While the legacy kernel interface is group-centric, the new iommufd interface is device-centric, relying on device fd and iommufd. To support both interfaces in the QEMU VFIO device we reworked the vfio container abstraction so that the generic VFIO code can use either backend. The VFIOContainer object becomes a base object derived into a) the legacy VFIO container and b) the new iommufd based container. The base object implements generic code such as code related to memory_listener and address space management whereas the derived objects implement callbacks specific to either BE, legacy and iommufd. Indeed each backend has its own way to setup secure context and dma management interface. The below diagram shows how it looks like with both BEs. VFIO AddressSpace/Memory +-------+ +----------+ +-----+ +-----+ | pci | | platform | | ap | | ccw | +---+---+ +----+-----+ +--+--+ +--+--+ +----------------------+ | | | | | AddressSpace | | | | | +------------+---------+ +---V-----------V-----------V--------V----+ / | VFIOAddressSpace | <------------+ | | | MemoryListener | VFIOContainer list | +-------+----------------------------+----+ | | | | +-------V------+ +--------V----------+ | iommufd | | vfio legacy | | container | | container | +-------+------+ +--------+----------+ | | | /dev/iommu | /dev/vfio/vfio | /dev/vfio/devices/vfioX | /dev/vfio/$group_id Userspace | | ===========+============================+================================ Kernel | device fd | +---------------+ | group/container fd | (BIND_IOMMUFD | | (SET_CONTAINER/SET_IOMMU) | ATTACH_IOAS) | | device fd | | | | +-------V------------V-----------------+ iommufd | | vfio | (map/unmap | +---------+--------------------+-------+ ioas_copy) | | | map/unmap | | | +------V------+ +-----V------+ +------V--------+ | iommfd core | | device | | vfio iommu | +-------------+ +------------+ +---------------+ [Secure Context setup] - iommufd BE: uses device fd and iommufd to setup secure context (bind_iommufd, attach_ioas) - vfio legacy BE: uses group fd and container fd to setup secure context (set_container, set_iommu) [Device access] - iommufd BE: device fd is opened through /dev/vfio/devices/vfioX - vfio legacy BE: device fd is retrieved from group fd ioctl [DMA Mapping flow] - VFIOAddressSpace receives MemoryRegion add/del via MemoryListener - VFIO populates DMA map/unmap via the container BEs *) iommufd BE: uses iommufd *) vfio legacy BE: uses container fd Test done: - PCI and Platform device were tested - ccw and ap were only compile-tested - limited device hotplug test - vIOMMU test run for both legacy and iommufd backends (limited tests) This series was co-developed by Eric Auger and me based on the iommufd exploration kernel (https://github.com/luxis1999/iommufd/tree/iommufd-v5.17-rc6) which encompasses: - Jason’s IOMMUFD Generic interface series <https://lore.kernel.org/kvm/0-v1-e79cd8d168e8+6-iommufd_jgg@nvidia.com/> - Jason's vfio ccw life circle cleanup (Eric Farman has renewed it recently) <https://lore.kernel.org/all/0-v3-57c1502c62fd+2190-ccw_mdev_jgg@nvidia.com/ https://lore.kernel.org/kvm/20220602171948.2790690-1-farman@linux.ibm.com/> - Yi's move vfio_device allo/free into vfio core (not pushed to upstream yet) - Yi's vfio device cdev (introduces device-centric interface, not pushed to upstream yet) - Nicolin's vfio iommufd compat code (not pushed to upstream yet) This QEMU series can be fount at https://github.com/luxis1999/qemu/tree/qemu-for-5.17-rc6-vm-rfcv2. As iommufd kernel is in the early step (only iommufd generic interface is in mailing list), so this series hasn't made the iommufd backend fully on par with legacy backend w.r.t. features like: - p2p mappings - coherency tracking - live migration - vfio pci device hot reset - and etc. Only vfio-pci device supports iommufd BE, other types of devices still uses legacy BE. TODOs: - Add DMA alias check for iommufd BE (group level) - Make pci.c to be BE agnostic. Needs kernel change as well to fix the VFIO_DEVICE_PCI_HOT_RESET gap - Cleanup the VFIODevice fields as it's used in both backends - Add device fd parameter to vfio-device in case the iommufd option is used - Add locks - Replace list with g_tree - More tests Change log: v2: - remove the first three patches of rfcv1 - add open cdev helper suggested by Jason - remove the QOMification of the VFIOContainer and simply use standard ops (David) - add "-object iommufd" suggested by Alex v1: https://lore.kernel.org/qemu-devel/20220414104710.28534-1-yi.l.liu@intel.co… Thanks, Yi & Eric Eric Auger (10): scripts/update-linux-headers: Add iommufd.h linux-headers: Import latest vfio.h and iommufd.h vfio/container: Introduce vfio_[attach/detach]_device vfio/platform: Use vfio_[attach/detach]_device vfio/ap: Use vfio_[attach/detach]_device vfio/ccw: Use vfio_[attach/detach]_device vfio/container-base: Introduce [attach/detach]_device container callbacks vfio/container-base: Introduce VFIOContainer reset callback backends/iommufd: Introduce the iommufd object vfio/as: Allow the selection of a given iommu backend Yi Liu (5): vfio/common: Split common.c into common.c, container.c and as.c vfio: Add base container util/char_dev: Add open_cdev() vfio/iommufd: Implement the iommufd backend vfio/iommufd: Add IOAS_COPY_DMA support MAINTAINERS | 13 + backends/Kconfig | 5 + backends/iommufd.c | 265 +++ backends/meson.build | 1 + backends/trace-events | 12 + hw/vfio/ap.c | 62 +- hw/vfio/as.c | 1035 +++++++++++ hw/vfio/ccw.c | 118 +- hw/vfio/common.c | 2365 ++----------------------- hw/vfio/container-base.c | 172 ++ hw/vfio/container.c | 1298 ++++++++++++++ hw/vfio/iommufd.c | 546 ++++++ hw/vfio/meson.build | 6 + hw/vfio/migration.c | 5 +- hw/vfio/pci.c | 83 +- hw/vfio/platform.c | 42 +- hw/vfio/spapr.c | 22 +- hw/vfio/trace-events | 11 + include/hw/vfio/vfio-common.h | 101 +- include/hw/vfio/vfio-container-base.h | 148 ++ include/qemu/char_dev.h | 16 + include/sysemu/iommufd.h | 47 + linux-headers/linux/iommufd.h | 223 +++ linux-headers/linux/vfio.h | 84 + qapi/qom.json | 16 +- qemu-options.hx | 12 + scripts/update-linux-headers.sh | 2 +- util/chardev_open.c | 58 + util/meson.build | 1 + 29 files changed, 4291 insertions(+), 2478 deletions(-) create mode 100644 backends/iommufd.c create mode 100644 hw/vfio/as.c create mode 100644 hw/vfio/container-base.c create mode 100644 hw/vfio/container.c create mode 100644 hw/vfio/iommufd.c create mode 100644 include/hw/vfio/vfio-container-base.h create mode 100644 include/qemu/char_dev.h create mode 100644 include/sysemu/iommufd.h create mode 100644 linux-headers/linux/iommufd.h create mode 100644 util/chardev_open.c Base commit: 9b1f58854959c5a9bdb347e3e04c252ab7fc9ef5 -- 2.27.0

1 0

2024

2023

2022

Acc